Merge pull request iovisor#2149 from iovisor/yhs_dev

use kernel libbpf in bcc

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/cc/libbpf"]
+ path = src/cc/libbpf
+ url = https://github.com/libbpf/libbpf.git

CMakeLists.txt

@@ -9,6 +9,12 @@ endif()
 
 enable_testing()
 
+# populate submodules (libbpf)
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/cc/libbpf/src)
+ execute_process(COMMAND git submodule update --init --recursive
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+endif()
+
 include(cmake/GetGitRevisionDescription.cmake)
 include(cmake/version.cmake)
 include(CMakeDependentOption)

cmake/FindCompilerFlag.cmake

@@ -15,3 +15,16 @@ else()
  endif()
  set(CMAKE_REQUIRED_FLAGS "${_backup_c_flags}")
 endif()
+
+# check whether reallocarray availability
+# this is used to satisfy reallocarray usage under src/cc/libbpf/
+CHECK_CXX_SOURCE_COMPILES(
+"
+#define _GNU_SOURCE
+#include <stdlib.h>
+
+int main(void)
+{
+ return !!reallocarray(NULL, 1, 1);
+}
+" HAVE_REALLOCARRAY_SUPPORT)

examples/cpp/CMakeLists.txt

@@ -3,6 +3,7 @@
 
 include_directories(${CMAKE_SOURCE_DIR}/src/cc)
 include_directories(${CMAKE_SOURCE_DIR}/src/cc/api)
+include_directories(${CMAKE_SOURCE_DIR}/src/cc/libbpf/include/uapi)
 
 option(INSTALL_CPP_EXAMPLES "Install C++ examples. Those binaries are statically linked and can take plenty of disk space" OFF)
 

examples/cpp/UseExternalMap.cc

@@ -2,7 +2,7 @@
  * UseExternalMap shows how to access an external map through
  * C++ interface. The external map could be a pinned map.
  * This example simulates the pinned map through a locally
- * created map by calling libbpf bpf_create_map.
+ * created map by calling libbpf bcc_create_map.
  *
  * Copyright (c) Facebook, Inc.
  * Licensed under the Apache License, Version 2.0 (the "License")
@@ -79,10 +79,10 @@ int main() {
  int ctrl_map_fd;
  uint32_t val;
 
- // create a map through bpf_create_map, bcc knows nothing about this map.
- ctrl_map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, "control", sizeof(uint32_t),
+ // create a map through bcc_create_map, bcc knows nothing about this map.
+ ctrl_map_fd = bcc_create_map(BPF_MAP_TYPE_ARRAY, "control", sizeof(uint32_t),
  sizeof(uint32_t), 1, 0);
- CHECK(ctrl_map_fd < 0, "bpf_create_map failure");
+ CHECK(ctrl_map_fd < 0, "bcc_create_map failure");
 
  // populate control map into TableStorage
  std::unique_ptr<ebpf::TableStorage> local_ts =

introspection/CMakeLists.txt

@@ -3,6 +3,7 @@
 
 include_directories(${CMAKE_SOURCE_DIR}/src/cc)
 include_directories(${CMAKE_SOURCE_DIR}/src/cc/api)
+include_directories(${CMAKE_SOURCE_DIR}/src/cc/libbpf/include/uapi)
 
 option(INSTALL_INTROSPECTION "Install BPF introspection tools" ON)
 

scripts/build-deb.sh

@@ -16,11 +16,24 @@ function cleanup() {
 }
 trap cleanup EXIT
 
+# populate submodules
+git submodule update --init --recursive
+
 . scripts/git-tag.sh
 
-git archive HEAD --prefix=bcc/ --format=tar.gz -o $TMP/bcc_$revision.orig.tar.gz
+git archive HEAD --prefix=bcc/ --format=tar -o $TMP/bcc_$revision.orig.tar
+
+# archive submodules
+pushd src/cc/libbpf
+git archive HEAD --prefix=bcc/src/cc/libbpf/ --format=tar -o $TMP/bcc_libbpf_$revision.orig.tar
+popd
 
 pushd $TMP
+
+# merge all archives into bcc_$revision.orig.tar.gz
+tar -A -f bcc_$revision.orig.tar bcc_libbpf_$revision.orig.tar
+gzip bcc_$revision.orig.tar
+
 tar xf bcc_$revision.orig.tar.gz
 cd bcc
 

scripts/build-release-rpm.sh

@@ -14,9 +14,24 @@ mkdir $TMP/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
 
 llvmver=3.7.1
 
+# populate submodules
+git submodule update --init --recursive
+
 . scripts/git-tag.sh
 
-git archive HEAD --prefix=bcc/ --format=tar.gz -o $TMP/SOURCES/$git_tag_latest.tar.gz
+git archive HEAD --prefix=bcc/ --format=tar -o $TMP/SOURCES/bcc.tar
+
+# archive submodules
+pushd src/cc/libbpf
+git archive HEAD --prefix=bcc/src/cc/libbpf/ --format=tar -o $TMP/SOURCES/bcc_libbpf.tar
+popd
+
+# merge all archives into $git_tag_latest.tar.gz
+pushd $TMP/SOURCES
+tar -A -f bcc.tar bcc_libbpf.tar
+gzip -c bcc.tar > $git_tag_latest.tar.gz
+popd
+
 wget -P $TMP/SOURCES http:https://llvm.org/releases/$llvmver/{cfe,llvm}-$llvmver.src.tar.xz
 
 sed \

scripts/build-rpm.sh

@@ -14,9 +14,23 @@ mkdir $TMP/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
 
 llvmver=3.7.1
 
+# populate submodules
+git submodule update --init --recursive
+
 . scripts/git-tag.sh
 
-git archive HEAD --prefix=bcc/ --format=tar.gz -o $TMP/SOURCES/bcc.tar.gz
+git archive HEAD --prefix=bcc/ --format=tar -o $TMP/SOURCES/bcc.tar
+
+# archive submodules
+pushd src/cc/libbpf
+git archive HEAD --prefix=bcc/src/cc/libbpf/ --format=tar -o $TMP/SOURCES/bcc_libbpf.tar
+popd
+
+# merge all archives into bcc.tar.gz
+pushd $TMP/SOURCES
+tar -A -f bcc.tar bcc_libbpf.tar
+gzip bcc.tar
+popd
 
 sed \
  -e "s/^\(Version:\s*\)@REVISION@/\1$revision/" \

src/cc/CMakeLists.txt

@@ -10,22 +10,28 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/frontends/clang)
 include_directories(${LLVM_INCLUDE_DIRS})
 include_directories(${LIBELF_INCLUDE_DIRS})
 # todo: if check for kernel version
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/compat)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/libbpf/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/libbpf/include/uapi)
 add_definitions(${LLVM_DEFINITIONS})
 configure_file(libbcc.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libbcc.pc @ONLY)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -DBCC_PROG_TAG_DIR='\"${BCC_PROG_TAG_DIR}\"'")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -Wno-unused-result")
 
+if (NOT HAVE_REALLOCARRAY_SUPPORT)
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCOMPAT_NEED_REALLOCARRAY")
+endif()
+
 string(REGEX MATCH "^([0-9]+).*" _ ${LLVM_PACKAGE_VERSION})
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLLVM_MAJOR_VERSION=${CMAKE_MATCH_1}")
 
 include(static_libstdc++)
 
-add_library(bpf-static STATIC libbpf.c perf_reader.c)
+file(GLOB libbpf_sources "libbpf/src/*.c")
+add_library(bpf-static STATIC libbpf.c perf_reader.c ${libbpf_sources})
 set_target_properties(bpf-static PROPERTIES OUTPUT_NAME bpf)
-add_library(bpf-shared SHARED libbpf.c perf_reader.c)
+add_library(bpf-shared SHARED libbpf.c perf_reader.c ${libbpf_sources})
 set_target_properties(bpf-shared PROPERTIES VERSION ${REVISION_LAST} SOVERSION 0)
 set_target_properties(bpf-shared PROPERTIES OUTPUT_NAME bpf)
 
@@ -105,7 +111,7 @@ set(bcc-lua-static ${bcc-lua-static} ${bcc_common_libs_for_lua})
 install(TARGETS bcc-shared LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES ${bcc_table_headers} DESTINATION include/bcc)
 install(FILES ${bcc_api_headers} DESTINATION include/bcc)
-install(DIRECTORY compat/linux/ DESTINATION include/bcc/compat/linux FILES_MATCHING PATTERN "*.h")
+install(DIRECTORY libbpf/include/uapi/linux/ DESTINATION include/bcc/compat/linux FILES_MATCHING PATTERN "*.h")
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libbcc.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 endif(ENABLE_CLANG_JIT)
 install(FILES ${bcc_common_headers} DESTINATION include/bcc)

src/cc/README

@@ -0,0 +1,16 @@
+The libbpf directory is a git submodule for repository
+ https://github.com/libbpf/libbpf
+
+If you have any change in libbpf directory, please upstream to linux
+first as libbpf repo is a mirror of linux/tools/lib/bpf directory.
+
+If any top-commit update of libbpf submodule contains a uapi header
+change, the following are necessary steps to sync properly with
+rest of bcc:
+ 1. sync compat/linux/virtual_bpf.h with libbpf/include/uapi/linux/bpf.h
+ as virtual_bpf.h has an extra string wrapper for bpf.h.
+ 2. if new bpf.h has new helpers, add corresponding helper func define
+ in bcc:src/cc/export/helpers.h and helper entry for error reporting
+ in bcc:src/cc/libbpf.c.
+ 3. if new bpf.h has new map types, program types, update
+ bcc:introspection/bps.c for these new map/program types.

src/cc/api/BPF.cc

@@ -550,7 +550,7 @@ StatusTuple BPF::load_func(const std::string& func_name, bpf_prog_type type,
  else if (flag_ & DEBUG_BPF)
  log_level = 1;
 
- fd = bpf_prog_load(type, func_name.c_str(),
+ fd = bcc_prog_load(type, func_name.c_str(),
  reinterpret_cast<struct bpf_insn*>(func_start), func_size,
  bpf_module_->license(), bpf_module_->kern_version(),
  log_level, nullptr, 0);

src/cc/api/BPF.h

@@ -26,7 +26,7 @@
 #include "bcc_exception.h"
 #include "bcc_syms.h"
 #include "bpf_module.h"
-#include "compat/linux/bpf.h"
+#include "linux/bpf.h"
 #include "libbpf.h"
 #include "table_storage.h"
 

src/cc/bcc_syms.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 #include <stdint.h>
-#include "compat/linux/bpf.h"
+#include "linux/bpf.h"
 
 struct bcc_symbol {
  const char *name;