Merge remote-tracking branch 'origin/master' into iamkafai

.clang-format

@@ -1,5 +1,6 @@
 ---
 BasedOnStyle: Google
 AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
 IndentCaseLabels: false
-AccessModifierOffset: -2
+AccessModifierOffset: -1

.gitignore

@@ -2,6 +2,10 @@
 *.swp
 *.swo
 *.pyc
+.idea
 
-# Build artefacts
+# Build artifacts
 /build/
+cmake-build-debug
+debian/**/*.log
+obj-x86_64-linux-gnu

.travis.yml

@@ -0,0 +1,6 @@
+language: generic
+install:
+ - sudo apt-get install -y python-pip
+ - sudo pip install pep8
+script:
+ - find tools/ -type f -name "*.py" | xargs pep8 -r --show-source --ignore=E123,E125,E126,E127,E128,E302

CMakeLists.txt

@@ -3,21 +3,31 @@
 cmake_minimum_required(VERSION 2.8.7)
 
 project(bcc)
-set(CMAKE_BUILD_TYPE Release)
+if(NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE Release)
+endif()
 
 enable_testing()
 
 include(cmake/GetGitRevisionDescription.cmake)
 include(cmake/version.cmake)
+include(CMakeDependentOption)
 include(GNUInstallDirs)
+include(CheckCXXCompilerFlag)
+include(cmake/FindCompilerFlag.cmake)
+
+option(ENABLE_CLANG_JIT "Enable Loading BPF through Clang Frontend" ON)
+option(ENABLE_USDT "Enable User-level Statically Defined Tracing" ON)
+CMAKE_DEPENDENT_OPTION(ENABLE_CPP_API "Enable C++ API" ON "ENABLE_USDT" OFF)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
-if(NOT PYTHON_ONLY)
+if(NOT PYTHON_ONLY AND ENABLE_CLANG_JIT)
 find_package(BISON)
 find_package(FLEX)
 find_package(LLVM REQUIRED CONFIG)
-message(STATUS "Found LLVM: ${LLVM_INCLUDE_DIRS}")
+message(STATUS "Found LLVM: ${LLVM_INCLUDE_DIRS} ${LLVM_PACKAGE_VERSION}")
+find_package(LibElf REQUIRED)
 
 # clang is linked as a library, but the library path searching is
 # primitively supported, unlike libLLVM
@@ -47,30 +57,35 @@ if(NOT DEFINED BCC_KERNEL_MODULES_DIR)
  set(BCC_KERNEL_MODULES_DIR "/lib/modules")
 endif()
 
-find_package(LibElf REQUIRED)
-
-# Set to non-zero if system installs kernel headers with split source and build
-# directories in /lib/modules/`uname -r`/. This is the case for debian and
-# suse, to the best of my knowledge.
-if(BCC_KERNEL_HAS_SOURCE_DIR)
- set(BCC_KERNEL_HAS_SOURCE_DIR 1)
- set(BCC_KERNEL_MODULES_SUFFIX "source")
-else()
- set(BCC_KERNEL_HAS_SOURCE_DIR 0)
+if(NOT DEFINED BCC_PROG_TAG_DIR)
+ set(BCC_PROG_TAG_DIR "/var/tmp/bcc")
 endif()
 
-# Similar to above, set to custom value if kernel headers in
-# /lib/modules/`uname -r` sit in a different location than build/.
-if(NOT DEFINED BCC_KERNEL_MODULES_SUFFIX)
- set(BCC_KERNEL_MODULES_SUFFIX "build")
+# As reported in issue #735, GCC 6 has some behavioral problems when
+# dealing with -isystem. Hence, skip the warning optimization
+# altogether on that compiler.
+option(USINGISYSTEM "using -isystem" ON)
+execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
+if (USINGISYSTEM AND GCC_VERSION VERSION_LESS 6.0)
+ # iterate over all available directories in LLVM_INCLUDE_DIRS to
+ # generate a correctly tokenized list of parameters
+ foreach(ONE_LLVM_INCLUDE_DIR ${LLVM_INCLUDE_DIRS})
+ set(CXX_ISYSTEM_DIRS "${CXX_ISYSTEM_DIRS} -isystem ${ONE_LLVM_INCLUDE_DIR}")
+ endforeach()
 endif()
 
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_STANDARD 11)
+
+endif(NOT PYTHON_ONLY AND ENABLE_CLANG_JIT)
+
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall")
-endif()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall ${CXX_ISYSTEM_DIRS}")
 
+add_subdirectory(src)
+if(ENABLE_CLANG_JIT)
 add_subdirectory(examples)
 add_subdirectory(man)
-add_subdirectory(src)
 add_subdirectory(tests)
 add_subdirectory(tools)
+endif(ENABLE_CLANG_JIT)

CODEOWNERS

@@ -0,0 +1,25 @@
+# This file should be kept up to date with the list of maintainers responsible
+# for the different subdirectories within BCC. One of these people SHOULD
+# review code that touches the respective areas, and MUST review it if the
+# change is substantial or API-breaking.
+
+# see https://help.github.com/articles/about-codeowners/ for syntax
+
+# Miscellaneous
+* @drzaeus77 @goldshtn @yonghong-song @4ast @brendangregg
+
+# Documentation
+/docs/ @brendangregg @goldshtn
+/man/ @brendangregg @goldshtn
+
+# Tools
+/tools/ @brendangregg @goldshtn
+
+# Compiler, C API
+/src/cc/ @drzaeus77 @yonghong-song @4ast
+
+# Python API
+/src/python/ @drzaeus77 @goldshtn
+
+# Tests
+/tests/ @drzaeus77 @yonghong-song

CONTRIBUTING-SCRIPTS.md

@@ -15,7 +15,7 @@ More detail for each below.
 
 ## Examples
 
-These are grouped into subdirectories (networking, tracing). Your example can either be a Python program with embedded C (eg, tracing/strlen_count.py), or separate Python and C files (eg, tracing/bitehist.*).
+These are grouped into subdirectories (networking, tracing). Your example can either be a Python program with embedded C (eg, tracing/strlen_count.py), or separate Python and C files (eg, tracing/vfsreadlat.*).
 
 As said earlier: keep it short, neat, and documented (code comments).
 
@@ -31,11 +31,14 @@ A checklist for bcc tool development:
 1. **Measure the overhead of the tool**. If you are running a micro-benchmark, how much slower is it with the tool running. Is more CPU consumed? Try to determine the worst case: run the micro-benchmark so that CPU headroom is exhausted, and then run the bcc tool. Can overhead be lowered?
 1. **Test again, and stress test**. You want to discover and fix all the bad things before others hit them.
 1. **Consider command line options**. Should it have -p for filtering on a PID? -T for timestamps? -i for interval? See other tools for examples, and copy the style: the usage message should list example usage at the end. Remember to keep the tool doing one thing and doing it well. Also, if there's one option that seems to be the common case, perhaps it should just be the first argument and not need a switch (no -X). A special case of this is *stat tools, like iostat/vmstat/etc, where the convention is [interval [count]].
+1. **Concise, intuitive, self-explanatory output**. The default output should meet the common need concisely. Leave much less useful fields and data to be shown with options: -v for verbose, etc. Consider including a startup message that's self-explanatory, eg "Tracing block I/O. Output every 1 seconds. Ctrl-C to end.". Also, try hard to keep the output less than 80 characters wide, especially the default output of the tool. That way, the output not only fits on the smallest reasonable terminal, it also fits well in slide decks, blog posts, articles, and printed material, all of which help education and adoption. Publishers of technical books often have templates they require books to conform to: it may not be an option to shrink or narrow the font to fit your output.
 1. **Use pep8 to check Python style**: pep8 --show-source --ignore=E123,E125,E126,E127,E128,E302 filename . Note that it misses some things, like consistent usage, so you'll still need to double check your script.
+1. **Make sure your script is Python3-ready**: Adding `from __future__ import absolute_import, division, print_function, unicode_literals` helps make your script Python3-ready.
 1. **Write an _example.txt file**. Copy the style in tools/biolatency_example.txt: start with an intro sentence, then have examples, and finish with the USAGE message. Explain everything: the first example should explain what we are seeing, even if this seems obvious. For some people it won't be obvious. Also explain why we are running the tool: what problems it's solving. It can take a long time (hours) to come up with good examples, but it's worth it. These will get copied around (eg, presentations, articles).
 1. **Read your example.txt file**. Does this sound too niche or convoluted? Are you spending too much time explaining caveats? These can be hints that perhaps you should fix your tool, or abandon it! Perhaps it better belongs as an /example, and not a tool. I've abandoned many tools at this stage.
 1. **Write a man page**. Either ROFF (.8), markdown (.md), or plain text (.txt): so long as it documents the important sections, particularly columns (fields) and caveats. These go under man/man8. See the other examples. Include a section on overhead, and pull no punches. It's better for end users to know about high overhead beforehand, than to discover it the hard way. Also explain caveats. Don't assume those will be obvious to tool users.
 1. **Read your man page**. For ROFF: nroff -man filename. Like before, this exercise is like saying something out loud. Does it sound too niche or convoluted? Again, hints that you might need to go back and fix things, or abandon it.
 1. **Spell check your documentation**. Use a spell checker like aspell to check your document quality before committing.
 1. **Add an entry to README.md**.
+1. **Add a smoke test** to [test_tools_smoke.py](https://github.com/iovisor/bcc/blob/master/tests/python/test_tools_smoke.py), which serves as a basic check that your tool still works when we make changes to the core library.
 1. If you made it this far, pull request!

Dockerfile.debian

@@ -1,14 +1,17 @@
-FROM debian:jessie
+FROM debian:stretch
 
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv 15CF4D18AF4F7421 && \
- echo "deb http:https://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" > /etc/apt/sources.list.d/llvm.list && \
- apt-get update && \
- apt-get install -y --no-install-recommends build-essential fakeroot bison cmake debhelper devscripts flex git libedit-dev python zlib1g-dev libllvm3.8 llvm-3.8-dev libclang-3.8-dev libelf-dev luajit libluajit-5.1-dev && \
- mkdir -p /usr/share/llvm-3.8 && \
- ln -s /usr/lib/llvm-3.8/share/llvm/cmake /usr/share/llvm-3.8/cmake
+MAINTAINER Brenden Blanco <[email protected]>
+
+RUN DEBIAN_RELEASE=stretch && \
+ # Adding non-free repo for netperf
+ echo "deb http:https://deb.debian.org/debian ${DEBIAN_RELEASE} non-free" > \
+ /etc/apt/sources.list.d/debian-non-free.list && \
+ apt-get -qq update && \
+ apt-get -y install pbuilder aptitude
 
 COPY ./ /root/bcc
 
 WORKDIR /root/bcc
 
-RUN ./scripts/build-deb.sh
+RUN /usr/lib/pbuilder/pbuilder-satisfydepends && \
+ ./scripts/build-deb.sh

Dockerfile.ubuntu

@@ -1,20 +1,13 @@
-# File to be used for building an Ubuntu .deb
+FROM ubuntu:xenial
 
-FROM ubuntu:trusty
+MAINTAINER Brenden Blanco <[email protected]>
 
-MAINTAINER Brenden Blanco <[email protected]>
+RUN apt-get -qq update && \
+ apt-get -y install pbuilder aptitude
 
-RUN apt-get -y install wget
-RUN printf "deb http:https://llvm.org/apt/trusty/ llvm-toolchain-trusty main\ndeb-src http:https://llvm.org/apt/trusty/ llvm-toolchain-trusty main\n" > /etc/apt/sources.list.d/llvm.list
-RUN wget -q -O - http:https://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
-RUN apt-get -y update
+COPY ./ /root/bcc
 
-RUN apt-get -y install bison build-essential cmake debhelper devscripts flex git libedit-dev python zlib1g-dev
-RUN apt-get -y install libllvm3.8 llvm-3.8-dev libclang-3.8-dev
-
-RUN mkdir -p /root/bcc/build
-COPY ./ /root/bcc/
-WORKDIR /root
-RUN tar zcf bcc_0.1.1.orig.tar.gz bcc/
 WORKDIR /root/bcc
-RUN DEB_BUILD_OPTIONS="nocheck parallel=4" debuild -us -uc
+
+RUN /usr/lib/pbuilder/pbuilder-satisfydepends && \
+ ./scripts/build-deb.sh