loader: Add support for reading kernel headers from /proc

BCC right now relies on the filesystem to have kernel headers however
this is quite a hindrance to embedded and Android systems, and even
distros that may not have the kernel headers for the running kernel.
This patch makes use of the new /proc/kheaders.tar.xz archive that is
proposed upstream: https://lore.kernel.org/patchwork/patch/1059427/

The approach involves creating a temporary directory containing the
headers and compiling from there. It is used as a last resort if headers
could not be found by other means.

Testing shows this adds around 400ms to start up time of BCC, however
the cost is 0 if the headers already exists in /lib or was previously
extracted into /tmp directory.

Signed-off-by: Joel Fernandes (Google) <[email protected]>

src/cc/frontends/clang/kbuild_helper.cc

@@ -13,9 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <fstream>
+#include <iostream>
+
+#include <dirent.h>
 #include <fcntl.h>
 #include <stdlib.h>
-#include <iostream>
+#include <sys/utsname.h>
+#include <unistd.h>
+
 #include "kbuild_helper.h"
 
 namespace ebpf {
@@ -107,4 +113,82 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
  return 0;
 }
 
+static inline int file_exists(const char *f)
+{
+ struct stat buffer;
+ return (stat(f, &buffer) == 0);
+}
+
+static inline int proc_kheaders_exists(void)
+{
+ return file_exists(PROC_KHEADERS_PATH);
+}
+
+static inline int extract_kheaders(const std::string &dirpath,
+ const struct utsname &uname_data)
+{
+ char tar_cmd[256], dirpath_tmp[256];
+ int ret;
+ bool module = false;
+
+ if (!proc_kheaders_exists()) {
+ ret = system("modprobe kheaders");
+ if (ret)
+ return ret;
+ module = true;
+ if (!proc_kheaders_exists()) {
+ ret = -1;
+ goto cleanup;
+ }
+ }
+
+ snprintf(dirpath_tmp, 256, "/tmp/kheaders-%s-XXXXXX", uname_data.release);
+ if (mkdtemp(dirpath_tmp) == NULL) {
+ ret = -1;
+ goto cleanup;
+ }
+
+ snprintf(tar_cmd, 256, "tar -xf %s -C %s", PROC_KHEADERS_PATH, dirpath_tmp);
+ ret = system(tar_cmd);
+ if (ret) {
+ system(("rm -rf " + std::string(dirpath_tmp)).c_str());
+ goto cleanup;
+ }
+
+ /*
+ * If the new directory exists, it could have raced with a parallel
+ * extraction, in this case just delete the old directory and ignore.
+ */
+ ret = rename(dirpath_tmp, dirpath.c_str());
+ if (ret)
+ ret = system(("rm -rf " + std::string(dirpath_tmp)).c_str());
+
+cleanup:
+ if (module) {
+ int ret1 = system("rmmod kheaders");
+ if (ret1)
+ return ret1;
+ }
+
+ return ret;
+}
+
+int get_proc_kheaders(std::string &dirpath)
+{
+ struct utsname uname_data;
+ char dirpath_tmp[256];
+
+ if (uname(&uname_data))
+ return -errno;
+
+ snprintf(dirpath_tmp, 256, "/tmp/kheaders-%s", uname_data.release);
+ dirpath = std::string(dirpath_tmp);
+
+ if (file_exists(dirpath_tmp))
+ return 0;
+
+ // First time so extract it
+ return extract_kheaders(dirpath, uname_data);
+}
+
 } // namespace ebpf

src/cc/frontends/clang/kbuild_helper.h

@@ -21,6 +21,8 @@
 #include <errno.h>
 #include <ftw.h>
 
+#define PROC_KHEADERS_PATH "/proc/kheaders.tar.xz"
+
 namespace ebpf {
 
 struct FileDeleter {
@@ -101,4 +103,5 @@ class KBuildHelper {
  bool has_source_dir_;
 };
 
+int get_proc_kheaders(std::string &dir);
 } // namespace ebpf

src/cc/frontends/clang/loader.cc

@@ -120,6 +120,7 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
  const char *version_override = ::getenv("BCC_LINUX_VERSION_CODE");
  bool has_kpath_source = false;
  string vmacro;
+ std::string tmpdir;
 
  if (kpath_env) {
  kpath = string(kpath_env);
@@ -130,6 +131,13 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
  kpath = kdir + "/" + kernel_path_info.second;
  }
 
+ // If all attempts to obtain kheaders fail, check for /proc/kheaders.tar.xz
+ if (!is_dir(kpath)) {
+ int ret = get_proc_kheaders(tmpdir);
+ if (!ret)
+ kpath = tmpdir;
+ }
+
  if (flags_ & DEBUG_PREPROCESSOR)
  std::cout << "Running from kernel directory at: " << kpath.c_str() << "\n";