Implement JitEngine

CMakeLists.txt

@@ -11,7 +11,7 @@ option(BUILD_TESTS "Builder unit tests" ON)
 option(FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." ON)
 
 include(CompilerToolchain)
-find_package(LLVM REQUIRED CONFIG)
+find_package(LLVM 8 REQUIRED CONFIG)
 
 add_subdirectory(src/jitmap)
 add_subdirectory(tools)

include/jitmap/query/compiler.h

@@ -1,10 +1,12 @@
+#pragma once
+
 #include <memory>
 
 #include <jitmap/query/query.h>
 #include <jitmap/size.h>
 
 namespace llvm {
-class Function;
+class LLVMContext;
 class Module;
 } // namespace llvm
 
@@ -17,6 +19,13 @@ class CompilerException : public Exception {
 };
 
 struct CompilerOptions {
+ // Controls LLVM optimization level (-O0, -O1, -O2, -O3). Anything above 3
+ // will be clamped to 3.
+ uint8_t optimization_level = 3;
+
+ // Number of background compiler threads.
+ uint8_t compiler_threads = 0;
+
  // Controls the number of scalar, i.e. width of the vector aggregate value in
  // the loop. A value of 1 will emit a scalar value instead of a vector value.
  //
@@ -25,40 +34,59 @@ struct CompilerOptions {
  //
  // LLVM's optimizer is able to perform the auto-vectorization of the loop. In
  // cases where it can't, change the vector width here.
- uint8_t vector_width_ = 1;
+ uint8_t vector_width = 1;
 
  // Controls the width of each scalar (in bits). See the documentation
  // of your hardware platform for the optimal value.
- uint8_t scalar_width_ = kBytesPerBitsetWord;
+ uint8_t scalar_width = kBitsPerBitsetWord;
 };
 
-class QueryCompiler {
+// The QueryIRCodeGen class generates LLVM IR for query expression on dense
+// bitmaps. The QueryIRCodeGen is backed by a single llvm::Module where each
+// Query object is represented by a single llvm::Function.
+class QueryIRCodeGen {
  public:
- QueryCompiler(const std::string& module_name, CompilerOptions options = {});
+ QueryIRCodeGen(const std::string& module_name, CompilerOptions options = {});
+ QueryIRCodeGen(QueryIRCodeGen&&);
 
- // Compile a query expression in an llvm::Function.
+ // Compile a query expression into the module.
  //
- // - The query's name must be unique within the QueryCompiler object. It
- // always returns the first inserted function irregardless of the
- // expression.
+ // - The query's name must be unique within the QueryIRCodeGen object.
  // - The generated function is not executable as-is, it is only the LLVM IR
  // representation.
  //
  // \param[in] query, the query to compile into LLVM IR.
  //
- // \return an llvm function object
- //
- // \throws CompilerExpception if any errors is encountered.
- llvm::Function* Compile(const Query& query);
+ // \throws CompilerException if any errors is encountered.
+ void Compile(const Query& query);
 
- // Return the LLVM IR representation of the module and all the compiled queries.
- std::string DumpIR() const;
+ // Compile a query expression into the module.
+ template <typename Ptr>
+ void Compile(const Ptr& query) {
+ JITMAP_PRE_NE(query, nullptr);
+ Compile(*query);
+ }
 
- ~QueryCompiler();
+ // Finalize and consume as a module and context.
+ //
+ // The QueryIRCodeGen must be moved to invoke this function, e.g.
+ //
+ // auto [mod_, ctx_] = std::move(query_compiler).Finish();
+ //
+ // The move is required because the module and context ownership are
+ // transferred to the caller. Instead of resetting the internal state, the
+ // caller must create a new QueryIRCodeGen.
+ using ModuleAndContext =
+ std::pair<std::unique_ptr<llvm::Module>, std::unique_ptr<llvm::LLVMContext>>;
+ ModuleAndContext Finish() &&;
+
+ ~QueryIRCodeGen();
 
  private:
  class Impl;
  std::unique_ptr<Impl> impl_;
+
+ QueryIRCodeGen(const QueryIRCodeGen&) = delete;
 };
 
 } // namespace query

include/jitmap/query/jit.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+
+#include <jitmap/query/compiler.h>
+#include <jitmap/util/exception.h>
+
+namespace jitmap {
+namespace query {
+
+class JitException : public Exception {
+ public:
+ using Exception::Exception;
+};
+
+// Signature of generated functions
+typedef void (*DenseEvalFn)(const BitsetWordType**, BitsetWordType*);
+
+// The JitEngine class transforms IR queries into executable functions.
+class JitEngine {
+ public:
+ JitEngine(CompilerOptions options = {});
+ ~JitEngine();
+
+ // Return the LLVM name for the host CPU.
+ //
+ // This is the string given to `-march/-mtune/-mcpu`. See
+ // http:https://llvm.org/doxygen/Host_8h_source.html for more information.
+ std::string GetTargetCPU() const;
+
+ // Return the LLVM features string for the host CPU.
+ //
+ // An array delimited by comma of symbols referencing a specific cpu feature.
+ // The feature supported are prefixed by `+`, and unsupported by `-`. See
+ // http:https://llvm.org/doxygen/Host_8h_source.html for more information.
+ std::string GetTargetFeatureString() const;
+
+ // Return the LLVM target triple for the host.
+ //
+ // The format is ARCHITECTURE-VENDOR-OPERATING_SYSTEM-ENVIRONMENT. See
+ // http:https://llvm.org/doxygen/Triple_8h_source.html for more information.
+ std::string GetTargetTriple() const;
+
+ // Compile a query.
+ void Compile(QueryIRCodeGen query);
+
+ // Lookup a query
+ DenseEvalFn LookupUserQuery(const std::string& query_name);
+
+ private:
+ class Impl;
+ std::unique_ptr<Impl> impl_;
+};
+
+} // namespace query
+} // namespace jitmap

include/jitmap/size.h

@@ -19,5 +19,6 @@ constexpr size_t kBytesPerContainer = kBitsPerContainer / CHAR_BIT;
 using BitsetWordType = uint32_t;
 constexpr size_t kBytesPerBitsetWord = sizeof(BitsetWordType);
 constexpr size_t kBitsPerBitsetWord = kBytesPerBitsetWord * CHAR_BIT;
+constexpr size_t kWordsPerContainers = kBytesPerContainer / kBytesPerBitsetWord;
 
 } // namespace jitmap

src/jitmap/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(SOURCES
  query/compiler.cc
  query/expr.cc
+ query/jit.cc
  query/matcher.cc
  query/optimizer.cc
  query/parser.cc
@@ -14,5 +15,10 @@ target_include_directories(jitmap PUBLIC
  )
 target_compile_options(jitmap PRIVATE ${CXX_WARNING_FLAGS})
 target_include_directories(jitmap PUBLIC ${LLVM_INCLUDE_DIRS})
-llvm_map_components_to_libnames(LLVM_LIBRARIES support core irreader mcjit)
+
+# Required for the jit engine
+set(LLVM_CORE_COMPONENTS support core irreader orcjit vectorize)
+set(LLVM_NATIVE_JIT_COMPONENTS x86codegen x86asmparser x86disassembler x86asmprinter x86desc x86info x86utils)
+
+llvm_map_components_to_libnames(LLVM_LIBRARIES ${LLVM_CORE_COMPONENTS} ${LLVM_NATIVE_JIT_COMPONENTS})
 target_link_libraries(jitmap ${LLVM_LIBRARIES})