Implement Query::Eval method

include/jitmap/query/query.h

@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+#include <jitmap/size.h>
 #include <jitmap/util/pimpl.h>
 
 namespace jitmap {
@@ -27,38 +28,83 @@ class Expr;
 class ExecutionContext;
 
 class QueryImpl;
+
 class Query : util::Pimpl<QueryImpl> {
  public:
  // Create a new query object based on an expression.
  //
- // \param[in] name, the name of the query
- // \param[in] expr, the expression of the query
- // \param[in] context, the context where queries are compiled
+ // \param[in] name, the name of the query, see further note for restriction.
+ // \param[in] expr, the expression of the query.
+ // \param[in] context, the context where queries are compiled.
  //
  // \return a new query object
  //
- // \throws ParserException with a reason why the parsing failed.
+ // \throws ParserException if the expression is not valid, CompilerException
+ // if any failure was encountered while compiling the expression or if the
+ // query name is not valid.
+ //
+ // A query expression is compiled into a native function. The function is
+ // loaded in the process executable memory under a unique symbol name. This
+ // symbol is constructed partly from the query name. Thus, the query name must
+ // start with an alpha-numeric character and the remaining characters must be
+ // alpha-numeric or an underscore.
  static std::shared_ptr<Query> Make(const std::string& name, const std::string& query,
  ExecutionContext* context);
 
+ // Evaluate the expression on dense bitmaps.
+ //
+ // \param[in] inputs, pointers to input bitmaps, see further note on ordering.
+ // \param[in] output, pointer where the resulting bitmap will be written to,
+ // must not be nullptr.
+ //
+ // \throws Exception if any of the inputs/output pointers are nullptr.
+ //
+ // All the bitmaps must have allocated of the proper size, i.e.
+ // `kBytesPerContainer`. The
+ //
+ // The query expression references bitmaps by name, e.g. the expression
+ // `a & (b ^ !c) & (!a ^ c)` depends on the `a`, `b`, and `c` bitmaps. Since
+ // the expression is a tree structure, there is multiple ways to order the
+ // bitmaps. For this reason, the `Query::variables` method indicates the
+ // order in which bitmaps are expected by the `Query::Eval` method. The
+ // following pseudo-code shows how to do this.
+ //
+ // ```
+ // Bitmap a, b, c, output;
+ // auto order = query->variables();
+ // auto ordered_bitmaps = ReorderInputs({"a": a, "b": b, "c": c}, order);
+ // query->Eval(ordered_bitmaps, output);
+ // ```
+ void Eval(const BitsetWordType** inputs, BitsetWordType* output);
+
+ // Return the referenced variables and the expected order.o
+ //
+ // The ordering is fix once a Query object is constructed, i.e. the ordering
+ // is fixed for a given instance of a Query. If the query is deleted and then
+ // reconstructed with the same expression, there is not guarantee.
+ const std::vector<std::string>& variables() const;
+
  // Return the name of the query.
  const std::string& name() const;
 
  // Return the expression of the query.
  const Expr& expr() const;
 
  private:
+ // Private constructor, see Query::Make.
  Query(std::string name, std::string query, ExecutionContext* context);
 };
 
 class JitEngine;
 
 class ExecutionContext {
  public:
+ explicit ExecutionContext(std::shared_ptr<JitEngine> jit) : jit_(std::move(jit)) {}
+
  JitEngine* jit() { return jit_.get(); }
 
  private:
- std::unique_ptr<JitEngine> jit_;
+ std::shared_ptr<JitEngine> jit_;
 };
 
 } // namespace query

src/jitmap/query/query.cc

@@ -14,6 +14,7 @@
 
 #include "jitmap/query/query.h"
 
+#include <algorithm>
 #include <memory>
 #include <string>
 #include <vector>
@@ -32,14 +33,17 @@ class QueryImpl {
  query_(std::move(query)),
  expr_(Parse(query_, &builder_)),
  optimized_expr_(Optimizer(&builder_).Optimize(*expr_)),
- context_(context) {}
+ context_(context),
+ variables_(expr_->Variables()) {}
 
  // Accessors
  const std::string& name() const { return name_; }
  const std::string& query() const { return query_; }
  const Expr& expr() const { return *expr_; }
  const Expr& optimized_expr() const { return *optimized_expr_; }
- DenseEvalFn impl() const { return context_->jit()->LookupUserQuery(name()); }
+ const std::vector<std::string>& variables() const { return variables_; }
+
+ DenseEvalFn dense_eval_fn() const { return context_->jit()->LookupUserQuery(name()); }
 
  private:
  std::string name_;
@@ -48,23 +52,63 @@ class QueryImpl {
  Expr* expr_;
  Expr* optimized_expr_;
  ExecutionContext* context_;
+
+ std::vector<std::string> variables_;
 };
 
+static inline void ValidateQueryName(const std::string& name) {
+ if (name.empty()) {
+ throw CompilerException("Query name must have at least one character");
+ }
+
+ auto first = name[0];
+ if (!std::isalnum(first)) {
+ throw CompilerException(
+ "The first character of the Query name must be an alpha numeric character but "
+ "got",
+ first);
+ }
+
+ auto is_valid_char = [](auto c) { return std::isalnum(c) || c == '_'; };
+ if (!std::all_of(name.cbegin(), name.cend(), is_valid_char)) {
+ throw CompilerException(
+ "The characters of a query name must either be an alpha numeric character or an "
+ "underscore.");
+ }
+}
+
 Query::Query(std::string name, std::string query, ExecutionContext* context)
  : Pimpl(std::make_unique<QueryImpl>(std::move(name), std::move(query), context)) {}
 
 std::shared_ptr<Query> Query::Make(const std::string& name, const std::string& expr,
  ExecutionContext* context) {
+ JITMAP_PRE_NE(context, nullptr);
+
+ // Ensure that the query names follows the restriction
+ ValidateQueryName(name);
+
  auto query = std::shared_ptr<Query>(new Query(name, expr, context));
- if (context != nullptr) {
- context->jit()->Compile(query->name(), query->expr());
- }
+ context->jit()->Compile(query->name(), query->expr());
 
  return query;
 }
 
 const std::string& Query::name() const { return impl().name(); }
 const Expr& Query::expr() const { return impl().expr(); }
+const std::vector<std::string>& Query::variables() const { return impl().variables(); }
+
+void Query::Eval(const BitsetWordType** inputs, BitsetWordType* output) {
+ JITMAP_PRE_NE(inputs, nullptr);
+ JITMAP_PRE_NE(output, nullptr);
+
+ auto n_inputs = variables().size();
+ for (size_t i = 0; i < n_inputs; i++) {
+ JITMAP_PRE_NE(inputs[i], nullptr);
+ }
+
+ auto eval_fn = impl().dense_eval_fn();
+ eval_fn(inputs, output);
+}
 
 } // namespace query
 } // namespace jitmap

tests/jitmap_benchmark.cc

@@ -37,25 +37,19 @@ static void StaticIntersection2(benchmark::State& state) {
 }
 
 static void JitIntersection2(benchmark::State& state) {
- std::vector<std::vector<BitsetWordType>> bitmaps;
- std::vector<const BitsetWordType*> inputs;
- for (size_t i = 0; i < 2; i++) {
- bitmaps.emplace_back(kWordsPerContainers, 0UL);
- inputs.emplace_back(bitmaps[i].data());
- }
- std::vector<BitsetWordType> output(kWordsPerContainers, 0UL);
+ std::array<BitsetWordType, kWordsPerContainers> a, b, output;
+ std::vector<const BitsetWordType*> inputs{a.data(), b.data()};
 
- auto engine = query::JitEngine::Make();
- auto query = query::Query::Make("benchmark_query2", "a & b", nullptr);
- engine->Compile(query->name(), query->expr());
- auto eval_fn = engine->LookupUserQuery("benchmark_query2");
+ query::ExecutionContext engine{query::JitEngine::Make()};
+ auto query = query::Query::Make("benchmark_query_2", "a & b", &engine);
 
  for (auto _ : state) {
- eval_fn(inputs.data(), output.data());
+ query->Eval(inputs.data(), output.data());
  }
 
  state.SetBytesProcessed(kBytesPerContainer * 2 * state.iterations());
 }
+
 static void StaticIntersection3(benchmark::State& state) {
  DenseBitmap a, b, c;
 
@@ -67,46 +61,22 @@ static void StaticIntersection3(benchmark::State& state) {
 }
 
 static void JitIntersection3(benchmark::State& state) {
- std::vector<std::vector<BitsetWordType>> bitmaps;
- std::vector<const BitsetWordType*> inputs;
- for (size_t i = 0; i < 3; i++) {
- bitmaps.emplace_back(kWordsPerContainers, 0UL);
- inputs.emplace_back(bitmaps[i].data());
- }
- std::vector<BitsetWordType> output(kWordsPerContainers, 0UL);
+ std::array<BitsetWordType, kWordsPerContainers> a, b, c, output;
+ std::vector<const BitsetWordType*> inputs{a.data(), b.data(), c.data()};
 
- auto engine = query::JitEngine::Make();
- auto query = query::Query::Make("benchmark_query3", "a & b & c", nullptr);
- engine->Compile(query->name(), query->expr());
- auto eval_fn = engine->LookupUserQuery("benchmark_query3");
+ query::ExecutionContext engine{query::JitEngine::Make()};
+ auto query = query::Query::Make("benchmark_query_3", "a & b & c", &engine);
 
  for (auto _ : state) {
- eval_fn(inputs.data(), output.data());
+ query->Eval(inputs.data(), output.data());
  }
 
  state.SetBytesProcessed(kBytesPerContainer * 3 * state.iterations());
 }
-/*
-
-
-static void StaticIntersection4(benchmark::State& state) {
- DenseBitmap a, b, c, d;
-
- for (auto _ : state) {
- benchmark::DoNotOptimize(a & b & c & d);
- }
-
- state.SetBytesProcessed(((kBitsPerContainer / 8) * 4) * state.iterations());
-}
-*/
 
 BENCHMARK(StaticIntersection2);
 BENCHMARK(JitIntersection2);
 BENCHMARK(StaticIntersection3);
 BENCHMARK(JitIntersection3);
 
-/*
-BENCHMARK(StaticIntersection3);
-BENCHMARK(StaticIntersection4);
-*/
 } // namespace jitmap

tests/query/compiler_test.cc

@@ -18,6 +18,7 @@
 
 #include <jitmap/query/compiler.h>
 #include <jitmap/query/parser.h>
+#include <jitmap/util/compiler.h>
 
 namespace jitmap {
 namespace query {
@@ -27,12 +28,25 @@ class JitTest : public QueryTest {
  void AssertQueryResult(const std::string& query_expr,
  std::vector<BitsetWordType> input_words,
  BitsetWordType output_word) {
- auto query_name = "query_" + std::to_string(id++);
- auto query_fn = CompileAndLookup(query_expr, query_name);
- ASSERT_NE(query_fn, nullptr);
+ auto query = Query::Make(query_name(), query_expr, &ctx);
 
- // Populate input bitmaps each with a repeated word.
+ auto [_, inputs] = InitInputs(input_words);
+ JITMAP_UNUSED(_);
+
+ std::vector<BitsetWordType> output(kWordsPerContainers, 0UL);
+ EXPECT_THAT(output, testing::Each(0UL));
+
+ query->Eval(inputs.data(), output.data());
+ EXPECT_THAT(output, testing::Each(output_word));
+ }
+
+ private:
+ std::string query_name() { return "query_" + std::to_string(id++); }
+
+ std::tuple<std::vector<std::vector<BitsetWordType>>, std::vector<const BitsetWordType*>>
+ InitInputs(std::vector<BitsetWordType> input_words) {
  size_t n_bitmaps = input_words.size();
+
  std::vector<std::vector<BitsetWordType>> bitmaps;
  std::vector<const BitsetWordType*> inputs;
  for (size_t i = 0; i < n_bitmaps; i++) {
@@ -42,29 +56,17 @@ class JitTest : public QueryTest {
  EXPECT_THAT(bitmaps[i], testing::Each(repeated_word));
  }
 
- // Populate output bitmap
- std::vector<BitsetWordType> output(kWordsPerContainers, 0UL);
- EXPECT_THAT(output, testing::Each(0UL));
-
- query_fn(inputs.data(), output.data());
- EXPECT_THAT(output, testing::Each(output_word));
- }
-
- DenseEvalFn CompileAndLookup(const std::string& query_expr,
- const std::string& query_name) {
- auto query = Query::Make(query_name, query_expr, nullptr);
- engine_->Compile(query->name(), query->expr());
- return engine_->LookupUserQuery(query_name);
+ return {std::move(bitmaps), std::move(inputs)};
  }
 
  protected:
  std::atomic<uint32_t> id = 0;
- std::shared_ptr<JitEngine> engine_ = JitEngine::Make();
+ ExecutionContext ctx{JitEngine::Make()};
 };
 
 TEST_F(JitTest, CpuDetection) {
- EXPECT_NE(engine_->GetTargetCPU(), "");
- EXPECT_NE(engine_->GetTargetTriple(), "");
+ EXPECT_NE(ctx.jit()->GetTargetCPU(), "");
+ EXPECT_NE(ctx.jit()->GetTargetTriple(), "");
 }
 
 TEST_F(JitTest, CompileAndExecuteTest) {