LibSQL: Introduce Serializer as a mediator between Heap and client code

Classes reading and writing to the data heap would communicate directly
with the Heap object, and transfer ByteBuffers back and forth with it.
This makes things like caching and locking hard. Therefore all data
persistence activity will be funneled through a Serializer object which
in turn submits it to the Heap.

Introducing this unfortunately resulted in a huge amount of churn, in
which a number of smaller refactorings got caught up as well.

Tests/LibSQL/TestSqlBtreeIndex.cpp

@@ -120,23 +120,23 @@ constexpr static u32 pointers[] = {
  4,
 };
 
-NonnullRefPtr<SQL::BTree> setup_btree(SQL::Heap& heap);
-void insert_and_get_to_and_from_btree(int num_keys);
-void insert_into_and_scan_btree(int num_keys);
+NonnullRefPtr<SQL::BTree> setup_btree(SQL::Serializer&);
+void insert_and_get_to_and_from_btree(int);
+void insert_into_and_scan_btree(int);
 
-NonnullRefPtr<SQL::BTree> setup_btree(SQL::Heap& heap)
+NonnullRefPtr<SQL::BTree> setup_btree(SQL::Serializer& serializer)
 {
  NonnullRefPtr<SQL::TupleDescriptor> tuple_descriptor = adopt_ref(*new SQL::TupleDescriptor);
  tuple_descriptor->append({ "key_value", SQL::SQLType::Integer, SQL::Order::Ascending });
 
- auto root_pointer = heap.user_value(0);
+ auto root_pointer = serializer.heap().user_value(0);
  if (!root_pointer) {
- root_pointer = heap.new_record_pointer();
- heap.set_user_value(0, root_pointer);
+ root_pointer = serializer.heap().new_record_pointer();
+ serializer.heap().set_user_value(0, root_pointer);
  }
- auto btree = SQL::BTree::construct(heap, tuple_descriptor, true, root_pointer);
+ auto btree = SQL::BTree::construct(serializer, tuple_descriptor, true, root_pointer);
  btree->on_new_root = [&]() {
- heap.set_user_value(0, btree->root());
+ serializer.heap().set_user_value(0, btree->root());
  };
  return btree;
 }
@@ -146,7 +146,8 @@ void insert_and_get_to_and_from_btree(int num_keys)
  ScopeGuard guard([]() { unlink("/tmp/test.db"); });
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto btree = setup_btree(heap);
+ SQL::Serializer serializer(heap);
+ auto btree = setup_btree(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(btree->descriptor());
@@ -161,13 +162,14 @@ void insert_and_get_to_and_from_btree(int num_keys)
 
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto btree = setup_btree(heap);
+ SQL::Serializer serializer(heap);
+ auto btree = setup_btree(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(btree->descriptor());
  k[0] = keys[ix];
  auto pointer_opt = btree->get(k);
- EXPECT(pointer_opt.has_value());
+ VERIFY(pointer_opt.has_value());
  EXPECT_EQ(pointer_opt.value(), pointers[ix]);
  }
  }
@@ -178,7 +180,8 @@ void insert_into_and_scan_btree(int num_keys)
  ScopeGuard guard([]() { unlink("/tmp/test.db"); });
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto btree = setup_btree(heap);
+ SQL::Serializer serializer(heap);
+ auto btree = setup_btree(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(btree->descriptor());
@@ -194,13 +197,14 @@ void insert_into_and_scan_btree(int num_keys)
 
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto btree = setup_btree(heap);
+ SQL::Serializer serializer(heap);
+ auto btree = setup_btree(serializer);
 
  int count = 0;
  SQL::Tuple prev;
  for (auto iter = btree->begin(); !iter.is_end(); iter++, count++) {
  auto key = (*iter);
- if (prev.length()) {
+ if (prev.size()) {
  EXPECT(prev < key);
  }
  auto key_value = (int)key[0];

Tests/LibSQL/TestSqlHashIndex.cpp

@@ -117,22 +117,22 @@ constexpr static u32 pointers[] = {
  4,
 };
 
-NonnullRefPtr<SQL::HashIndex> setup_hash_index(SQL::Heap& heap);
-void insert_and_get_to_and_from_hash_index(int num_keys);
-void insert_into_and_scan_hash_index(int num_keys);
+NonnullRefPtr<SQL::HashIndex> setup_hash_index(SQL::Serializer&);
+void insert_and_get_to_and_from_hash_index(int);
+void insert_into_and_scan_hash_index(int);
 
-NonnullRefPtr<SQL::HashIndex> setup_hash_index(SQL::Heap& heap)
+NonnullRefPtr<SQL::HashIndex> setup_hash_index(SQL::Serializer& serializer)
 {
  NonnullRefPtr<SQL::TupleDescriptor> tuple_descriptor = adopt_ref(*new SQL::TupleDescriptor);
  tuple_descriptor->append({ "key_value", SQL::SQLType::Integer, SQL::Order::Ascending });
  tuple_descriptor->append({ "text_value", SQL::SQLType::Text, SQL::Order::Ascending });
 
- auto directory_pointer = heap.user_value(0);
+ auto directory_pointer = serializer.heap().user_value(0);
  if (!directory_pointer) {
- directory_pointer = heap.new_record_pointer();
- heap.set_user_value(0, directory_pointer);
+ directory_pointer = serializer.heap().new_record_pointer();
+ serializer.heap().set_user_value(0, directory_pointer);
  }
- auto hash_index = SQL::HashIndex::construct(heap, tuple_descriptor, directory_pointer);
+ auto hash_index = SQL::HashIndex::construct(serializer, tuple_descriptor, directory_pointer);
  return hash_index;
 }
 
@@ -141,7 +141,8 @@ void insert_and_get_to_and_from_hash_index(int num_keys)
  ScopeGuard guard([]() { unlink("/tmp/test.db"); });
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto hash_index = setup_hash_index(heap);
+ SQL::Serializer serializer(heap);
+ auto hash_index = setup_hash_index(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(hash_index->descriptor());
@@ -157,14 +158,15 @@ void insert_and_get_to_and_from_hash_index(int num_keys)
 
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto hash_index = setup_hash_index(heap);
+ SQL::Serializer serializer(heap);
+ auto hash_index = setup_hash_index(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(hash_index->descriptor());
  k[0] = keys[ix];
  k[1] = String::formatted("The key value is {} and the pointer is {}", keys[ix], pointers[ix]);
  auto pointer_opt = hash_index->get(k);
- EXPECT(pointer_opt.has_value());
+ VERIFY(pointer_opt.has_value());
  EXPECT_EQ(pointer_opt.value(), pointers[ix]);
  }
  }
@@ -235,7 +237,8 @@ void insert_into_and_scan_hash_index(int num_keys)
  ScopeGuard guard([]() { unlink("/tmp/test.db"); });
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto hash_index = setup_hash_index(heap);
+ SQL::Serializer serializer(heap);
+ auto hash_index = setup_hash_index(serializer);
 
  for (auto ix = 0; ix < num_keys; ix++) {
  SQL::Key k(hash_index->descriptor());
@@ -251,7 +254,8 @@ void insert_into_and_scan_hash_index(int num_keys)
 
  {
  auto heap = SQL::Heap::construct("/tmp/test.db");
- auto hash_index = setup_hash_index(heap);
+ SQL::Serializer serializer(heap);
+ auto hash_index = setup_hash_index(serializer);
  Vector<bool> found;
  for (auto ix = 0; ix < num_keys; ix++) {
  found.append(false);

Tests/LibSQL/TestSqlValueAndTuple.cpp

@@ -95,11 +95,11 @@ TEST_CASE(serialize_text_value)
  SQL::Value v("Test");
  EXPECT(v.to_string() == "Test");
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT((String)v2 == "Test");
 }
 
@@ -146,11 +146,11 @@ TEST_CASE(serialize_int_value)
  EXPECT_EQ(v.type(), SQL::SQLType::Integer);
  EXPECT_EQ(v.to_int().value(), 42);
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT(!v2.is_null());
  EXPECT_EQ(v2.type(), SQL::SQLType::Integer);
  EXPECT_EQ(v2.to_int().value(), 42);
@@ -201,11 +201,11 @@ TEST_CASE(serialize_float_value)
  EXPECT_EQ(v.type(), SQL::SQLType::Float);
  EXPECT(v.to_double().value() - 3.14 < NumericLimits<double>().epsilon());
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT(!v2.is_null());
  EXPECT_EQ(v2.type(), SQL::SQLType::Float);
  EXPECT(v.to_double().value() - 3.14 < NumericLimits<double>().epsilon());
@@ -272,11 +272,11 @@ TEST_CASE(serialize_boolean_value)
  EXPECT_EQ(v.type(), SQL::SQLType::Boolean);
  EXPECT(bool(v));
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT(!v2.is_null());
  EXPECT_EQ(v2.type(), SQL::SQLType::Boolean);
  EXPECT(bool(v2));
@@ -374,11 +374,11 @@ TEST_CASE(serialize_tuple_value)
  values.append(SQL::Value(42));
  v = values;
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT(!v2.is_null());
  EXPECT_EQ(v2.type(), SQL::SQLType::Tuple);
  EXPECT_EQ(v, v2);
@@ -440,11 +440,11 @@ TEST_CASE(serialize_array_value)
  values.append(SQL::Value("Test 2"));
  v = values;
 
- ByteBuffer buffer;
- v.serialize_to(buffer);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Value>(v);
 
- size_t offset = 0;
- auto v2 = SQL::Value::deserialize_from(buffer, offset);
+ serializer.rewind();
+ auto v2 = serializer.deserialize<SQL::Value>();
  EXPECT(!v2.is_null());
  EXPECT_EQ(v2.type(), SQL::SQLType::Array);
  EXPECT_EQ(v, v2);
@@ -497,13 +497,14 @@ TEST_CASE(serialize_tuple)
  tuple["col1"] = "Test";
  tuple["col2"] = 42;
 
- auto buffer = ByteBuffer();
- tuple.serialize(buffer);
  EXPECT_EQ((String)tuple[0], "Test");
  EXPECT_EQ((int)tuple[1], 42);
 
- size_t offset = 0;
- SQL::Tuple tuple2(descriptor, buffer, offset);
+ SQL::Serializer serializer;
+ serializer.serialize<SQL::Tuple>(tuple);
+
+ serializer.rewind();
+ auto tuple2 = serializer.deserialize<SQL::Tuple>();
  EXPECT(tuple2[0] == "Test");
  EXPECT(tuple2[1] == 42);
 }

Userland/Libraries/LibSQL/BTree.cpp

@@ -10,14 +10,14 @@
 
 namespace SQL {
 
-BTree::BTree(Heap& heap, NonnullRefPtr<TupleDescriptor> const& descriptor, bool unique, u32 pointer)
- : Index(heap, descriptor, unique, pointer)
+BTree::BTree(Serializer& serializer, NonnullRefPtr<TupleDescriptor> const& descriptor, bool unique, u32 pointer)
+ : Index(serializer, descriptor, unique, pointer)
  , m_root(nullptr)
 {
 }
 
-BTree::BTree(Heap& heap, NonnullRefPtr<TupleDescriptor> const& descriptor, u32 pointer)
- : BTree(heap, descriptor, true, pointer)
+BTree::BTree(Serializer& serializer, NonnullRefPtr<TupleDescriptor> const& descriptor, u32 pointer)
+ : BTree(serializer, descriptor, true, pointer)
 {
 }
 
@@ -37,10 +37,9 @@ BTreeIterator BTree::end()
 void BTree::initialize_root()
 {
  if (pointer()) {
- if (pointer() < heap().size()) {
- auto buffer = read_block(pointer());
- size_t offset = 0;
- m_root = make<TreeNode>(*this, nullptr, pointer(), buffer, offset);
+ if (serializer().has_block(pointer())) {
+ serializer().get_block(pointer());
+ m_root = serializer().make_and_deserialize<TreeNode>(*this, pointer());
  } else {
  m_root = make<TreeNode>(*this, nullptr, pointer());
  }
@@ -50,13 +49,14 @@ void BTree::initialize_root()
  if (on_new_root)
  on_new_root();
  }
+ m_root->dump_if(0, "initialize_root");
 }
 
 TreeNode* BTree::new_root()
 {
  set_pointer(new_record_pointer());
  m_root = make<TreeNode>(*this, nullptr, m_root.leak_ptr(), pointer());
- add_to_write_ahead_log(m_root->as_index_node());
+ serializer().serialize_and_write(*m_root.ptr(), m_root->pointer());
  if (on_new_root)
  on_new_root();
  return m_root;

Userland/Libraries/LibSQL/BTree.h

@@ -43,7 +43,7 @@ class DownPointer {
  TreeNode* node();
 
 private:
- void inflate();
+ void deserialize(Serializer&);
 
  TreeNode* m_owner;
  u32 m_pointer { 0 };
@@ -53,27 +53,27 @@ class DownPointer {
 
 class TreeNode : public IndexNode {
 public:
+ TreeNode(BTree&, u32 = 0);
  TreeNode(BTree&, TreeNode*, u32 = 0);
  TreeNode(BTree&, TreeNode*, TreeNode*, u32 = 0);
- TreeNode(BTree&, TreeNode*, u32 pointer, ByteBuffer&, size_t&);
  ~TreeNode() override = default;
 
  [[nodiscard]] BTree& tree() const { return m_tree; }
  [[nodiscard]] TreeNode* up() const { return m_up; }
  [[nodiscard]] size_t size() const { return m_entries.size(); }
+ [[nodiscard]] size_t length() const;
  [[nodiscard]] Vector<Key> entries() const { return m_entries; }
  [[nodiscard]] u32 down_pointer(size_t) const;
  [[nodiscard]] TreeNode* down_node(size_t);
  [[nodiscard]] bool is_leaf() const { return m_is_leaf; }
 
- [[nodiscard]] size_t max_keys_in_node();
  Key const& operator[](size_t) const;
  bool insert(Key const&);
  bool update_key_pointer(Key const&);
  TreeNode* node_for(Key const&);
  Optional<u32> get(Key&);
- void serialize(ByteBuffer&) const override;
- IndexNode* as_index_node() override { return dynamic_cast<IndexNode*>(this); }
+ void deserialize(Serializer&);
+ void serialize(Serializer&) const;
 
 private:
  TreeNode(BTree&, TreeNode*, DownPointer&, u32 = 0);
@@ -111,8 +111,8 @@ class BTree : public Index {
  Function<void(void)> on_new_root;
 
 private:
- BTree(Heap& heap, NonnullRefPtr<TupleDescriptor> const&, bool unique, u32 pointer);
- BTree(Heap& heap, NonnullRefPtr<TupleDescriptor> const&, u32 pointer);
+ BTree(Serializer&, NonnullRefPtr<TupleDescriptor> const&, bool unique, u32 pointer);
+ BTree(Serializer&, NonnullRefPtr<TupleDescriptor> const&, u32 pointer);
  void initialize_root();
  TreeNode* new_root();
  OwnPtr<TreeNode> m_root { nullptr };

Userland/Libraries/LibSQL/BTreeIterator.cpp

@@ -232,7 +232,7 @@ bool BTreeIterator::update(Key const& new_value)
 
  // We are friend of BTree and TreeNode. Don't know how I feel about that.
  m_current->m_entries[m_index] = new_value;
- m_current->tree().add_to_write_ahead_log(m_current);
+ m_current->tree().serializer().serialize_and_write(*m_current, m_current->pointer());
  return true;
 }
 

Userland/Libraries/LibSQL/CMakeLists.txt

@@ -17,6 +17,7 @@ set(SOURCES
  Key.cpp
  Meta.cpp
  Row.cpp
+ Serializer.cpp
  SQLClient.cpp
  TreeNode.cpp
  Tuple.cpp