Fix a bunch of bugs with fallback mode

README.md

@@ -13,7 +13,15 @@ IndexedDB is not a great database. It's slow, hard to work with, and has very fe
 
 ## ... How well does it work?
 
-It works absurdly well. It consistently beats IndexedDB performance 10 fold or even more.
+It works absurdly well. It consistently beats IndexedDB performance up to 10x:
+
+Read performance: doing something like `SELECT SUM(value) FROM kv`:
+
+<img width="610" alt="perf-sum-chrome" src="https://user-images.githubusercontent.com/17031/129102253-8adf163a-76b6-4af8-a1cf-8e2e39012ab0.png">
+
+Write performance: doing a bulk insert:
+
+<img width="609" alt="perf-writes-chrome" src="https://user-images.githubusercontent.com/17031/129102454-b4c362b3-1b0a-4625-ac96-72fc276497f3.png">
 
 Why? It's simple once you think about it: since we are reading/writing data in 4K chunks (size is configurable), we automatically batch reads and writes. If you want to store 1 million objects into IDB, you need to do 1 million writes. With this absurd backend, it only needs to do ~12500 writes.
 

package.json

@@ -1,6 +1,6 @@
 {
- "name": "absurd-sql.js-backend",
- "version": "0.0.35",
+ "name": "absurd-sql",
+ "version": "0.0.45",
  "main": "./dist/index.js",
  "scripts": {
  "build": "rm -rf dist && rollup -c rollup.config.js",

src/examples/bench/main.worker.js

@@ -94,9 +94,11 @@ async function getDatabase() {
 
  let path = `/blocked/${dbName}`;
 
- // let stream = SQL.FS.open(path, 'a+');
- // await stream.node.contents.readIfFallback();
- // SQL.FS.close(stream);
+ if (typeof SharedArrayBuffer === 'undefined') {
+ let stream = SQL.FS.open(path, 'a+');
+ await stream.node.contents.readIfFallback();
+ SQL.FS.close(stream);
+ }
 
  _db = new SQL.Database(path, { filename: true });
 

src/examples/fts/main.worker.js

@@ -8,10 +8,10 @@ import IndexedDBBackend from '../../indexeddb/backend';
 
 let currentBackendType = 'idb';
 let cacheSize = 5000;
-let pageSize = 4096;
+let pageSize = 8192;
 let dbName = `fts.sqlite`;
 
-let idbBackend = new IndexedDBBackend(4096 * 2);
+let idbBackend = new IndexedDBBackend();
 let sqlFS;
 
 // Helper methods
@@ -68,6 +68,7 @@ async function getDatabase() {
  PRAGMA page_size=${pageSize};
  PRAGMA journal_mode=MEMORY;
  `);
+ _db.exec('VACUUM');
  output(
  `Opened ${getDBName()} (${currentBackendType}) cache size: ${cacheSize}`
  );

src/indexeddb/backend.js

@@ -1,6 +1,6 @@
 import { File } from '../sqlite-file';
 import * as perf from 'perf-deets';
-import { LOCK_TYPES, getPageSize, isSafeToWrite } from '../sqlite-util';
+import { LOCK_TYPES, getPageSize } from '../sqlite-util';
 import { FileOps } from './file-ops';
 import { FileOpsFallback } from './file-ops-fallback';
 

src/indexeddb/file-ops-fallback.js

@@ -1,3 +1,5 @@
+import { LOCK_TYPES, isSafeToWrite, getPageSize } from '../sqlite-util';
+
 function positionToKey(pos, blockSize) {
  // We are forced to round because of floating point error. `pos`
  // should always be divisible by `blockSize`
@@ -36,13 +38,37 @@ export class FileOpsFallback {
  this.cachedFirstBlock = null;
  this.blocks = new Map();
  this.writeQueue = [];
- this.lockType = null;
+ this.lockType = 0;
+ }
+
+ async getDb() {
+ if (this._openDb) {
+ return this._openDb;
+ }
+
+ this._openDb = await openDb(this.dbName);
+ return this._openDb;
+ }
+
+ closeDb() {
+ if (this._openDb) {
+ this._openDb.close();
+ this._openDb = null;
+ }
  }
 
  async readIfFallback() {
- this.db = await openDb(this.dbName);
+ // OK We need to fix this better - we don't block on the writes
+ // being flushed from closing the file, and we can't read in
+ // everything here because we might get old data. Need to track
+ // the last write and force it to be sequential
+ if (this.blocks.size > 0) {
+ return;
+ }
+
+ let db = await this.getDb(this.dbName);
 
- let trans = this.db.transaction(['data'], 'readonly');
+ let trans = db.transaction(['data'], 'readonly');
  let store = trans.objectStore('data');
 
  return new Promise((resolve, reject) => {
@@ -55,8 +81,7 @@ export class FileOpsFallback {
  this.blocks.set(cursor.key, cursor.value);
  cursor.continue();
  } else {
- this.cachedFirstBlock = this.blocks.get(0);
- resolve();
+ resolve(this.readMeta());
  }
  };
  });
@@ -66,38 +91,49 @@ export class FileOpsFallback {
  this.writeQueue.push({ key, value });
  }
 
- async flushWrites() {
+ // We need a snapshot of the current write + state in which it was
+ // written. We do writes async, so we can't check this state over
+ // time because it may change from underneath us
+ prepareFlush() {
+ let writeState = {
+ cachedFirstBlock: this.cachedFirstBlock,
+ writes: this.writeQueue,
+ lockType: this.lockType
+ };
+ this.writeQueue = [];
+ return writeState;
+ }
+
+ async flushWriteState(db, writeState) {
  // We need grab a readwrite lock on the db, and then read to check
  // to make sure we can write to it
- let trans = this.db.transaction(['data'], 'readwrite');
+ let trans = db.transaction(['data'], 'readwrite');
  let store = trans.objectStore('data');
 
  await new Promise((resolve, reject) => {
  let req = store.get(0);
  req.onsuccess = e => {
- if (
- !isSafeToWrite(
- new Uint8Array(req.result),
- new Uint8Array(this.cachedFirstBlock)
- )
- ) {
- console.log('SCREWED');
- reject('screwed');
- return;
+ if (writeState.lockType > LOCK_TYPES.NONE) {
+ if (!isSafeToWrite(req.result, writeState.cachedFirstBlock)) {
+ // TODO: We need to send a message to users somehow
+ console.log("OH NO WE CAN'T WRITE");
+ reject('screwed');
+ return;
+ }
  }
 
  // Flush all the writes
- for (let write of this.writeQueue) {
+ for (let write of writeState.writes) {
  store.put(write.value, write.key);
-
- if (write.key === 0) {
- this.cachedFirstBlock = write.value;
- }
  }
 
  trans.onsuccess = () => {
  resolve();
  };
+ trans.onerror = () => {
+ console.log('Flushing writes failed');
+ reject();
+ };
  };
  req.onerror = reject;
  });
@@ -108,6 +144,7 @@ export class FileOpsFallback {
  // locally (we can't see any writes from anybody else) and we just
  // want to track the lock so we know when it downgrades from write
  // to read
+ this.cachedFirstBlock = this.blocks.get(0);
  this.lockType = lockType;
  return true;
  }
@@ -117,19 +154,42 @@ export class FileOpsFallback {
  // Downgrading the lock from a write lock to a read lock. This
  // is where we actually flush out all the writes async if
  // possible
- this.flushWrites();
+ let writeState = this.prepareFlush();
+ this.getDb(this.dbName).then(db => this.flushWriteState(db, writeState));
  }
+ this.lockType = lockType;
  return true;
  }
 
- delete() {}
+ delete() {
+ let req = globalThis.indexedDB.deleteDatabase(this.dbName);
+ req.onerror = () => {
+ console.warn(`Deleting ${this.filename} database failed`);
+ };
+ req.onsuccess = () => {};
+ }
 
  open() {}
 
  close() {
  // Clear out the in-memory data in close (it will have to be fully
  // read in before opening again)
- this.buffer = null;
+ // this.buffer = null;
+
+ if (this._openDb) {
+ // The order is important here: we want to flush out any pending
+ // writes, and we expect the db to open. We use that and then
+ // immediately close it, but since we are going to close it we
+ // don't want anything else to use that db connection. So we
+ // clear it out and then close it later
+ let db = this._openDb;
+ this._openDb = null;
+
+ let writeState = this.prepareFlush();
+ this.flushWriteState(db, writeState).then(() => {
+ db.close();
+ });
+ }
  }
 
  readMeta() {

src/indexeddb/worker.js

@@ -107,11 +107,7 @@ class Transaction {
  let block = await this.prefetchFirstBlock(500);
  // TODO: when timeouts are implemented, detect timeout and return BUSY
 
- if (cached0 == null && block == null) {
- return true;
- }
-
- return isSafeToWrite(new Uint8Array(block), new Uint8Array(cached0));
+ return isSafeToWrite(block, cached0);
  }
 
  downgradeShared() {

src/sqlite-file.js

@@ -123,12 +123,18 @@ export class File {
  this.ops.open();
  let meta = this.ops.readMeta();
 
- if (meta == null) {
- // New file
- meta = { size: 0 };
+ // It's possible that `setattr` has already been called if opening
+ // the file in a mode that truncates it to 0
+ if (this.meta == null) {
+ if (meta == null) {
+ // New file
+
+ meta = { size: 0 };
+ }
+
+ this.meta = meta;
  }
 
- this.meta = meta;
  return meta;
  }
 
@@ -308,9 +314,11 @@ export class File {
  return length;
  }
 
- readIfFallback() {
+ async readIfFallback() {
  if (this.ops.readIfFallback) {
- return this.ops.readIfFallback();
+ // Reset the meta
+ let meta = await this.ops.readIfFallback();
+ this.meta = meta || { size: 0 };
  }
  }
 
@@ -420,6 +428,10 @@ export class File {
  }
 
  setattr(attr) {
+ if (this.meta == null) {
+ this.meta = {};
+ }
+
  // Size is the only attribute we actually persist. The rest are
  // stored in memory
 

src/sqlite-util.js

@@ -17,16 +17,25 @@ export function getPageSize(bufferView) {
 }
 
 export function isSafeToWrite(localData, diskData) {
- // See
- // https://github.com/sqlite/sqlite/blob/master/src/pager.c#L93-L96
- // (might be documented somewhere? I didn't see it this clearly in
- // the docs). At least one of these bytes change when sqlite3 writes
- // data. We can check this against our in-memory data to see if it's
- // safe to write (if something changes underneath us, it's not)
- for (let i = 24; i < 40; i++) {
- if (localData[i] !== diskData[i]) {
- return false;
+ if (localData != null && diskData != null) {
+ let localView = new Uint8Array(localData);
+ let diskView = new Uint8Array(diskData);
+
+ // See
+ // https://github.com/sqlite/sqlite/blob/master/src/pager.c#L93-L96
+ // (might be documented somewhere? I didn't see it this clearly in
+ // the docs). At least one of these bytes change when sqlite3 writes
+ // data. We can check this against our in-memory data to see if it's
+ // safe to write (if something changes underneath us, it's not)
+ for (let i = 24; i < 40; i++) {
+ if (localView[i] !== diskView[i]) {
+ return false;
+ }
  }
+ return true;
  }
- return true;
+
+ // One of them is null, so it's only safe if to write if both are
+ // null, otherwise they are different
+ return localData == null && diskData == null;
 }