WIP webkitFileSystem backend

rollup.config.js

@@ -45,5 +45,9 @@ export default [
  getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js'),
  getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js'),
  getConfig('src/indexeddb/backend.js', 'indexeddb-backend.js', true),
- getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true)
+ getConfig('src/indexeddb/main-thread.js', 'indexeddb-main-thread.js', true),
+ getConfig('src/webkitFileSystem/backend.js', 'webkitFileSystem-backend.js'),
+ getConfig('src/webkitFileSystem/main-thread.js', 'webkitFileSystem-main-thread.js'),
+ getConfig('src/webkitFileSystem/backend.js', 'webkitFileSystem-backend.js', true),
+ getConfig('src/webkitFileSystem/main-thread.js', 'webkitFileSystem-main-thread.js', true)
 ];

src/examples/bench/main.worker.js

@@ -2,19 +2,21 @@ import initSqlJs from '@jlongster/sql.js';
 import { SQLiteFS } from '../..';
 import MemoryBackend from '../../memory/backend';
 import IndexedDBBackend from '../../indexeddb/backend';
+import webkitFileSystemBackend from '../../webkitFileSystem/backend';
 import * as queries from './queries';
 import * as rawIDBQueries from './queries-raw-idb';
 
 // Various global state for the demo
 let currentBackendType = 'idb';
 let cacheSize = 0;
 let pageSize = 4096;
-let dbName = `db21.sqlite`;
+let dbName = `db23.sqlite`;
 let recordProfile = false;
 let useRawIDB = false;
 
 let memoryBackend = new MemoryBackend({});
 let idbBackend = new IndexedDBBackend();
+let fsBackend = new webkitFileSystemBackend();
 let sqlFS;
 
 // Helper methods
@@ -23,7 +25,7 @@ let SQL = null;
 async function init() {
  if (SQL == null) {
  SQL = await initSqlJs({ locateFile: file => file });
- sqlFS = new SQLiteFS(SQL.FS, idbBackend);
+ sqlFS = new SQLiteFS(SQL.FS, fsBackend);
  SQL.register_for_idb(sqlFS);
 
  if (typeof SharedArrayBuffer === 'undefined') {
@@ -172,7 +174,7 @@ async function populateSmall() {
 async function populateLarge() {
  clearTimings();
 
- let count = 400000;
+ let count = 50000;
  if (currentBackendType === 'memory') {
  output(
  'Cannot write 1,000,000 items to memory backend, reducing to 100,000'
@@ -329,6 +331,8 @@ if (typeof self !== 'undefined') {
  // but it works for the demo
  if (currentBackendType === 'memory') {
  sqlFS.backend = memoryBackend;
+ } else if (currentBackendType === 'fs') {
+ sqlFS.backend = fsBackend;
  } else {
  sqlFS.backend = idbBackend;
  }

src/indexeddb/backend.js

@@ -1,6 +1,5 @@
 import { File } from '../sqlite-file';
 import * as perf from 'perf-deets';
-import { LOCK_TYPES, getPageSize } from '../sqlite-util';
 import { FileOps } from './file-ops';
 import { FileOpsFallback } from './file-ops-fallback';
 

src/indexeddb/file-ops.js

@@ -1,4 +1,4 @@
-import { Reader, Writer } from './shared-channel';
+import { Reader, Writer } from '../shared-channel';
 
 function positionToKey(pos, blockSize) {
  // We are forced to round because of floating point error. `pos`

src/indexeddb/worker.js

@@ -1,4 +1,4 @@
-import { Reader, Writer } from './shared-channel';
+import { Reader, Writer } from '../shared-channel';
 import * as perf from 'perf-deets';
 import { LOCK_TYPES, isSafeToWrite } from '../sqlite-util';
 

src/sqlite-file.js

@@ -122,6 +122,7 @@ export class File {
  open() {
  this.ops.open();
  let meta = this.ops.readMeta();
+ console.log('got meta', meta);
 
  // It's possible that `setattr` has already been called if opening
  // the file in a mode that truncates it to 0
@@ -216,7 +217,7 @@ export class File {
  }
 
  write(bufferView, offset, length, position) {
- // console.log('writing', this.filename, offset, length, position);
+ console.log('writing', this.filename, offset, length, position);
 
  if (this.meta.blockSize == null) {
  // We don't have a block size yet (an empty file). The first

src/webkitFileSystem/backend.js

@@ -0,0 +1,100 @@
+import { File } from '../sqlite-file';
+import { getPageSize } from '../sqlite-util';
+import * as perf from 'perf-deets';
+
+class FileOps {
+ constructor(filename, rootEntry) {
+ this.filename = filename;
+ this.rootEntry = rootEntry;
+ }
+
+ getDatabaseName() {
+ return this.filename.replace(/\//g, '-');
+ }
+
+ lock(lockType) {
+ return true;
+ }
+
+ unlock(lockType) {
+ return true;
+ }
+
+ delete() {
+ console.log('deleting');
+ this.entry.remove();
+ }
+
+ open() {
+ this.entry = this.rootEntry.getFile(this.getDatabaseName(), {
+ create: true
+ });
+ }
+
+ close() {
+ // close
+ }
+
+ readMeta() {
+ let file = this.entry.file();
+ let reader = new FileReaderSync();
+
+ let blockSize = null;
+ if (file.size > 0) {
+ let data = reader.readAsArrayBuffer(file.slice(16, 18));
+ let arr = new Uint16Array(data);
+ blockSize = arr[0] * 256;
+ }
+
+ return { size: file.size, blockSize: blockSize == 0 ? null : blockSize };
+ }
+
+ writeMeta(meta) {
+ let writer = this.entry.createWriter();
+ writer.truncate(meta.size);
+ return 0;
+ }
+
+ readBlocks(positions, blockSize) {
+ let res = [];
+ let file = this.entry.file();
+ let reader = new FileReaderSync();
+ for (let pos of positions) {
+ let data = reader.readAsArrayBuffer(file.slice(pos, pos + blockSize));
+ res.push({
+ pos,
+ data: data.byteLength === 0 ? new ArrayBuffer(blockSize) : data
+ });
+ }
+ return res;
+ }
+
+ writeBlocks(writes, blockSize) {
+ for (let write of writes) {
+ let writer = this.entry.createWriter();
+ writer.seek(write.pos);
+ writer.write(new Blob([write.data], { type: 'octet/stream' }));
+ }
+ return 0;
+ }
+}
+
+export default class webkitFileSystemBackend {
+ constructor() {
+ // TODO: need to do `navigator.webkitPersistentStorage.requestQuota(10000000, function (quota) { console.log(quota) })`
+ this.FS = self.webkitRequestFileSystemSync(self.PERSISTENT, 100000);
+ this.rootEntry = this.FS.root.getDirectory('dbs', { create: true });
+ }
+
+ createFile(filename) {
+ return new File(filename, new FileOps(filename, this.rootEntry));
+ }
+
+ startProfile() {
+ perf.start();
+ }
+
+ stopProfile() {
+ perf.stop();
+ }
+}