feat(ext/kv): connect to remote database (denoland#20178)

This patch adds a `remote` backend for `ext/kv`. This supports
connection to Deno Deploy and potentially other services compatible with
the KV Connect protocol.

.github/workflows/bench_cron.yml

@@ -31,6 +31,12 @@ jobs:
 
  - uses: dsherret/rust-toolchain-file@v1
 
+ - name: Install protoc
+ uses: arduino/setup-protoc@v2
+ with:
+ version: "21.12"
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
  - name: Build release
  run: cargo build --release --locked --all-targets
 

.github/workflows/ci.generate.ts

@@ -168,6 +168,11 @@ const installNodeStep = {
  uses: "actions/setup-node@v3",
  with: { "node-version": 18 },
 };
+const installProtocStep = {
+ name: "Install protoc",
+ uses: "arduino/setup-protoc@v2",
+ with: { "version": "21.12", "repo-token": "${{ secrets.GITHUB_TOKEN }}" },
+};
 const installDenoStep = {
  name: "Install Deno",
  uses: "denoland/setup-deno@v1",
@@ -434,6 +439,7 @@ const ci = {
  if: "matrix.job == 'bench'",
  ...installNodeStep,
  },
+ installProtocStep,
  {
  if: [
  "matrix.profile == 'release' &&",

.github/workflows/ci.yml

@@ -170,6 +170,12 @@ jobs:
  uses: actions/setup-node@v3
  with:
  node-version: 18
+ - name: Install protoc
+ uses: arduino/setup-protoc@v2
+ with:
+ version: '21.12'
+ repo-token: '${{ secrets.GITHUB_TOKEN }}'
+ if: '!(github.event_name == ''pull_request'' && matrix.skip_pr)'
  - if: |-
  !(github.event_name == 'pull_request' && matrix.skip_pr) && (matrix.profile == 'release' &&
  matrix.job == 'test' &&

Cargo.toml

@@ -82,6 +82,7 @@ brotli = "3.3.4"
 bytes = "1.4.0"
 cache_control = "=0.2.0"
 cbc = { version = "=0.1.2", features = ["alloc"] }
+chrono = { version = "=0.4.26", default-features = false, features = ["std", "serde", "clock"] }
 console_static_text = "=0.8.1"
 data-url = "=0.2.0"
 dlopen = "0.1.8"
@@ -110,10 +111,12 @@ parking_lot = "0.12.0"
 percent-encoding = "=2.3.0"
 pin-project = "1.0.11" # don't pin because they yank crates from cargo
 pretty_assertions = "=1.3.0"
+prost = "0.11"
+prost-build = "0.11"
 rand = "=0.8.5"
 regex = "^1.7.0"
 lazy-regex = "2.5.0"
-reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks"] }
+reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] }
 ring = "=0.16.20"
 rusqlite = { version = "=0.29.0", features = ["unlock_notify", "bundled"] }
 rustls = "0.21.0"

README.md

@@ -60,6 +60,11 @@ scoop install deno
 Build and install from source using [Cargo](https://crates.io/crates/deno):
 
 ```sh
+# Install the Protobuf compiler
+apt install -y protobuf-compiler # Linux
+brew install protobuf # macOS
+
+# Build and install Deno
 cargo install deno --locked
 ```
 

cli/Cargo.toml

@@ -66,7 +66,7 @@ base32 = "=0.4.0"
 base64.workspace = true
 bincode = "=1.3.3"
 cache_control.workspace = true
-chrono = { version = "=0.4.26", default-features = false, features = ["std"] }
+chrono.workspace = true
 clap = { version = "=4.3.3", features = ["string"] }
 clap_complete = "=4.3.1"
 clap_complete_fig = "=4.3.1"

cli/schemas/kv-metadata-exchange-response.v1.json

@@ -0,0 +1,54 @@
+{
+ "$id": "https://deno.land/x/deno/cli/schemas/kv-metadata-exchange-response.v1.json",
+ "$schema": "http:https://json-schema.org/draft-07/schema#",
+ "definitions": {
+ "Uuid": {
+ "type": "string",
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ },
+ "DateTime": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "EndpointInfo": {
+ "type": "object",
+ "properties": {
+ "url": {
+ "type": "string"
+ },
+ "consistency": {
+ "type": "string"
+ }
+ },
+ "required": ["url", "consistency"],
+ "additionalProperties": false
+ },
+ "DatabaseMetadata": {
+ "type": "object",
+ "properties": {
+ "version": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "databaseId": {
+ "$ref": "#/definitions/Uuid"
+ },
+ "endpoints": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/EndpointInfo"
+ }
+ },
+ "token": {
+ "type": "string"
+ },
+ "expiresAt": {
+ "$ref": "#/definitions/DateTime"
+ }
+ },
+ "required": ["version", "databaseId", "endpoints", "token", "expiresAt"],
+ "additionalProperties": false
+ }
+ },
+ "$ref": "#/definitions/DatabaseMetadata"
+}

cli/tests/unit/kv_test.ts

@@ -20,6 +20,9 @@ try {
  isCI = true;
 }
 
+// Defined in test_util/src/lib.rs
+Deno.env.set("DENO_KV_ACCESS_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+
 Deno.test({
  name: "openKv :memory: no permissions",
  permissions: {},
@@ -1932,3 +1935,74 @@ Deno.test({
  }
  },
 });
+
+Deno.test({
+ name: "remote backend",
+ async fn() {
+ const db = await Deno.openKv("http:https://localhost:4545/kv_remote_authorize");
+ try {
+ await db.set(["some-key"], 1);
+ const entry = await db.get(["some-key"]);
+ assertEquals(entry.value, null);
+ assertEquals(entry.versionstamp, null);
+ } finally {
+ db.close();
+ }
+ },
+});
+
+Deno.test({
+ name: "remote backend invalid format",
+ async fn() {
+ const db = await Deno.openKv(
+ "http:https://localhost:4545/kv_remote_authorize_invalid_format",
+ );
+ let ok = false;
+ try {
+ await db.set(["some-key"], 1);
+ } catch (e) {
+ if (
+ e.name === "TypeError" &&
+ e.message.startsWith("Metadata error: Failed to decode metadata: ")
+ ) {
+ ok = true;
+ } else {
+ throw e;
+ }
+ } finally {
+ db.close();
+ }
+
+ if (!ok) {
+ throw new Error("did not get expected error");
+ }
+ },
+});
+
+Deno.test({
+ name: "remote backend invalid version",
+ async fn() {
+ const db = await Deno.openKv(
+ "http:https://localhost:4545/kv_remote_authorize_invalid_version",
+ );
+ let ok = false;
+ try {
+ await db.set(["some-key"], 1);
+ } catch (e) {
+ if (
+ e.name === "TypeError" &&
+ e.message === "Metadata error: Unsupported metadata version: 2"
+ ) {
+ ok = true;
+ } else {
+ throw e;
+ }
+ } finally {
+ db.close();
+ }
+
+ if (!ok) {
+ throw new Error("did not get expected error");
+ }
+ },
+});

ext/kv/Cargo.toml

@@ -17,13 +17,20 @@ path = "lib.rs"
 anyhow.workspace = true
 async-trait.workspace = true
 base64.workspace = true
+chrono.workspace = true
 deno_core.workspace = true
 hex.workspace = true
 log.workspace = true
 num-bigint.workspace = true
+prost.workspace = true
 rand.workspace = true
+reqwest.workspace = true
 rusqlite.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
+url.workspace = true
 uuid.workspace = true
+
+[build-dependencies]
+prost-build.workspace = true