Add import/export support to Realtime Database

CHANGELOG.md

@@ -1,3 +1,4 @@
 - Adds support for regular expression-based custom headers and rewrites for Firebase Hosting. (#2391)
 - Fixes a bug with RTDB Rules hot reloading in the RTDB emulator. (#2371)
 - Changes default functions runtime to Node.js 10 for `firebase init`.
+- Adds import/export support to the Realtime Database emulator.

scripts/triggers-end-to-end-tests/tests.ts

@@ -264,4 +264,86 @@ describe("import/export end to end", () => {
 
  expect(true).to.be.true;
  });
+
+ it("should be able to import/export rtdb data", async function() {
+ // eslint-disable-next-line no-invalid-this
+ this.timeout(2 * TEST_SETUP_TIMEOUT);
+ await new Promise((resolve) => setTimeout(resolve, 2000));
+
+ // Start up emulator suite
+ const emulatorsCLI = new CLIProcess("1", __dirname);
+ await emulatorsCLI.start(
+ "emulators:start",
+ FIREBASE_PROJECT,
+ ["--only", "database"],
+ (data: unknown) => {
+ if (typeof data != "string" && !Buffer.isBuffer(data)) {
+ throw new Error(`data is not a string or buffer (${typeof data})`);
+ }
+ return data.includes(ALL_EMULATORS_STARTED_LOG);
+ }
+ );
+
+ // Write some data to export
+ const config = readConfig();
+ const app = admin.initializeApp(
+ {
+ databaseURL: "http:https://localhost:9000",
+ },
+ "rtdb-export"
+ );
+ const aRef = admin
+ .database(app)
+ .refFromURL(`http:https://localhost:${config.emulators!.database.host}/?ns=namespace-a`);
+ await aRef.set({
+ ns: "namespace-a",
+ });
+ const bRef = admin
+ .database(app)
+ .refFromURL(`http:https://localhost:${config.emulators!.database.host}/?ns=namespace-b`);
+ await bRef.set({
+ ns: "namespace-b",
+ });
+
+ // Ask for export
+ const exportCLI = new CLIProcess("2", __dirname);
+ const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data"));
+ await exportCLI.start("emulators:export", FIREBASE_PROJECT, [exportPath], (data: unknown) => {
+ if (typeof data != "string" && !Buffer.isBuffer(data)) {
+ throw new Error(`data is not a string or buffer (${typeof data})`);
+ }
+ return data.includes("Export complete");
+ });
+ await exportCLI.stop();
+
+ // Check that the right export files are created
+ const dbExportPath = path.join(exportPath, "database_export");
+ const dbExportFiles = fs.readdirSync(dbExportPath);
+ expect(dbExportFiles).to.eql(["namespace-a.json", "namespace-b.json"]);
+
+ // Stop the suite
+ await emulatorsCLI.stop();
+
+ // Attempt to import
+ const importCLI = new CLIProcess("3", __dirname);
+ await importCLI.start(
+ "emulators:start",
+ FIREBASE_PROJECT,
+ ["--only", "database", "--import", exportPath],
+ (data: unknown) => {
+ if (typeof data != "string" && !Buffer.isBuffer(data)) {
+ throw new Error(`data is not a string or buffer (${typeof data})`);
+ }
+ return data.includes(ALL_EMULATORS_STARTED_LOG);
+ }
+ );
+
+ // Read the data
+ const aSnap = await aRef.once("value");
+ const bSnap = await bRef.once("value");
+ expect(aSnap.val()).to.eql({ ns: "namespace-a" });
+ expect(bSnap.val()).to.eql({ ns: "namespace-b" });
+
+ await importCLI.stop();
+ });
 });

src/emulator/controller.ts

@@ -2,6 +2,7 @@ import * as _ from "lodash";
 import * as clc from "cli-color";
 import * as fs from "fs";
 import * as path from "path";
+import * as http from "http";
 
 import * as logger from "../logger";
 import * as track from "../track";
@@ -374,6 +375,52 @@ export async function startAll(options: any, noUi: boolean = false): Promise<voi
 
  const databaseEmulator = new DatabaseEmulator(args);
  await startEmulator(databaseEmulator);
+
+ if (exportMetadata.database) {
+ const importDirAbsPath = path.resolve(options.import);
+ const databaseExportDir = path.join(importDirAbsPath, exportMetadata.database.path);
+
+ const files = fs.readdirSync(databaseExportDir);
+ for (const f of files) {
+ const fPath = path.join(databaseExportDir, f);
+ const ns = f.split(".json")[0];
+
+ databaseLogger.logLabeled("BULLET", "database", `Importing data from ${fPath}`);
+
+ const readStream = fs.createReadStream(fPath);
+ await new Promise((resolve, reject) => {
+ const req = http.request(
+ {
+ method: "POST",
+ host: `${databaseAddr.host}`,
+ port: databaseAddr.port,
+ path: `/.json?ns=${ns}&disableTriggers=true`,
+ headers: {
+ Authorization: "Bearer owner",
+ "Content-Type": "application/json",
+ },
+ },
+ (response) => {
+ if (response.statusCode === 200) {
+ resolve();
+ } else {
+ databaseLogger.log("DEBUG", "Database import failed: " + response.statusCode);
+ response
+ .on("data", (d) => {
+ databaseLogger.log("DEBUG", d.toString());
+ })
+ .on("end", reject);
+ }
+ }
+ );
+
+ req.on("error", reject);
+ readStream.pipe(req, { end: true });
+ }).catch((e) => {
+ throw new FirebaseError("Error during database import.", { original: e, exit: 1 });
+ });
+ }
+ }
  }
 
  if (shouldStart(options, Emulators.HOSTING)) {

src/emulator/hubExport.ts

@@ -1,7 +1,9 @@
 import * as path from "path";
 import * as fs from "fs";
+import * as http from "http";
 
 import * as api from "../api";
+import * as logger from "../logger";
 import { IMPORT_EXPORT_EMULATORS, Emulators, ALL_EMULATORS } from "./types";
 import { EmulatorRegistry } from "./registry";
 import { FirebaseError } from "../error";
@@ -15,6 +17,10 @@ export interface ExportMetadata {
  path: string;
  metadata_file: string;
  };
+ database?: {
+ version: string;
+ path: string;
+ };
 }
 
 export class HubExport {
@@ -53,6 +59,14 @@ export class HubExport {
  await this.exportFirestore(metadata);
  }
 
+ if (this.shouldExport(Emulators.DATABASE)) {
+ metadata.database = {
+ version: getDownloadDetails(Emulators.DATABASE).version,
+ path: "database_export",
+ };
+ await this.exportDatabase(metadata);
+ }
+
  const metadataPath = path.join(this.exportPath, HubExport.METADATA_FILE_NAME);
  fs.writeFileSync(metadataPath, JSON.stringify(metadata));
  }
@@ -74,6 +88,43 @@ export class HubExport {
  });
  }
 
+ private async exportDatabase(metadata: ExportMetadata): Promise<void> {
+ const databaseInfo = EmulatorRegistry.get(Emulators.DATABASE)!.getInfo();
+ const databaseAddr = `http:https://${databaseInfo.host}:${databaseInfo.port}`;
+
+ // Make sure the export directory exists
+ if (!fs.existsSync(this.exportPath)) {
+ fs.mkdirSync(this.exportPath);
+ }
+
+ const dbExportPath = path.join(this.exportPath, metadata.database!.path);
+ if (!fs.existsSync(dbExportPath)) {
+ fs.mkdirSync(dbExportPath);
+ }
+
+ // Get the list of namespaces
+ const inspectURL = `http:https://${databaseInfo.host}:${databaseInfo.port}/.inspect/databases.json?ns=${this.projectId}`;
+ const inspectRes = await api.request("GET", inspectURL);
+
+ for (const instance of inspectRes.body) {
+ logger.debug(`Exporting database instance: ${instance}`);
+
+ const ns = instance.name;
+ const url = `${databaseAddr}/.json?ns=${ns}`;
+
+ const exportFile = path.join(dbExportPath, `${ns}.json`);
+ const writeStream = fs.createWriteStream(exportFile);
+
+ await new Promise((resolve, reject) => {
+ http
+ .get(url, (response) => {
+ response.pipe(writeStream).once("close", resolve);
+ })
+ .on("error", reject);
+ });
+ }
+ }
+
  private shouldExport(e: Emulators): boolean {
  return IMPORT_EXPORT_EMULATORS.indexOf(e) >= 0 && EmulatorRegistry.isRunning(e);
  }

src/emulator/types.ts

@@ -24,8 +24,8 @@ export const DOWNLOADABLE_EMULATORS = [
  Emulators.UI,
 ];
 
-export type ImportExportEmulators = Emulators.FIRESTORE;
-export const IMPORT_EXPORT_EMULATORS = [Emulators.FIRESTORE];
+export type ImportExportEmulators = Emulators.FIRESTORE | Emulators.DATABASE;
+export const IMPORT_EXPORT_EMULATORS = [Emulators.FIRESTORE, Emulators.DATABASE];
 
 export const ALL_SERVICE_EMULATORS = [
  Emulators.FUNCTIONS,