feat: add big query integration

packages/builder/src/components/backend/DatasourceNavigator/icons/BigQuery.svelte

@@ -0,0 +1,17 @@
+<script>
+ export let width = "100"
+ export let height = "100"
+</script>
+
+<svg {width} {height} viewBox="0 0 64 64" xmlns="http:https://www.w3.org/2000/svg">
+ <path
+ d="M14.48 58.196L.558 34.082c-.744-1.288-.744-2.876 0-4.164L14.48 5.805c.743-1.287 2.115-2.08 3.6-2.082h27.857c1.48.007 2.845.8 3.585 2.082l13.92 24.113c.744 1.288.744 2.876 0 4.164L49.52 58.196c-.743 1.287-2.115 2.08-3.6 2.082H18.07c-1.483-.005-2.85-.798-3.593-2.082z"
+ fill="#4386fa"
+ /><path
+ d="M40.697 24.235s3.87 9.283-1.406 14.545-14.883 1.894-14.883 1.894L43.95 60.27h1.984c1.486-.002 2.858-.796 3.6-2.082L58.75 42.23z"
+ opacity=".1"
+ /><path
+ d="M45.267 43.23L41 38.953a.67.67 0 0 0-.158-.12 11.63 11.63 0 1 0-2.032 2.037.67.67 0 0 0 .113.15l4.277 4.277a.67.67 0 0 0 .947 0l1.12-1.12a.67.67 0 0 0 0-.947zM31.64 40.464a8.75 8.75 0 1 1 8.749-8.749 8.75 8.75 0 0 1-8.749 8.749zm-5.593-9.216v3.616c.557.983 1.363 1.803 2.338 2.375v-6.013zm4.375-2.998v9.772a6.45 6.45 0 0 0 2.338 0V28.25zm6.764 6.606v-2.142H34.85v4.5a6.43 6.43 0 0 0 2.338-2.368z"
+ fill="#fff"
+ />
+</svg>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -16,6 +16,7 @@ import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
 import Custom from "./Custom.svelte"
+import BigQuery from "./BigQuery.svelte"
 
 const ICONS = {
  BUDIBASE: Budibase,
@@ -36,6 +37,7 @@ const ICONS = {
  REDIS: Redis,
  SNOWFLAKE: Snowflake,
  CUSTOM: Custom,
+ BIG_QUERY: BigQuery,
 }
 
 export default ICONS

packages/builder/src/constants/backend/index.js

@@ -183,6 +183,7 @@ export const IntegrationTypes = {
  FIRESTORE: "FIRESTORE",
  REDIS: "REDIS",
  SNOWFLAKE: "SNOWFLAKE",
+ BIG_QUERY: "BIG_QUERY",
 }
 
 export const IntegrationNames = {
@@ -203,6 +204,7 @@ export const IntegrationNames = {
  [IntegrationTypes.FIRESTORE]: "Firestore",
  [IntegrationTypes.REDIS]: "Redis",
  [IntegrationTypes.SNOWFLAKE]: "Snowflake",
+ [IntegrationTypes.BIG_QUERY]: "Big Query",
 }
 
 export const SchemaTypeOptions = [

packages/builder/src/constants/index.js

@@ -22,6 +22,7 @@ export const IntegrationNames = {
  ARANGODB: "ArangoDB",
  ORACLE: "Oracle",
  GOOGLE_SHEETS: "Google Sheets",
+ BIG_QUERY: "Big Query",
 }
 
 // fields on the user table that cannot be edited

packages/server/package.json

@@ -85,6 +85,7 @@
  "@bull-board/api": "3.7.0",
  "@bull-board/koa": "3.9.4",
  "@elastic/elasticsearch": "7.10.0",
+ "@google-cloud/bigquery": "^6.0.2",
  "@google-cloud/firestore": "5.0.2",
  "@koa/router": "8.0.0",
  "@sendgrid/mail": "7.1.1",

packages/server/src/integrations/bigQuery.ts

@@ -0,0 +1,136 @@
+import {
+ DatasourceFieldTypes,
+ Integration,
+ QueryTypes,
+ SqlQuery,
+} from "../definitions/datasource"
+
+import { BigQuery, BigQueryOptions } from "@google-cloud/bigquery"
+import { Table } from "../definitions/common"
+
+const SCHEMA: Integration = {
+ docs: "https://cloud.google.com/bigquery/docs",
+ datasource: {
+ projectId: {
+ type: DatasourceFieldTypes.STRING,
+ required: true,
+ },
+ datasetId: {
+ type: DatasourceFieldTypes.STRING,
+ required: true,
+ },
+ apiEndpoint: {
+ type: DatasourceFieldTypes.STRING,
+ required: false,
+ },
+ email: {
+ type: DatasourceFieldTypes.STRING,
+ required: true,
+ },
+ privateKey: {
+ type: DatasourceFieldTypes.STRING,
+ required: true,
+ },
+ },
+ query: {
+ create: {
+ type: QueryTypes.SQL,
+ },
+ read: {
+ type: QueryTypes.SQL,
+ },
+ update: {
+ type: QueryTypes.SQL,
+ },
+ delete: {
+ type: QueryTypes.SQL,
+ },
+ },
+ friendlyName: "BigQuery",
+ type: "Relational",
+ description:
+ "BigQuery is a serverless, cost-effective and multicloud data warehouse designed to help you turn big data into valuable business insights.",
+}
+
+interface BigQueryConfig extends BigQueryOptions {
+ projectId: string
+ datasetId: string
+ apiEndpoint?: string
+ email: string
+ privateKey: string
+}
+
+class BigQueryIntegration {
+ private client: any
+ private static readonly BIG_QUERY_SCOPES = [
+ "https://www.googleapis.com/auth/bigquery",
+ "https://www.googleapis.com/auth/drive",
+ ]
+ private readonly datasetId: string
+ constructor(config: BigQueryConfig) {
+ this.client = new BigQuery({
+ projectId: config.projectId,
+ apiEndpoint: config.apiEndpoint ? config.apiEndpoint : undefined,
+ credentials: {
+ client_email: config.email,
+ private_key: config.privateKey?.replace(/\\n/g, "\n"),
+ },
+ scopes: BigQueryIntegration.BIG_QUERY_SCOPES,
+ })
+
+ this.datasetId = config.datasetId
+
+ this.createDatasetIfDoesNotExist()
+ this.buildSchema("hello", {})
+ }
+
+ async createDatasetIfDoesNotExist() {
+ try {
+ const existingDataset = await this.client.dataset(this.datasetId).get()
+ if (!existingDataset) {
+ await this.client.createDataset(this.datasetId)
+ }
+ } catch (err: any) {
+ throw err?.message.split(":")[1] || err?.message
+ }
+ }
+
+ async internalQuery(query: SqlQuery) {
+ try {
+ return await this.client.createQueryJob(query.sql)
+ } catch (err: any) {
+ throw err?.message.split(":")[1] || err?.message
+ }
+ }
+
+ async create(query: SqlQuery) {
+ return this.internalQuery(query)
+ }
+
+ async read(query: SqlQuery) {
+ const [job] = await this.internalQuery(query)
+ const [rows] = await job.getQueryResults()
+ return rows
+ }
+
+ async update(query: SqlQuery) {
+ return this.internalQuery(query)
+ }
+
+ async delete(query: SqlQuery) {
+ return this.internalQuery(query)
+ }
+
+ async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+ // fetch all existing tables
+
+ const dataset = this.client.dataset(this.datasetId)
+ const tables = await dataset.getTables()
+ console.log(tables)
+ }
+}
+
+module.exports = {
+ schema: SCHEMA,
+ integration: BigQueryIntegration,
+}

packages/server/src/integrations/index.ts

@@ -13,6 +13,7 @@ import googlesheets from "./googlesheets"
 import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
+import bigQuery from "./bigQuery"
 import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
@@ -34,6 +35,7 @@ const DEFINITIONS: { [key: string]: Integration } = {
  [SourceName.FIRESTORE]: firebase.schema,
  [SourceName.REDIS]: redis.schema,
  [SourceName.SNOWFLAKE]: snowflake.schema,
+ [SourceName.BIG_QUERY]: bigQuery.schema,
 }
 
 const INTEGRATIONS: { [key: string]: any } = {
@@ -53,6 +55,7 @@ const INTEGRATIONS: { [key: string]: any } = {
  [SourceName.REDIS]: redis.integration,
  [SourceName.FIRESTORE]: firebase.integration,
  [SourceName.SNOWFLAKE]: snowflake.integration,
+ [SourceName.BIG_QUERY]: bigQuery.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

packages/server/src/integrations/tests/bigQuery.spec.js

@@ -0,0 +1,31 @@
+const bigQuery = require("@google-cloud/bigquery")
+const BigQueryIntegration = require("../bigQuery")
+jest.mock("@google-cloud/bigquery")
+
+class TestConfiguration {
+ constructor(config = {}) {
+ this.integration = new BigQueryIntegration.integration(config) 
+ }
+}
+
+describe("BigQuery Integration", () => {
+ let config 
+
+ beforeEach(() => {
+ config = new TestConfiguration({
+ projectId: 'projectId',
+ email: '[email protected]',
+ privateKey: '-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDUzve3Eqp3PRFN\nzcNkzPSNn2v3pUzuuaC1M8T073Qox7XQ3uDzjZ57Ne29y08p6qiJbLyOyAANrtuU\n6pzMS3zUs7+joP9WmX1WszdzKZB2aI6++cAwtbNo+A3TS4ybCfyJH+UG0QkMQ6Q3\npyQH+qs5Q3/B3OaQtgvcrhm0xA7ZE/tE3HrpIWkGGYO7peFctfft/ZT6PF4SGAc/\nTxovLX/+tPbMOcyVpBhB+Sq+AH00yIHFFDRMVrS7pMK++4A3YSQn/8d2yLjrp/jB\nRGUMzKYpMMP9Fr5G3CI9iLBL4cY8Qlqf7ktgat1xem4n9wX9DF3vGdLSN9KEATyz\nCSL6lpbtAgMBAAECggEAGxA5G4yhpilbfoAI9dRE04/vLJ7WglOUFwc+6IFdTeRc\nAHSfugmMGpI4qlblOejwRw3Piv7vVs4iebU+fJKrDOfRGs+aOVqqwHBpQFmMWCqi\n4LGlWex5pQNKytDEUGE8PvoR58SEuI0PoM4+PE+KH+2wHSjpH4UvTADOHxmB4+Q1\ntQlbAfuWUvg8PuEfEzo9F1ilhiklHuZc1ut6OAtdD80BooLd+Tl27kdJhvQAHInZ\njJCnD0/XuK8dgZyzSxjT1exNLSYsAXdeEWXb33dMz+MBzMqD3jXXdBCpOByubLGs\nsj50Tuf/BHc/i4TbnCuDRHX7JDJB6UEBuiHwI8p/cQKBgQD0pbn5xl3tasIEbhCn\n02UKDHpW7HnrqC+ElYwoVyfv6LGQfuCVEIfj9JbRhrTjh0OdPlMSa3ALsrN3z0Qe\nDxKFeMlEnrnfg2WpNGe7X0PIhbTkzDtNJePqxMCXmBOUh40RA2L+XV2Lswgn71uk\nfnd3UW+A/CY0+I/tgufqD7DjuQKBgQDerwNLwcKwromKucxkPGEHoid7tMFwpWug\n3QQqTHJyXQ59bjbYsqH7VSIY49MV1FhOYBPA9Mrcj0hdqn3+edKnQwM+WXfI4MK3\nd/o/wpmskIXBreQhKzzQr0CdvTEwkr+4CXUXfn00HqumoSacc8/gfSkUiZ7btoIw\nLR1UFB0O1QKBgHjs+fI2VPMnk+MwrFboLMc8x7Pzi4gqR+KXMQI3omv5bttne4by\n9th8a5gBp6PXllpBFjrClE2T9RXBg4AAHz2OKJ4cfu+2OSfb2XJKcmzJelKliKJn\nmjLPMgs8hmEiZ14DeIkWiUimI9/pdjjmshJuVFlDSXdhbXMPA6c0PlExAoGAcaWy\nMeyeVxuMmJ9AX/usrX+lVO4oNzxFVKDXqlq/ofw6E+u21Bs+rg2BzGAhb6eitcU0\n76o/CheaICuOB9zWlISP2DdC+eMznPz/W6EOWtKbYQBFSGRPslVuzdIrk5WhgORa\nvPXSIlJw2iaulPRKKFDYMWIXEBzyDnJH4IwvVE0CgYEA5XvZi5fsDh+zOnAvGmbH\n6amOgdLAQEXzyuFk7Q3bqPWzE5IbHu7dESC6V6vt3XXcrgi49fLa1YUuuLkJGlq9\nE3i0KGunR1jBlG86wZAgEYBZ+v8lqptflVR+UUNNTWzicH7EOtbn+3P9BrrU29aw\n6XXUZWEqYg4s54aQd/L1Ggs=\n-----END PRIVATE KEY-----\n'
+ })
+ })
+
+ it("calls the read method with the correct params", async () => {
+ const sql = "select * from users;"
+ await config.integration.read({
+ sql
+ })
+ expect(bigQuery.createQueryJob).toHaveBeenCalledWith(sql)
+ })
+
+
+ })

packages/types/src/sdk/datasources.ts

@@ -51,6 +51,7 @@ export enum SourceName {
  FIRESTORE = "FIRESTORE",
  REDIS = "REDIS",
  SNOWFLAKE = "SNOWFLAKE",
+ BIG_QUERY = "BIG_QUERY",
  UNKNOWN = "unknown",
 }