Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: BQ integration #1

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Add foundation for coercing Biquery field types no native types.
  • Loading branch information
bilus committed Sep 7, 2022
commit 2efcb77acd67f77765d9a9fdf745330185101886
38 changes: 35 additions & 3 deletions packages/server/src/integrations/bigQuery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,16 @@ interface BigQueryConfig extends BigQueryOptions {
privateKey: string
}

type RawRow = {
[key: string]: any
}

type CoercedFieldValue = string | Date | null

type CoercedRow = {
[key: string]: CoercedFieldValue
}

class BigQueryIntegration {
private client: any
private static readonly BIG_QUERY_SCOPES = [
Expand Down Expand Up @@ -112,7 +122,10 @@ class BigQueryIntegration {
async read(query: SqlQuery) {
const [job] = await this.internalQuery(query)
const [rows] = await job.getQueryResults()
return rows
rows.forEach((row: any) => {
console.log(row)
})
return rows.map((row: RawRow) => this.coerceRow(row))
}

async update(query: SqlQuery) {
Expand All @@ -125,9 +138,28 @@ class BigQueryIntegration {

async buildSchema(datasourceId: string, entities: Record<string, Table>) {
// fetch all existing tables
// const dataset = this.client.dataset(this.datasetId)
// const tables = await dataset.getTables()
}

const dataset = this.client.dataset(this.datasetId)
const tables = await dataset.getTables()
coerceRow(row: RawRow): CoercedRow {
return Object.keys(row).reduce((newRow: CoercedRow, key: string) => {
newRow[key] = this.coerceValue(row[key])
return newRow
}, {})
}

coerceValue(v: any): CoercedFieldValue {
// TODO: Add support for all Bigquery types (e.g. BigQueryTimestamp).
if (v === null || v === undefined) {
return null
} else if (typeof v.value === "function") {
return v.value()
} else if (typeof v === "string") {
return v
} else {
return "UNSUPPORTED"
}
}
}

Expand Down