Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 8x 8x 8x 8x 8x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 2x 2x 2x 2x 1x 1x 1x 1x 2x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 25x 25x 25x 25x 25x 24x 24x 25x 1x 1x 25x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | export {}
const { BigQuery } = require('@google-cloud/bigquery')
const { Storage } = require('@google-cloud/storage')
const GoogleCloudAdapter = require('../GoogleCloudAdapter')
const { ErrorUtils } = require('../../../utils/error')
const $error = new ErrorUtils()
const $storage = new Storage()
// type TGoogleCloudLocation = string
type TBigQueryDatasetName = string
type TBigQueryTableName = string
type TBigQueryCreateDatasetProps = {
name: string
}
type TBigQueryGetDatasetProps = {
name: TBigQueryDatasetName
}
type TBigQueryDeleteDatasetProps = {
name: TBigQueryDatasetName
force?: boolean
}
type TBigQueryLoadFromBucketProps = {
dataset: TBigQueryDatasetName
table: TBigQueryTableName
bucketName: string
fileName: string
sourceFormat?: string
skipLeadingRows?: number
schema: object
}
type TBigQueryRunQueryProps = {
sql: string
}
type TBigQueryProps = {
location?: string
}
// function mapOperatorToSql(operator: string) {
// const operatorMap = {
// equals: '=',
// notEquals: '!=',
// lessThan: '<',
// lessOrEquals: '<=',
// greaterThan: '>',
// greaterOrEquals: '>=',
// includes: 'LIKE',
// notIncludes: 'NOT LIKE',
// in: 'IN',
// notIn: 'NOT IN',
// // Add more operators as needed
// }
// if (operatorMap[operator as keyof typeof operatorMap] === undefined) {
// throw new Error(`Operator ${operator} is not supported`)
// }
// return operatorMap[operator as keyof typeof operatorMap]
// }
// function formatValueForSql(value: any) {
// // Add logic here to format the value based on its type (string, number, date, etc.)
// if (typeof value === 'string') {
// if (value.includes('(') && value.includes(')')) {
// return value
// }
// return `'${value}'`
// }
// return value
// }
export class BigQueryAdapter extends GoogleCloudAdapter {
constructor(props?: TBigQueryProps) {
super()
this.location = props?.location ?? 'us-central1'
this.bigquery = new BigQuery(props)
// this.parseFilterToSql = this.parseFilterToSql.bind(this)
}
async createDataset({ name }: TBigQueryCreateDatasetProps) {
try {
const [dataset] = await this.bigquery.dataset(name).create({
location: this.location,
})
return dataset
} catch (error) {
throw $error.errorHandler({ error })
}
}
async getDataset({ name }: TBigQueryGetDatasetProps) {
try {
const [dataset] = await this.bigquery.dataset(name).get({
location: this.location,
})
return dataset
} catch (error) {
throw $error.errorHandler({ error })
}
}
async deleteDataset({ name, force = true }: TBigQueryDeleteDatasetProps) {
try {
await this.bigquery
.dataset(name)
.delete({ location: this.location, force })
return true
} catch (error) {
throw $error.errorHandler({ error })
}
}
async loadFromBucket(props: TBigQueryLoadFromBucketProps) {
try {
const metadata = {
sourceFormat: props.sourceFormat ?? 'CSV',
skipLeadingRows: props.skipLeadingRows ?? 0,
schema: {
fields: props.schema,
},
fieldDelimiter: '~',
location: this.location,
}
const storageRef = $storage.bucket(props.bucketName).file(props.fileName)
const [job] = await this.bigquery
.dataset(props.dataset)
.table(props.table)
.load(storageRef, metadata)
if (job.status.errors && job.status.errors.length > 0) {
throw new Error(job.status.errors)
}
return job
} catch (error) {
throw $error.errorHandler({ error })
}
}
async runQuery(props: TBigQueryRunQueryProps) {
try {
const [job] = await this.bigquery.createQueryJob({
query: props.sql,
location: this.location,
})
const [rows] = await job.getQueryResults()
return rows
} catch (error) {
throw $error.errorHandler({ error })
}
}
// parseFilterToSql(filter: any) {
// const self = this
// if (!filter) {
// return ''
// }
// if (filter.or) {
// const orConditions = filter.or.map(self.parseFilterToSql)
// return `(${orConditions.join(' OR ')})`
// }
// if (filter.and) {
// const andConditions = filter.and.map(self.parseFilterToSql)
// return `(${andConditions.join(' AND ')})`
// }
// if (filter.operator && filter.field && filter.value) {
// const { operator, field, value } = filter
// return `${field} ${mapOperatorToSql(operator)} ${formatValueForSql(value)}`
// }
// return ''
// }
}
|