expose dataset location option for BQ spark

rowyio · Jul 5, 2021 · 154be1e · 154be1e
1 parent cf1857e
commit 154be1e
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/ft_build/sparksLib/bigqueryIndex.ts b/ft_build/sparksLib/bigqueryIndex.ts
@@ -163,7 +163,7 @@ const transformToSQLType = (ftType: string) => {
 };
 
 const bigqueryIndex = async (payload, sparkContext) => {
-  const { objectID, index, fieldsToSync, projectID } = payload;
+  const { objectID, index, fieldsToSync, projectID, datasetLocation } = payload;
 
   const { triggerType, change, fieldTypes } = sparkContext;
   const record = rowReducer(fieldsToSync, sparkContext.row);
@@ -178,7 +178,9 @@ const bigqueryIndex = async (payload, sparkContext) => {
 
   // create dataset with exact name "firetable" if not exists
   async function preprocessDataset() {
-    const dataset = bigquery.dataset("firetable");
+    const dataset = bigquery.dataset("firetable", {
+      location: datasetLocation ?? "US",
+    });
     const res = await dataset.exists();
     const exists = res[0];
     if (!exists) {