Skip to content

Commit

Permalink
Bucket filtering in the functions emulator (firebase#3893)
Browse files Browse the repository at this point in the history 
* added filtering

* change to use headers

* using data attribute instead of header & adding integration tests

* adding default-bucket into tests

* local testing keeps failing, trying again on github workflows....

* removed 'default-bucket', added reset counter to framework, and finally fixed the tests

* add changelog entry
  • Loading branch information
@colerogers @devpeerapong
colerogers authored and devpeerapong committed Dec 14, 2021
1 parent b9e7d1a commit 2658c4a
Show file tree
Hide file tree
Showing 6 changed files with 317 additions and 40 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
@@ -0,0 +1 @@
- Fixes issue with filtering on a specific storage bucket using functions in the emulator (#3893)
90 changes: 86 additions & 4 deletions scripts/integration-helpers/framework.ts
View file
Expand Up @@ -14,6 +14,14 @@ const STORAGE_FUNCTION_V2_ARCHIVED_LOG = "========== STORAGE V2 FUNCTION ARCHIVE
const STORAGE_FUNCTION_V2_DELETED_LOG = "========== STORAGE V2 FUNCTION DELETED ==========";
const STORAGE_FUNCTION_V2_FINALIZED_LOG = "========== STORAGE V2 FUNCTION FINALIZED ==========";
const STORAGE_FUNCTION_V2_METADATA_LOG = "========== STORAGE V2 FUNCTION METADATA ==========";
const STORAGE_BUCKET_FUNCTION_V2_ARCHIVED_LOG =
"========== STORAGE BUCKET V2 FUNCTION ARCHIVED ==========";
const STORAGE_BUCKET_FUNCTION_V2_DELETED_LOG =
"========== STORAGE BUCKET V2 FUNCTION DELETED ==========";
const STORAGE_BUCKET_FUNCTION_V2_FINALIZED_LOG =
"========== STORAGE BUCKET V2 FUNCTION FINALIZED ==========";
const STORAGE_BUCKET_FUNCTION_V2_METADATA_LOG =
"========== STORAGE BUCKET V2 FUNCTION METADATA ==========";
/* Functions V1 */
const RTDB_FUNCTION_LOG = "========== RTDB FUNCTION ==========";
const FIRESTORE_FUNCTION_LOG = "========== FIRESTORE FUNCTION ==========";
Expand All @@ -23,6 +31,13 @@ const STORAGE_FUNCTION_ARCHIVED_LOG = "========== STORAGE FUNCTION ARCHIVED ====
const STORAGE_FUNCTION_DELETED_LOG = "========== STORAGE FUNCTION DELETED ==========";
const STORAGE_FUNCTION_FINALIZED_LOG = "========== STORAGE FUNCTION FINALIZED ==========";
const STORAGE_FUNCTION_METADATA_LOG = "========== STORAGE FUNCTION METADATA ==========";
const STORAGE_BUCKET_FUNCTION_ARCHIVED_LOG =
"========== STORAGE BUCKET FUNCTION ARCHIVED ==========";
const STORAGE_BUCKET_FUNCTION_DELETED_LOG = "========== STORAGE BUCKET FUNCTION DELETED ==========";
const STORAGE_BUCKET_FUNCTION_FINALIZED_LOG =
"========== STORAGE BUCKET FUNCTION FINALIZED ==========";
const STORAGE_BUCKET_FUNCTION_METADATA_LOG =
"========== STORAGE BUCKET FUNCTION METADATA ==========";
const ALL_EMULATORS_STARTED_LOG = "All emulators ready";

interface ConnectionInfo {
Expand Down Expand Up @@ -65,13 +80,21 @@ export class TriggerEndToEndTest {
storageDeletedTriggerCount = 0;
storageFinalizedTriggerCount = 0;
storageMetadataTriggerCount = 0;
storageBucketArchivedTriggerCount = 0;
storageBucketDeletedTriggerCount = 0;
storageBucketFinalizedTriggerCount = 0;
storageBucketMetadataTriggerCount = 0;

/* Functions V2 */
pubsubV2TriggerCount = 0;
storageV2ArchivedTriggerCount = 0;
storageV2DeletedTriggerCount = 0;
storageV2FinalizedTriggerCount = 0;
storageV2MetadataTriggerCount = 0;
storageBucketV2ArchivedTriggerCount = 0;
storageBucketV2DeletedTriggerCount = 0;
storageBucketV2FinalizedTriggerCount = 0;
storageBucketV2MetadataTriggerCount = 0;

rtdbFromFirestore = false;
firestoreFromRtdb = false;
Expand All @@ -90,6 +113,33 @@ export class TriggerEndToEndTest {
}
}

resetCounts(): void {
/* Functions V1 */
this.firestoreTriggerCount = 0;
this.rtdbTriggerCount = 0;
this.pubsubTriggerCount = 0;
this.authTriggerCount = 0;
this.storageArchivedTriggerCount = 0;
this.storageDeletedTriggerCount = 0;
this.storageFinalizedTriggerCount = 0;
this.storageMetadataTriggerCount = 0;
this.storageBucketArchivedTriggerCount = 0;
this.storageBucketDeletedTriggerCount = 0;
this.storageBucketFinalizedTriggerCount = 0;
this.storageBucketMetadataTriggerCount = 0;

/* Functions V2 */
this.pubsubV2TriggerCount = 0;
this.storageV2ArchivedTriggerCount = 0;
this.storageV2DeletedTriggerCount = 0;
this.storageV2FinalizedTriggerCount = 0;
this.storageV2MetadataTriggerCount = 0;
this.storageBucketV2ArchivedTriggerCount = 0;
this.storageBucketV2DeletedTriggerCount = 0;
this.storageBucketV2FinalizedTriggerCount = 0;
this.storageBucketV2MetadataTriggerCount = 0;
}

/*
* Check that all directions of database <-> functions <-> firestore
* worked.
Expand Down Expand Up @@ -138,6 +188,18 @@ export class TriggerEndToEndTest {
if (data.includes(STORAGE_FUNCTION_METADATA_LOG)) {
this.storageMetadataTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_ARCHIVED_LOG)) {
this.storageBucketArchivedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_DELETED_LOG)) {
this.storageBucketDeletedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_FINALIZED_LOG)) {
this.storageBucketFinalizedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_METADATA_LOG)) {
this.storageBucketMetadataTriggerCount++;
}
/* Functions V2 */
if (data.includes(PUBSUB_FUNCTION_V2_LOG)) {
this.pubsubV2TriggerCount++;
Expand All @@ -154,6 +216,18 @@ export class TriggerEndToEndTest {
if (data.includes(STORAGE_FUNCTION_V2_METADATA_LOG)) {
this.storageV2MetadataTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_V2_ARCHIVED_LOG)) {
this.storageBucketV2ArchivedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_V2_DELETED_LOG)) {
this.storageBucketV2DeletedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_V2_FINALIZED_LOG)) {
this.storageBucketV2FinalizedTriggerCount++;
}
if (data.includes(STORAGE_BUCKET_FUNCTION_V2_METADATA_LOG)) {
this.storageBucketV2MetadataTriggerCount++;
}
});

this.cliProcess = cli;
Expand Down Expand Up @@ -209,12 +283,20 @@ export class TriggerEndToEndTest {
return this.invokeHttpFunction("writeToScheduledPubsub");
}

writeToStorage(): Promise<Response> {
return this.invokeHttpFunction("writeToStorage");
writeToDefaultStorage(): Promise<Response> {
return this.invokeHttpFunction("writeToDefaultStorage");
}

writeToSpecificStorageBucket(): Promise<Response> {
return this.invokeHttpFunction("writeToSpecificStorageBucket");
}

updateDeleteFromDefaultStorage(): Promise<Response> {
return this.invokeHttpFunction("updateDeleteFromDefaultStorage");
}

updateDeleteFromStorage(): Promise<Response> {
return this.invokeHttpFunction("updateDeleteFromStorage");
updateDeleteFromSpecificStorageBucket(): Promise<Response> {
return this.invokeHttpFunction("updateDeleteFromSpecificStorageBucket");
}

waitForCondition(
Expand Down
159 changes: 136 additions & 23 deletions scripts/triggers-end-to-end-tests/functions/index.js
View file
Expand Up @@ -23,12 +23,27 @@ const STORAGE_FUNCTION_ARCHIVED_LOG = "========== STORAGE FUNCTION ARCHIVED ====
const STORAGE_FUNCTION_DELETED_LOG = "========== STORAGE FUNCTION DELETED ==========";
const STORAGE_FUNCTION_FINALIZED_LOG = "========== STORAGE FUNCTION FINALIZED ==========";
const STORAGE_FUNCTION_METADATA_LOG = "========== STORAGE FUNCTION METADATA ==========";
const STORAGE_BUCKET_FUNCTION_ARCHIVED_LOG =
"========== STORAGE BUCKET FUNCTION ARCHIVED ==========";
const STORAGE_BUCKET_FUNCTION_DELETED_LOG = "========== STORAGE BUCKET FUNCTION DELETED ==========";
const STORAGE_BUCKET_FUNCTION_FINALIZED_LOG =
"========== STORAGE BUCKET FUNCTION FINALIZED ==========";
const STORAGE_BUCKET_FUNCTION_METADATA_LOG =
"========== STORAGE BUCKET FUNCTION METADATA ==========";
/* Functions V2 */
const PUBSUB_FUNCTION_V2_LOG = "========== PUBSUB V2 FUNCTION ==========";
const STORAGE_FUNCTION_V2_ARCHIVED_LOG = "========== STORAGE V2 FUNCTION ARCHIVED ==========";
const STORAGE_FUNCTION_V2_DELETED_LOG = "========== STORAGE V2 FUNCTION DELETED ==========";
const STORAGE_FUNCTION_V2_FINALIZED_LOG = "========== STORAGE V2 FUNCTION FINALIZED ==========";
const STORAGE_FUNCTION_V2_METADATA_LOG = "========== STORAGE V2 FUNCTION METADATA ==========";
const STORAGE_BUCKET_FUNCTION_V2_ARCHIVED_LOG =
"========== STORAGE BUCKET V2 FUNCTION ARCHIVED ==========";
const STORAGE_BUCKET_FUNCTION_V2_DELETED_LOG =
"========== STORAGE BUCKET V2 FUNCTION DELETED ==========";
const STORAGE_BUCKET_FUNCTION_V2_FINALIZED_LOG =
"========== STORAGE BUCKET V2 FUNCTION FINALIZED ==========";
const STORAGE_BUCKET_FUNCTION_V2_METADATA_LOG =
"========== STORAGE BUCKET V2 FUNCTION METADATA ==========";

/*
* We install onWrite triggers for START_DOCUMENT_NAME in both the firestore and
Expand Down Expand Up @@ -99,20 +114,34 @@ exports.writeToAuth = functions.https.onRequest(async (req, res) => {
res.json({ created: "ok" });
});

exports.writeToStorage = functions.https.onRequest(async (req, res) => {
exports.writeToDefaultStorage = functions.https.onRequest(async (req, res) => {
await admin.storage().bucket().file(STORAGE_FILE_NAME).save("hello world!");
console.log("Wrote to Storage bucket");
console.log("Wrote to default Storage bucket");
res.json({ created: "ok" });
});

exports.updateDeleteFromStorage = functions.https.onRequest(async (req, res) => {
exports.writeToSpecificStorageBucket = functions.https.onRequest(async (req, res) => {
await admin.storage().bucket("test-bucket").file(STORAGE_FILE_NAME).save("hello world!");
console.log("Wrote to a specific Storage bucket");
res.json({ created: "ok" });
});

exports.updateDeleteFromDefaultStorage = functions.https.onRequest(async (req, res) => {
await admin.storage().bucket().file(STORAGE_FILE_NAME).save("something new!");
console.log("Wrote to Storage bucket");
await admin.storage().bucket().file(STORAGE_FILE_NAME).delete();
console.log("Deleted from Storage bucket");
res.json({ done: "ok" });
});

exports.updateDeleteFromSpecificStorageBucket = functions.https.onRequest(async (req, res) => {
await admin.storage().bucket("test-bucket").file(STORAGE_FILE_NAME).save("something new!");
console.log("Wrote to a specific Storage bucket");
await admin.storage().bucket("test-bucket").file(STORAGE_FILE_NAME).delete();
console.log("Deleted from a specific Storage bucket");
res.json({ done: "ok" });
});

exports.firestoreReaction = functions.firestore
.document(START_DOCUMENT_NAME)
.onWrite(async (/* change, ctx */) => {
Expand Down Expand Up @@ -175,29 +204,41 @@ exports.authReaction = functions.auth.user().onCreate((user, ctx) => {
return true;
});

exports.storageArchiveReaction = functions.storage.object().onArchive((object, context) => {
console.log(STORAGE_FUNCTION_ARCHIVED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});
exports.storageArchiveReaction = functions.storage
.bucket()
.object()
.onArchive((object, context) => {
console.log(STORAGE_FUNCTION_ARCHIVED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageDeleteReaction = functions.storage.object().onDelete((object, context) => {
console.log(STORAGE_FUNCTION_DELETED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});
exports.storageDeleteReaction = functions.storage
.bucket()
.object()
.onDelete((object, context) => {
console.log(STORAGE_FUNCTION_DELETED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageFinalizeReaction = functions.storage.object().onFinalize((object, context) => {
console.log(STORAGE_FUNCTION_FINALIZED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});
exports.storageFinalizeReaction = functions.storage
.bucket()
.object()
.onFinalize((object, context) => {
console.log(STORAGE_FUNCTION_FINALIZED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageMetadataReaction = functions.storage.object().onMetadataUpdate((object, context) => {
console.log(STORAGE_FUNCTION_METADATA_LOG);
console.log("Object", JSON.stringify(object));
return true;
});
exports.storageMetadataReaction = functions.storage
.bucket()
.object()
.onMetadataUpdate((object, context) => {
console.log(STORAGE_FUNCTION_METADATA_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storagev2archivedreaction = functionsV2.storage.onObjectArchived((cloudevent) => {
console.log(STORAGE_FUNCTION_V2_ARCHIVED_LOG);
Expand All @@ -222,3 +263,75 @@ exports.storagev2metadatareaction = functionsV2.storage.onObjectMetadataUpdated(
console.log("Object", JSON.stringify(cloudevent.data));
return true;
});

exports.storageBucketArchiveReaction = functions.storage
.bucket("test-bucket")
.object()
.onArchive((object, context) => {
console.log(STORAGE_BUCKET_FUNCTION_ARCHIVED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageBucketDeleteReaction = functions.storage
.bucket("test-bucket")
.object()
.onDelete((object, context) => {
console.log(STORAGE_BUCKET_FUNCTION_DELETED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageBucketFinalizeReaction = functions.storage
.bucket("test-bucket")
.object()
.onFinalize((object, context) => {
console.log(STORAGE_BUCKET_FUNCTION_FINALIZED_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storageBucketMetadataReaction = functions.storage
.bucket("test-bucket")
.object()
.onMetadataUpdate((object, context) => {
console.log(STORAGE_BUCKET_FUNCTION_METADATA_LOG);
console.log("Object", JSON.stringify(object));
return true;
});

exports.storagebucketv2archivedreaction = functionsV2.storage.onObjectArchived(
"test-bucket",
(cloudevent) => {
console.log(STORAGE_BUCKET_FUNCTION_V2_ARCHIVED_LOG);
console.log("Object", JSON.stringify(cloudevent.data));
return true;
}
);

exports.storagebucketv2deletedreaction = functionsV2.storage.onObjectDeleted(
"test-bucket",
(cloudevent) => {
console.log(STORAGE_BUCKET_FUNCTION_V2_DELETED_LOG);
console.log("Object", JSON.stringify(cloudevent.data));
return true;
}
);

exports.storagebucketv2finalizedreaction = functionsV2.storage.onObjectFinalized(
"test-bucket",
(cloudevent) => {
console.log(STORAGE_BUCKET_FUNCTION_V2_FINALIZED_LOG);
console.log("Object", JSON.stringify(cloudevent.data));
return true;
}
);

exports.storagebucketv2metadatareaction = functionsV2.storage.onObjectMetadataUpdated(
"test-bucket",
(cloudevent) => {
console.log(STORAGE_BUCKET_FUNCTION_V2_METADATA_LOG);
console.log("Object", JSON.stringify(cloudevent.data));
return true;
}
);

0 comments on commit 2658c4a

Please sign in to comment.