GCS fixes (#3854)

* adding service account api calls

* fixing edge case for function regions/buckets/triggers, added code to set the iam role on storage service agent

* spacing

* adding reducer to clean up code

* fix additional edge cases

* addressing comments

* cleaning up names

* exposing more location apis and clean up

* linter

* refactor into services file

* fixing comments

* clean up sinon stubs

* changing dispatch function to undefined for noop & pubsub services

* linter

src/deploy/functions/checkIam.ts

@@ -3,12 +3,16 @@ import { bold } from "cli-color";
 import { logger } from "../../logger";
 import { getFilterGroups, functionMatchesAnyGroup } from "./functionsDeployHelper";
 import { FirebaseError } from "../../error";
-import { testIamPermissions, testResourceIamPermissions } from "../../gcp/iam";
+import * as iam from "../../gcp/iam";
 import * as args from "./args";
 import * as backend from "./backend";
 import * as track from "../../track";
+import * as utils from "../../utils";
 import { Options } from "../../options";
 
+import { getIamPolicy, setIamPolicy } from "../../gcp/resourceManager";
+import { Service, serviceForEndpoint } from "./services";
+
 const PERMISSION = "cloudfunctions.functions.setIamPolicy";
 
 /**
@@ -20,7 +24,7 @@ export async function checkServiceAccountIam(projectId: string): Promise<void> {
   const saEmail = `${projectId}@appspot.gserviceaccount.com`;
   let passed = false;
   try {
-    const iamResult = await testResourceIamPermissions(
+    const iamResult = await iam.testResourceIamPermissions(
       "https://iam.googleapis.com",
       "v1",
       `projects/${projectId}/serviceAccounts/${saEmail}`,
@@ -79,7 +83,7 @@ export async function checkHttpIam(
 
   let passed = true;
   try {
-    const iamResult = await testIamPermissions(context.projectId, [PERMISSION]);
+    const iamResult = await iam.testIamPermissions(context.projectId, [PERMISSION]);
     passed = iamResult.passed;
   } catch (e) {
     logger.debug(
@@ -104,3 +108,89 @@ export async function checkHttpIam(
   }
   logger.debug("[functions] found setIamPolicy permission, proceeding with deploy");
 }
+
+/** Callback reducer function */
+function reduceEventsToServices(services: Array<Service>, endpoint: backend.Endpoint) {
+  const service = serviceForEndpoint(endpoint);
+  if (service.requiredProjectBindings && !services.find((s) => s.name === service.name)) {
+    services.push(service);
+  }
+  return services;
+}
+
+/** Helper to merge all required bindings into the IAM policy */
+export function mergeBindings(policy: iam.Policy, allRequiredBindings: iam.Binding[][]) {
+  for (const requiredBindings of allRequiredBindings) {
+    if (requiredBindings.length === 0) {
+      continue;
+    }
+    for (const requiredBinding of requiredBindings) {
+      const ndx = policy.bindings.findIndex(
+        (policyBinding) => policyBinding.role === requiredBinding.role
+      );
+      if (ndx === -1) {
+        policy.bindings.push(requiredBinding);
+        continue;
+      }
+      requiredBinding.members.forEach((updatedMember) => {
+        if (!policy.bindings[ndx].members.find((member) => member === updatedMember)) {
+          policy.bindings[ndx].members.push(updatedMember);
+        }
+      });
+    }
+  }
+}
+
+/**
+ * Checks and sets the roles for specific resource service agents
+ * @param projectId project identifier
+ * @param want backend that we want to deploy
+ * @param have backend that we have currently deployed
+ */
+export async function ensureServiceAgentRoles(
+  projectId: string,
+  want: backend.Backend,
+  have: backend.Backend
+): Promise<void> {
+  // find new services
+  const wantServices = backend.allEndpoints(want).reduce(reduceEventsToServices, []);
+  const haveServices = backend.allEndpoints(have).reduce(reduceEventsToServices, []);
+  const newServices = wantServices.filter(
+    (wantS) => !haveServices.find((haveS) => wantS.name === haveS.name)
+  );
+  if (newServices.length === 0) {
+    return;
+  }
+  // get the full project iam policy
+  let policy: iam.Policy;
+  try {
+    policy = await getIamPolicy(projectId);
+  } catch (err) {
+    utils.logLabeledBullet(
+      "functions",
+      "Could not verify the necessary IAM configuration for the following newly-integrated services: " +
+        `${newServices.map((service) => service.api).join(", ")}` +
+        ". Deployment may fail.",
+      "warn"
+    );
+    return;
+  }
+  // run in parallel all the missingProjectBindings jobs
+  const findRequiredBindings: Array<Promise<Array<iam.Binding>>> = [];
+  newServices.forEach((service) =>
+    findRequiredBindings.push(service.requiredProjectBindings!(projectId, policy))
+  );
+  const allRequiredBindings = await Promise.all(findRequiredBindings);
+  mergeBindings(policy, allRequiredBindings);
+  // set the updated policy
+  try {
+    await setIamPolicy(projectId, policy, "bindings");
+  } catch (err) {
+    throw new FirebaseError(
+      "We failed to modify the IAM policy for the project. The functions " +
+        "deployment requires specific roles to be granted to service agents," +
+        " otherwise the deployment will fail.",
+      { original: err }
+    );
+  }
+}

src/deploy/functions/containerCleaner.ts

@@ -15,34 +15,6 @@ import * as docker from "../../gcp/docker";
 import * as utils from "../../utils";
 import * as poller from "../../operation-poller";
 
-// A flattening of container_registry_hosts and
-// region_multiregion_map from regionconfig.borg
-export const SUBDOMAIN_MAPPING: Record<string, string> = {
-  "us-west2": "us",
-  "us-west3": "us",
-  "us-west4": "us",
-  "us-central1": "us",
-  "us-central2": "us",
-  "us-east1": "us",
-  "us-east4": "us",
-  "northamerica-northeast1": "us",
-  "southamerica-east1": "us",
-  "europe-west1": "eu",
-  "europe-west2": "eu",
-  "europe-west3": "eu",
-  "europe-west5": "eu",
-  "europe-west6": "eu",
-  "europe-central2": "eu",
-  "asia-east1": "asia",
-  "asia-east2": "asia",
-  "asia-northeast1": "asia",
-  "asia-northeast2": "asia",
-  "asia-northeast3": "asia",
-  "asia-south1": "asia",
-  "asia-southeast2": "asia",
-  "australia-southeast1": "asia",
-};
-
 async function retry<Return>(func: () => Promise<Return>): Promise<Return> {
   const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
   const MAX_RETRIES = 3;
@@ -105,7 +77,7 @@ export async function cleanupBuildImages(
       try {
         await gcrCleaner.cleanupFunction(func);
       } catch (err) {
-        const path = `${func.project}/${SUBDOMAIN_MAPPING[func.region]}/gcf`;
+        const path = `${func.project}/${docker.GCR_SUBDOMAIN_MAPPING[func.region]}/gcf`;
         failedDomains.add(`https://console.cloud.google.com/gcr/images/${path}`);
       }
     })
@@ -191,7 +163,7 @@ export class ContainerRegistryCleaner {
   readonly helpers: Record<string, DockerHelper> = {};
 
   private helper(location: string): DockerHelper {
-    const subdomain = SUBDOMAIN_MAPPING[location] || "us";
+    const subdomain = docker.GCR_SUBDOMAIN_MAPPING[location] || "us";
     if (!this.helpers[subdomain]) {
       const origin = `https://${subdomain}.${containerRegistryDomain}`;
       this.helpers[subdomain] = new DockerHelper(origin);
@@ -266,14 +238,14 @@ export async function listGcfPaths(
   dockerHelpers: Record<string, DockerHelper> = {}
 ): Promise<string[]> {
   if (!locations) {
-    locations = Object.keys(SUBDOMAIN_MAPPING);
+    locations = Object.keys(docker.GCR_SUBDOMAIN_MAPPING);
   }
-  const invalidRegion = locations.find((loc) => !SUBDOMAIN_MAPPING[loc]);
+  const invalidRegion = locations.find((loc) => !docker.GCR_SUBDOMAIN_MAPPING[loc]);
   if (invalidRegion) {
     throw new FirebaseError(`Invalid region ${invalidRegion} supplied`);
   }
   const locationsSet = new Set(locations); // for quick lookup
-  const subdomains = new Set(Object.values(SUBDOMAIN_MAPPING));
+  const subdomains = new Set(Object.values(docker.GCR_SUBDOMAIN_MAPPING));
   const failedSubdomains: string[] = [];
   const listAll: Promise<Stat>[] = [];
 
@@ -310,7 +282,7 @@ export async function listGcfPaths(
   }
 
   return gcfDirs.map((loc) => {
-    return `${SUBDOMAIN_MAPPING[loc]}.${containerRegistryDomain}/${projectId}/gcf/${loc}`;
+    return `${docker.GCR_SUBDOMAIN_MAPPING[loc]}.${containerRegistryDomain}/${projectId}/gcf/${loc}`;
   });
 }
 
@@ -329,20 +301,21 @@ export async function deleteGcfArtifacts(
   dockerHelpers: Record<string, DockerHelper> = {}
 ): Promise<void> {
   if (!locations) {
-    locations = Object.keys(SUBDOMAIN_MAPPING);
+    locations = Object.keys(docker.GCR_SUBDOMAIN_MAPPING);
   }
-  const invalidRegion = locations.find((loc) => !SUBDOMAIN_MAPPING[loc]);
+  const invalidRegion = locations.find((loc) => !docker.GCR_SUBDOMAIN_MAPPING[loc]);
   if (invalidRegion) {
     throw new FirebaseError(`Invalid region ${invalidRegion} supplied`);
   }
-  const subdomains = new Set(Object.values(SUBDOMAIN_MAPPING));
+  const subdomains = new Set(Object.values(docker.GCR_SUBDOMAIN_MAPPING));
   const failedSubdomains: string[] = [];
 
   const deleteLocations = locations.map((loc) => {
+    const subdomain = docker.GCR_SUBDOMAIN_MAPPING[loc]!;
     try {
-      return getHelper(dockerHelpers, SUBDOMAIN_MAPPING[loc]).rm(`${projectId}/gcf/${loc}`);
+      return getHelper(dockerHelpers, subdomain).rm(`${projectId}/gcf/${loc}`);
     } catch (err) {
-      failedSubdomains.push(SUBDOMAIN_MAPPING[loc]);
+      failedSubdomains.push(subdomain);
       logger.debug(err);
     }
   });

src/deploy/functions/eventTypes.ts

@@ -0,0 +1,8 @@
+export const STORAGE_V2_EVENTS = [
+  "google.cloud.storage.object.v1.finalized",
+  "google.cloud.storage.object.v1.archived",
+  "google.cloud.storage.object.v1.deleted",
+  "google.cloud.storage.object.v1.metadataUpdated",
+];
+
+export const PUBSUB_V2_EVENT = "google.cloud.pubsub.topic.v1.messagePublished";

src/deploy/functions/prepare.ts

@@ -18,7 +18,8 @@ import * as runtimes from "./runtimes";
 import * as validate from "./validate";
 import * as utils from "../../utils";
 import { logger } from "../../logger";
-import { lookupMissingTriggerRegions } from "./triggerRegionHelper";
+import { ensureTriggerRegions } from "./triggerRegionHelper";
+import { ensureServiceAgentRoles } from "./checkIam";
 
 function hasUserConfig(config: Record<string, unknown>): boolean {
   // "firebase" key is always going to exist in runtime config.
@@ -153,8 +154,9 @@ export async function prepare(
   });
 
   const haveBackend = await backend.existingBackend(context);
+  await ensureServiceAgentRoles(projectId, wantBackend, haveBackend);
   inferDetailsFromExisting(wantBackend, haveBackend, usedDotenv);
-  await lookupMissingTriggerRegions(wantBackend);
+  await ensureTriggerRegions(wantBackend);
 
   // Display a warning and prompt if any functions in the release have failurePolicies.
   await promptForFailurePolicies(options, matchingBackend, haveBackend);

src/deploy/functions/runtimes/node/parseTriggers.ts

@@ -9,16 +9,10 @@ import * as api from "../../../../api";
 import * as proto from "../../../../gcp/proto";
 import * as args from "../../args";
 import * as runtimes from "../../runtimes";
+import { STORAGE_V2_EVENTS } from "../../eventTypes";
 
 const TRIGGER_PARSER = path.resolve(__dirname, "./triggerParser.js");
 
-export const GCS_EVENTS: Set<string> = new Set<string>([
-  "google.cloud.storage.object.v1.finalized",
-  "google.cloud.storage.object.v1.archived",
-  "google.cloud.storage.object.v1.deleted",
-  "google.cloud.storage.object.v1.metadataUpdated",
-]);
-
 export interface ScheduleRetryConfig {
   retryCount?: number;
   maxRetryDuration?: string;
@@ -203,7 +197,7 @@ export function addResourcesToBackend(
 
       // TODO: yank this edge case for a v2 trigger on the pre-container contract
       // once we use container contract for the functionsv2 experiment.
-      if (GCS_EVENTS.has(annotation.eventTrigger?.eventType || "")) {
+      if (STORAGE_V2_EVENTS.find((event) => event === (annotation.eventTrigger?.eventType || ""))) {
         triggered.eventTrigger.eventFilters = {
           bucket: annotation.eventTrigger!.resource,
         };

src/deploy/functions/services/index.ts

@@ -0,0 +1,59 @@
+import * as backend from "../backend";
+import * as iam from "../../../gcp/iam";
+import { obtainStorageBindings, ensureStorageTriggerRegion } from "./storage";
+
+const noop = (): Promise<void> => Promise.resolve();
+
+/** A service interface for the underlying GCP event services */
+export interface Service {
+  readonly name: string;
+  readonly api: string;
+
+  // dispatch functions
+  requiredProjectBindings: ((pId: any, p: any) => Promise<Array<iam.Binding>>) | undefined;
+  ensureTriggerRegion: (ep: backend.Endpoint, et: backend.EventTrigger) => Promise<void>;
+}
+
+/** A noop service object, useful for v1 events */
+export const NoOpService: Service = {
+  name: "noop",
+  api: "",
+  requiredProjectBindings: undefined,
+  ensureTriggerRegion: noop,
+};
+/** A pubsub service object */
+export const PubSubService: Service = {
+  name: "pubsub",
+  api: "pubsub.googleapis.com",
+  requiredProjectBindings: undefined,
+  ensureTriggerRegion: noop,
+};
+/** A storage service object */
+export const StorageService = {
+  name: "storage",
+  api: "storage.googleapis.com",
+  requiredProjectBindings: obtainStorageBindings,
+  ensureTriggerRegion: ensureStorageTriggerRegion,
+};
+
+/** Mapping from event type string to service object */
+export const EVENT_SERVICE_MAPPING: Record<string, any> = {
+  "google.cloud.pubsub.topic.v1.messagePublished": PubSubService,
+  "google.cloud.storage.object.v1.finalized": StorageService,
+  "google.cloud.storage.object.v1.archived": StorageService,
+  "google.cloud.storage.object.v1.deleted": StorageService,
+  "google.cloud.storage.object.v1.metadataUpdated": StorageService,
+};
+
+/**
+ * Find the Service object for the given endpoint
+ * @param endpoint the endpoint that we want the service for
+ * @returns a Service object that corresponds to the event type of the endpoint or noop
+ */
+export function serviceForEndpoint(endpoint: backend.Endpoint): Service {
+  if (!backend.isEventTriggered(endpoint)) {
+    return NoOpService;
+  }
+
+  return EVENT_SERVICE_MAPPING[endpoint.eventTrigger.eventType] || NoOpService;
+}