Use Terraform to manage API deployments in oss-vdb-test (#947)

deployment/README.md

@@ -6,6 +6,28 @@ export GOOGLE_CLOUD_PROJECT=<PROJECT_ID>
 ```
 The official project id of OSV is `oss-vdb`.
 
+## API Domain Name
+
+Due to [terraform complexities](https://github.com/hashicorp/terraform-provider-google/issues/5528),
+setting up the OSV API requires a custom domain to serve it on.
+
+For example, if you own `custom-domain.name` and wish to serve the api on `api.custom-domain.name`:
+
+1. Verify the ownership of your domain:
+
+    Go to 
+
+    `https://www.google.com/webmasters/verification/verification?authuser=0&domain=custom-domain.name`
+
+    (Replace `custom-domain.name` in the url with the actual domain to be verified.)
+
+    (This link is usually generated when adding a domain mapping to a service in Cloud Run.
+    I don't know how to navigate to that page otherwise. Trying to add a property from
+    [Webmaster Central](https://www.google.com/webmasters/verification/home)
+    adds it as a site, rather than as a domain.)
+
+2. Add DNS CNAME record mapping `api.custom-domain.name` to `ghs.googlehosted.com.`
+
 ## Terraform
 
 Go to the relevant directory `/deployment/terraform/environments/<PROJECT_ID>`:

deployment/oss-vdb-test/api/api_config.yaml → deployment/terraform/environments/oss-vdb-test/api/api_config.tftpl

@@ -14,7 +14,7 @@
 
 type: google.api.Service
 config_version: 3
-name: osv-grpc-v1-ivqje5d2hq-uc.a.run.app
+name: ${service_name}
 title: OSV (v1)
 apis:
   - name: osv.v1.OSV
@@ -29,4 +29,4 @@ authentication:
 backend:
   rules:
     - selector: "*"
-      address: grpcs://osv-grpc-backend-ivqje5d2hq-uc.a.run.app
+      address: ${backend_url}

deployment/terraform/environments/oss-vdb-test/api/api_descriptor.pb

@@ -0,0 +1 @@
+../../../../../gcp/api/v1/api_descriptor.pb

deployment/terraform/environments/oss-vdb-test/main.tf

@@ -5,6 +5,9 @@ module "osv_test" {
 
   public_import_logs_bucket     = "osv-test-public-import-logs"
   vulnerabilities_export_bucket = "osv-test-vulnerabilities"
+
+  api_url               = "api.test.osv.dev"
+  api_backend_image_tag = "20230105"
 }
 
 

deployment/terraform/modules/osv/apis.tf → deployment/terraform/modules/osv/gcp_apis.tf

@@ -21,8 +21,9 @@ resource "google_project_service" "redis_api" {
 }
 
 resource "google_project_service" "datastore_api" {
-  project = var.project_id
-  service = "datastore.googleapis.com"
+  project            = var.project_id
+  service            = "datastore.googleapis.com"
+  disable_on_destroy = false
 }
 
 resource "google_project_service" "vpcaccess_api" {

deployment/terraform/modules/osv/osv_api.tf

@@ -0,0 +1,116 @@
+# The OSV API on Cloud Run
+# Adapted from https://github.com/hashicorp/terraform-provider-google/issues/5528#issuecomment-1136040976
+
+data "google_container_registry_image" "api_backend" {
+  project = var.project_id
+  name    = "osv-server"
+  tag     = var.api_backend_image_tag
+}
+
+resource "google_cloud_run_service" "api_backend" {
+  project  = var.project_id
+  name     = "osv-grpc-backend"
+  location = "us-central1"
+
+  template {
+    spec {
+      containers {
+        image = data.google_container_registry_image.api_backend.image_url
+      }
+    }
+  }
+
+  traffic {
+    percent         = 100
+    latest_revision = true
+  }
+}
+
+resource "google_endpoints_service" "grpc_service" {
+  project      = var.project_id
+  service_name = var.api_url
+  grpc_config = templatefile(
+    "api/api_config.tftpl",
+    {
+      service_name = var.api_url,
+      backend_url  = replace(google_cloud_run_service.api_backend.status[0].url, "https://", "grpcs://")
+  })
+  protoc_output_base64 = filebase64("api/api_descriptor.pb")
+}
+
+resource "google_project_service" "grpc_service_api" {
+  project = var.project_id
+  service = google_endpoints_service.grpc_service.service_name
+}
+
+
+data "external" "esp_version" {
+  program = ["bash", "${path.module}/scripts/esp_full_version"]
+}
+
+resource "null_resource" "grpc_proxy_image" {
+  triggers = {
+    # Update this when the config changes or there is a new ESP image
+    config_id   = google_endpoints_service.grpc_service.config_id
+    esp_version = data.external.esp_version.result.esp_full_version
+  }
+
+  # Script obtained from:
+  # https://github.com/GoogleCloudPlatform/esp-v2/blob/master/docker/serverless/gcloud_build_image
+  provisioner "local-exec" {
+    command = <<EOS
+      bash ${path.module}/scripts/gcloud_build_image \
+        -s ${var.api_url} \
+        -c ${google_endpoints_service.grpc_service.config_id} \
+        -p ${var.project_id}
+    EOS
+  }
+}
+
+data "google_container_registry_image" "api" {
+  project = var.project_id
+  name    = "endpoints-runtime-serverless"
+  tag = format(
+    "%s-%s-%s",
+    data.external.esp_version.result.esp_full_version,
+    var.api_url,
+    google_endpoints_service.grpc_service.config_id
+  )
+  depends_on = [null_resource.grpc_proxy_image]
+}
+
+
+resource "google_cloud_run_service" "api" {
+  project  = var.project_id
+  name     = "osv-grpc-v1"
+  location = "us-central1"
+
+  template {
+    spec {
+      containers {
+        image = data.google_container_registry_image.api.image_url
+        env {
+          name  = "ESPv2_ARGS"
+          value = "^++^--transcoding_preserve_proto_field_names++--envoy_connection_buffer_limit_bytes=10485760"
+        }
+      }
+    }
+  }
+
+  traffic {
+    percent         = 100
+    latest_revision = true
+  }
+}
+
+resource "google_cloud_run_domain_mapping" "api" {
+  project  = var.project_id
+  name     = var.api_url
+  location = google_cloud_run_service.api.location
+  metadata {
+    namespace = var.project_id
+  }
+  spec {
+    route_name = google_cloud_run_service.api.name
+  }
+}

deployment/terraform/modules/osv/scripts/esp_full_version

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Get the full tag of latest ESP image for use in terraform.
+# This is required as the gcloud_build_image script tags its built image with
+# the version number.
+# Optional in terraform external data source is "query = {esp_tag = <ESP_TAG>}"
+# which is equivalent to the -v flag of gcloud_build_image.
+
+set -e
+
+BASE_IMAGE_NAME="gcr.io/endpoints-release/endpoints-runtime-serverless"
+
+function error_exit() {
+  # ${BASH_SOURCE[1]} is the file name of the caller.
+  echo "${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${1:-Unknown Error.} (exit ${2:-1})" 1>&2
+  exit ${2:-1}
+}
+
+# Extract esp_tag (if it exists) from input into ESP_TAG shell variable.
+eval "$(jq -r '@sh "ESP_TAG=\(.esp_tag // empty)"')"
+if [ -z "${ESP_TAG}" ]; then
+  ESP_TAG="2"  # default as in gcloud_build_image.
+fi;
+
+ALL_TAGS=$(gcloud container images list-tags "${BASE_IMAGE_NAME}" \
+    --filter="tags~^${ESP_TAG}$" \
+    --format="value(tags)")
+IFS=',' read -ra TAGS_ARRAY <<< "${ALL_TAGS}"
+
+if [ ${#TAGS_ARRAY[@]} -eq 0 ]; then
+error_exit "Did not find ESP version: ${ESP_TAG}"
+fi;
+
+# Find the tag with the longest length.
+ESP_FULL_VERSION=""
+for tag in "${TAGS_ARRAY[@]}"; do
+    if [ ${#tag} -gt ${#ESP_FULL_VERSION} ]; then
+    ESP_FULL_VERSION=${tag}
+    fi
+done
+
+# Produce JSON object containing esp_full_version
+jq -n --arg esp_full_version "$ESP_FULL_VERSION" '{"esp_full_version":$esp_full_version}'