From 83816d5f3851286508a5fa9cdafb988606bc018a Mon Sep 17 00:00:00 2001 From: evamaxfield Date: Mon, 15 Aug 2022 10:40:53 -0700 Subject: [PATCH 01/16] Minor dev-infra README updates --- .editorconfig | 3 +++ dev-infrastructure/README.md | 36 ++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.editorconfig b/.editorconfig index 09084781..5d93397f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,6 @@ insert_final_newline = false [Makefile] indent_style = tab + +[Justfile] +indent_style = tab \ No newline at end of file diff --git a/dev-infrastructure/README.md b/dev-infrastructure/README.md index 0a1de564..492714e5 100644 --- a/dev-infrastructure/README.md +++ b/dev-infrastructure/README.md @@ -59,10 +59,10 @@ following commands to setup your local machine with credentials to both services ```bash cd cdp-backend/dev-infrastructure -make login -make init project={project-name} -make gen-key project={project-name} -make build +just login +just init project={project-name} +just gen-key project={project-name} +just build ``` After generating the key, name your `key` file in `cdp-backend/.keys` to `cdp-dev.json`. In case you have many keys, note that by default, the random and minimal event pipelines use the key named `cdp-dev.json`. @@ -74,19 +74,19 @@ All of these commands should be run from within the `cdp-backend/dev-infrastruct - To log in to GCloud and Pulumi: ```bash - make login + just login ``` - To create a new service account JSON key: ```bash - make gen-key project={project-name} + just gen-key project={project-name} ``` - To create a new dev infrastructure: ```bash - make init project={project-name} + just init project={project-name} ``` And follow the link logged to link a billing account to the created project. @@ -96,10 +96,10 @@ All of these commands should be run from within the `cdp-backend/dev-infrastruct - To set up infrastructure: ```bash - make build + just build ``` - **Note:** You should run `make gen-key` prior to build and ensure you have + **Note:** You should run `just gen-key` prior to build and ensure you have `GOOGLE_CREDENTIALS` set in your environment variables using: ```bash @@ -117,7 +117,7 @@ All of these commands should be run from within the `cdp-backend/dev-infrastruct - To clean and remove all database documents and file store objects: ```bash - make clean key={path-to-key} + just clean key={path-to-key} ``` **Note:** Cleaning infrastructure is best practice when comparing pipeline @@ -126,18 +126,18 @@ All of these commands should be run from within the `cdp-backend/dev-infrastruct - To reset infrastructure but reuse the same Google project: ```bash - make reset + just reset ``` **Note:** Reseting infrastructure is likely required when iterating on database models (specifically database indices). Cleaning infrastructure - should always be attempted first before reset or destroy as `make clean` + should always be attempted first before reset or destroy as `just clean` will not use any extra Google Cloud (or Firebase) projects and applications. - To delete all Pulumi and GCloud resources entirely: ```bash - make destroy project={project-name} + just destroy project={project-name} ``` **Note:** This will delete the GCP project. @@ -147,8 +147,8 @@ limits for how many projects and Firestore applications a single user can have. ### All Commands -- See Makefile commands with `make help`. - Or simply open the Makefile. All the commands are decently easy to follow. +- See Justfile commands with `just`. + Or simply open the Justfile. All the commands are decently easy to follow. - See Pulumi [CLI documentation](https://www.pulumi.com/docs/reference/cli/) for all Pulumi commands. @@ -160,18 +160,18 @@ parameters, please see the documentation for the ## Running Pipelines Against Dev Infra -Once you have a dev infrastructure set up and a key downloaded (`make gen-key`) +Once you have a dev infrastructure set up and a key downloaded (`just gen-key`) you can run pipelines and store data in the infrastructure by moving up to the base directory of this repository and running the following from `cdp-backend/`: - To run a semi-random (large permutation) event pipeline: ```bash - make run-rand-event-pipeline + just run-rand-event-pipeline ``` - To run a minimal (by definition) event pipeline: ```bash - make run-min-event-pipeline + just run-min-event-pipeline ``` From a2ff4cbffd9112d2770364ef74bdb0f5d31b8b1d Mon Sep 17 00:00:00 2001 From: evamaxfield Date: Mon, 15 Aug 2022 14:49:50 -0700 Subject: [PATCH 02/16] Start on stack debugging --- cdp_backend/infrastructure/cdp_stack.py | 10 +++++----- dev-infrastructure/Justfile | 2 +- pyproject.toml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cdp_backend/infrastructure/cdp_stack.py b/cdp_backend/infrastructure/cdp_stack.py index 412566f4..640a4a97 100644 --- a/cdp_backend/infrastructure/cdp_stack.py +++ b/cdp_backend/infrastructure/cdp_stack.py @@ -163,7 +163,7 @@ def __init__( self.firebase_init = gcp.firebase.Project( resource_name=f"{self.gcp_project_id}-firebase-init", project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self, depends_on=[self.firestore_app]), + opts=pulumi.ResourceOptions(parent=self.firestore_app), ) # Connect app engine (firestore) + bucket @@ -180,7 +180,7 @@ def __init__( bucket=self.firestore_app.default_bucket, entity="allUsers", role="READER", - opts=pulumi.ResourceOptions(parent=self.firestore_app), + opts=pulumi.ResourceOptions(parent=self.firebase_project), ) # Create all firestore indexes @@ -204,7 +204,7 @@ def __init__( idx_set_name = "_".join(idx_set_name_parts) fq_idx_set_name = f"{model_cls.collection_name}-{idx_set_name}" - # Create depends on list + # Create depends on list # We don't want to create a ton of indexes in parallel if prior_index is None: depends_on: List[pulumi.Resource] = [] @@ -220,7 +220,7 @@ def __init__( fields=idx_set_fields, query_scope="COLLECTION", opts=pulumi.ResourceOptions( - parent=self.firestore_app, + parent=self.firebase_project, depends_on=depends_on, ), ) @@ -241,7 +241,7 @@ def __init__( } ), project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self.firestore_app), + opts=pulumi.ResourceOptions(parent=self.firebase_project), ) super().register_outputs({}) diff --git a/dev-infrastructure/Justfile b/dev-infrastructure/Justfile index 5074f829..1ae8b0a6 100644 --- a/dev-infrastructure/Justfile +++ b/dev-infrastructure/Justfile @@ -58,7 +58,7 @@ destroy project: # generate a service account JSON gen-key project: - mkdir ../.keys/ -p + mkdir -p ../.keys/ rm -rf ../.keys/{{project}}.json gcloud iam service-accounts create {{project}} \ --description="CDP Dev Service Account for {{USER}}" \ diff --git a/pyproject.toml b/pyproject.toml index 46e7512e..cbb140f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,9 +53,9 @@ Documentation = "https://CouncilDataProject.github.io/cdp-backend" # https://peps.python.org/pep-0621/#dependencies-optional-dependencies [project.optional-dependencies] infrastructure = [ - "pulumi~=3.31", - "pulumi-google-native~=0.18", - "pulumi-gcp~=6.0", + "pulumi==3.33.2", + "pulumi-google-native==0.20", + "pulumi-gcp==6.26.0", ] pipeline = [ "dask[distributed]>=2021.7.0", From 01364dcd68dd6b6ca75cd9852f4668eb86f0232b Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Tue, 16 Aug 2022 15:31:31 -0700 Subject: [PATCH 03/16] Remove pulumi --- cdp_backend/infrastructure/__init__.py | 6 - cdp_backend/infrastructure/cdp_stack.py | 247 ------------------------ dev-infrastructure/Pulumi.yaml | 3 - dev-infrastructure/__main__.py | 18 -- dev-infrastructure/requirements.txt | 1 - 5 files changed, 275 deletions(-) delete mode 100644 cdp_backend/infrastructure/__init__.py delete mode 100644 cdp_backend/infrastructure/cdp_stack.py delete mode 100644 dev-infrastructure/Pulumi.yaml delete mode 100644 dev-infrastructure/__main__.py delete mode 100644 dev-infrastructure/requirements.txt diff --git a/cdp_backend/infrastructure/__init__.py b/cdp_backend/infrastructure/__init__.py deleted file mode 100644 index 5d73d761..00000000 --- a/cdp_backend/infrastructure/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Infrastructure package for cdp_backend.""" - - -from .cdp_stack import CDPStack # noqa: F401 diff --git a/cdp_backend/infrastructure/cdp_stack.py b/cdp_backend/infrastructure/cdp_stack.py deleted file mode 100644 index 640a4a97..00000000 --- a/cdp_backend/infrastructure/cdp_stack.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import json -from typing import List - -import pulumi -import pulumi_gcp as gcp -from pulumi_google_native.firestore import v1 as firestore - -from ..database import DATABASE_MODELS -from ..version import __version__ - -############################################################################### - - -class GoverningBody: - city_council = "city council" - county_council = "county council" - school_board = "school board" - other = "other" - - -class CDPStack(pulumi.ComponentResource): - def __init__( - self, - gcp_project_id: str, - municipality_name: str = "dev_municipality", - hosting_github_url: str = "no_github_host", - hosting_web_app_address: str = "no_web_app_address", - firestore_location: str = "us-west2", - governing_body: str = GoverningBody.other, - opts: pulumi.ResourceOptions = None, - ): - """ - Creates all required infrastructure and enables all required services for CDP - backend stacks. - - Parameters - ---------- - gcp_project_id: str - The id of the gcp_project, the Pulumi stack, and any other required - names for resources created during infrastructure creation will use this id - as a prefix. - I.E. `cdp-seattle` would create a Cloud Firestore instance called - `cdp-seattle`, a GCP bucket called `cdp-seattle`, etc. - - municipality_name: str - The name of the municipality this instance will be for. - I.E. "Seattle", "King County", etc. - - hosting_github_url: str - The GitHub repo this instance configuration details will be stored. - I.E. https://github.com/councildataproject/seattle-staging - - hosting_web_app_address: str - The web address for this instances web portal. - I.E. https://councildataproject.org/seattle-staging - - firestore_location: str - The location for the Cloud Firestore database and file storage servers - to be hosted. - List of locations: https://firebase.google.com/docs/firestore/locations - Default: "us-west2" - - governing_body: str - What governing body this instance will archive. - Default: GoverningBody.other - - opts: pulumi.ResourceOptions - Extra resource options to initialize the entire stack with. - Default: None - - Notes - ----- - When using this resource it is recommended to run set Pulumi parallel resource - creation to five (5) max. GCP has limits on how many resources you can create - in parallel. - - The default values for many parameters are set to fake values as this object - should primarily be used with the cookiecutter / automated scripts. The default - values in this case are set because when using this object outside of - the cookiecutter it is likely for dev infrastructures. - """ - super().__init__("CDPStack", gcp_project_id, None, opts) - - # Store parameters - self.gcp_project_id = gcp_project_id - self.firestore_location = firestore_location - - # Enable all required services - self.cloudresourcemanager = gcp.projects.Service( - f"{self.gcp_project_id}-cloudresourcemanager-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="cloudresourcemanager.googleapis.com", - opts=pulumi.ResourceOptions(parent=self), - ) - self.speech_service = gcp.projects.Service( - f"{self.gcp_project_id}-speech-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="speech.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firebase_service = gcp.projects.Service( - f"{self.gcp_project_id}-firebase-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firebase.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.app_engine_service = gcp.projects.Service( - f"{self.gcp_project_id}-app-engine-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="appengine.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firestore_service = gcp.projects.Service( - f"{self.gcp_project_id}-firestore-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firestore.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firebase_rules_service = gcp.projects.Service( - f"{self.gcp_project_id}-firebase-rules-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firebaserules.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - - # Create the firestore application - self.firestore_app = gcp.appengine.Application( - f"{self.gcp_project_id}-firestore-app", - project=self.gcp_project_id, - location_id=self.firestore_location, - database_type="CLOUD_FIRESTORE", - opts=pulumi.ResourceOptions( - parent=self, - depends_on=[ - self.firebase_service, - self.app_engine_service, - self.firestore_service, - self.firebase_rules_service, - ], - ), - ) - - # Init firebase project - self.firebase_init = gcp.firebase.Project( - resource_name=f"{self.gcp_project_id}-firebase-init", - project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self.firestore_app), - ) - - # Connect app engine (firestore) + bucket - self.firebase_project = gcp.firebase.ProjectLocation( - resource_name=f"{self.gcp_project_id}-firebase-project", - project=self.gcp_project_id, - location_id=self.firestore_location, - opts=pulumi.ResourceOptions(parent=self.firebase_init), - ) - - # Set full public read on bucket - self.storage_public_read = gcp.storage.DefaultObjectAccessControl( - f"{self.gcp_project_id}-storage-acl-public-viewer", - bucket=self.firestore_app.default_bucket, - entity="allUsers", - role="READER", - opts=pulumi.ResourceOptions(parent=self.firebase_project), - ) - - # Create all firestore indexes - prior_index = None - for model_cls in DATABASE_MODELS: - for idx_field_set in model_cls._INDEXES: - - # Add fields to field list - idx_set_name_parts = [] - idx_set_fields = [] - for idx_field in idx_field_set.fields: - idx_set_name_parts += [idx_field.name, idx_field.order] - idx_set_fields.append( - firestore.GoogleFirestoreAdminV1IndexFieldArgs( - field_path=idx_field.name, - order=idx_field.order, - ) - ) - - # Finish creating the index set name - idx_set_name = "_".join(idx_set_name_parts) - fq_idx_set_name = f"{model_cls.collection_name}-{idx_set_name}" - - # Create depends on list - # We don't want to create a ton of indexes in parallel - if prior_index is None: - depends_on: List[pulumi.Resource] = [] - else: - depends_on = [prior_index] - - # Create - prior_index = firestore.Index( - fq_idx_set_name, - project=self.gcp_project_id, - database_id="(default)", - collection_group_id=model_cls.collection_name, - fields=idx_set_fields, - query_scope="COLLECTION", - opts=pulumi.ResourceOptions( - parent=self.firebase_project, - depends_on=depends_on, - ), - ) - - # Add metadata document - gcp.firestore.Document( - f"{self.gcp_project_id}-metadata-doc", - collection="metadata", - document_id="configuration", - fields=json.dumps( - { - "infrastructure_version": {"stringValue": __version__}, - "municipality_name": {"stringValue": municipality_name}, - "hosting_github_url": {"stringValue": hosting_github_url}, - "hosting_web_app_address": {"stringValue": hosting_web_app_address}, - "firestore_location": {"stringValue": firestore_location}, - "governing_body": {"stringValue": governing_body}, - } - ), - project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self.firebase_project), - ) - - super().register_outputs({}) diff --git a/dev-infrastructure/Pulumi.yaml b/dev-infrastructure/Pulumi.yaml deleted file mode 100644 index 3a72b5f6..00000000 --- a/dev-infrastructure/Pulumi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: cdp-infrastructure -description: Dev CDP Pulumi stack -runtime: python diff --git a/dev-infrastructure/__main__.py b/dev-infrastructure/__main__.py deleted file mode 100644 index f100a2ba..00000000 --- a/dev-infrastructure/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This file is just used for dev setup and CI stack preview. -It is recommended to use the cookiecutter-cdp-deployment as it -""" - -from pulumi import export, get_stack - -from cdp_backend.infrastructure import CDPStack - -############################################################################### - -cdp_stack = CDPStack(get_stack()) - -export("firestore_address", cdp_stack.firestore_app.app_id) -export("gcp_bucket_name", cdp_stack.firestore_app.default_bucket) diff --git a/dev-infrastructure/requirements.txt b/dev-infrastructure/requirements.txt deleted file mode 100644 index f3229c5b..00000000 --- a/dev-infrastructure/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -.. From 50dd7c55b0b1c417cf0c0fd3b5780ba9ce9afb77 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Tue, 16 Aug 2022 15:32:08 -0700 Subject: [PATCH 04/16] Everything but indexes --- dev-infrastructure/Justfile | 104 ++++++++++++--------- dev-infrastructure/README.md | 142 +++++------------------------ dev-infrastructure/firebase.json | 12 +++ dev-infrastructure/firestore.rules | 7 ++ dev-infrastructure/storage.rules | 8 ++ 5 files changed, 110 insertions(+), 163 deletions(-) create mode 100644 dev-infrastructure/firebase.json create mode 100644 dev-infrastructure/firestore.rules create mode 100644 dev-infrastructure/storage.rules diff --git a/dev-infrastructure/Justfile b/dev-infrastructure/Justfile index 1ae8b0a6..2a85c49f 100644 --- a/dev-infrastructure/Justfile +++ b/dev-infrastructure/Justfile @@ -5,57 +5,20 @@ default: # get and store user USER := env_var("USER") +# Default region for infrastructures +default_region := "us-central1" +default_key := clean(join(justfile_directory(), "../.keys/cdp-dev.json")) +export GOOGLE_APPLICATION_CREDENTIALS := default_key + # run gcloud and pulumi logins login: - pulumi logout - pulumi login gcloud auth login gcloud auth application-default login -# create a new gcloud project and pulumi stack -init project: - gcloud projects create {{project}} --set-as-default - pulumi stack init {{project}} - echo "----------------------------------------------------------------------------" - echo "Follow the link to setup billing for the created GCloud account." - echo "https://console.cloud.google.com/billing/linkedaccount?project={{project}}" - -# set a CORS policy for the bucket -set-cors project: - gsutil cors set cors.json gs://{{project}}.appspot.com/ - -# run pulumi up for infra deploy -build: - pulumi up -p 4 - # switch active gcloud project switch-project project: gcloud config set project {{project}} -# remove all database documents (except search index) and filestore objects -clean key: - clean_cdp_database {{key}} - clean_cdp_filestore {{key}} - -# remove all database documents (including search index) and filestore objects -clean-full key: - clean_cdp_database {{key}} --clean-index - clean_cdp_filestore {{key}} - -# destroy just the stack (not the gcloud project) and rebuild -reset: - pulumi destroy -p 4 - echo "----------------------------------------------------------------------------" - echo "Sleeping for three minutes while resources clean up" - sleep 180 - make build - -# fully teardown project -destroy project: - pulumi stack rm {{project}} --force - gcloud projects delete {{project}} - rm -f ../.keys/{{project}}.json - # generate a service account JSON gen-key project: mkdir -p ../.keys/ @@ -72,4 +35,59 @@ gen-key project: echo "Sleeping for one minute while resources set up" sleep 60 cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json - echo "Key generation complete, updated ../.keys/cdp-dev.json to {{project}} key. Be sure to update GOOGLE_CREDENTIALS." \ No newline at end of file + +# create a new gcloud project and pulumi stack +init project: + gcloud projects create {{project}} --set-as-default + echo "----------------------------------------------------------------------------" + echo "Follow the link to setup billing for the created GCloud account." + echo "https://console.cloud.google.com/billing/linkedaccount?project={{project}}" + echo "----------------------------------------------------------------------------" + just gen-key {{project}} + +# enable gcloud services +enable-services: + gcloud services enable cloudresourcemanager.googleapis.com + gcloud services enable speech.googleapis.com \ + appengine.googleapis.com \ + storage.googleapis.com \ + firebase.googleapis.com \ + firestore.googleapis.com \ + firebaserules.googleapis.com \ + firebasestorage.googleapis.com + +# setup the basic gcloud app and firestore connection +setup region=default_region: + just enable-services + gcloud app create --region={{replace(region, "us-central1", "us-central")}} + gcloud alpha firestore databases create --region={{replace(region, "us-central1", "us-central")}} + +# deploy the CDP specific firestore and storage requirements +deploy project: + just enable-services + -firebase projects:addfirebase {{project}} + firebase use --add {{project}} + firebase deploy --only firestore:rules + firebase deploy --only firestore:indexes + firebase deploy --only storage + gsutil cors set cors.json gs://{{project}}.appspot.com/ + +# run both setup and deploy +setup-and-deploy project region=default_region: + just setup {{region}} + just deploy {{project}} + +# fully teardown project +destroy project: + gcloud projects delete {{project}} + rm -f ../.keys/{{project}}.json + +# remove all database documents (except search index) and filestore objects +clean key=default_key: + clean_cdp_database {{key}} + clean_cdp_filestore {{key}} + +# remove all database documents (including search index) and filestore objects +clean-full key=default_key: + clean_cdp_database {{key}} --clean-index + clean_cdp_filestore {{key}} \ No newline at end of file diff --git a/dev-infrastructure/README.md b/dev-infrastructure/README.md index 492714e5..b3903265 100644 --- a/dev-infrastructure/README.md +++ b/dev-infrastructure/README.md @@ -27,10 +27,6 @@ found in our Deploying the CDP infrastructure requires having `cdp-backend` installed. -```bash -pip install -e ../[dev] -``` - For more detailed information please see the [project installation details](https://github.com/CouncilDataProject/cdp-backend#installation). @@ -40,138 +36,44 @@ For more detailed information please see the ([Google Cloud Console Home](https://console.cloud.google.com/)) 2. Create (or re-use) a [billing account](https://console.cloud.google.com/billing) and attach it to your GCP account. -3. Create (or sign in to) a - [Pulumi account](https://app.pulumi.com/signup). ## Environment Setup -The only environment setup needed to run this deployment is to make sure `pulumi` itself -and the `gcloud` SDK are both installed. - -- [pulumi](https://www.pulumi.com/docs/get-started/install/) -- [gcloud](https://cloud.google.com/sdk/install) +The only environment setup needed to run this deployment is to make +sure the [`gcloud` SDK](https://cloud.google.com/sdk/install) is installed. _If this was the first time installing either of those packages, it is recommended to restart your terminal after installation._ -After `pulumi` and `gcloud` have both been installed and terminal restarted, run the -following commands to setup your local machine with credentials to both services. +After `gcloud` has both been installed and terminal restarted, run the +following commands to log in to gcloud: `just login` + +## Create a New Project and Deploy the Infrastructure ```bash -cd cdp-backend/dev-infrastructure -just login -just init project={project-name} -just gen-key project={project-name} -just build +just init {project-name} +just setup-and-deploy {project-name} {OPTIONAL: region} ``` -After generating the key, name your `key` file in `cdp-backend/.keys` to `cdp-dev.json`. In case you have many keys, note that by default, the random and minimal event pipelines use the key named `cdp-dev.json`. - -## Infrastructure Management Commands - -All of these commands should be run from within the `cdp-backend/dev-infrastructure` directory. - -- To log in to GCloud and Pulumi: - - ```bash - just login - ``` - -- To create a new service account JSON key: - - ```bash - just gen-key project={project-name} - ``` - -- To create a new dev infrastructure: - - ```bash - just init project={project-name} - ``` - - And follow the link logged to link a billing account to the created project. - - **Note:** This will create a new GCP project. - -- To set up infrastructure: - - ```bash - just build - ``` - - **Note:** You should run `just gen-key` prior to build and ensure you have - `GOOGLE_CREDENTIALS` set in your environment variables using: - - ```bash - export GOOGLE_CREDENTIALS=$(cat ../.keys/{project-name}-sa-dev.json) - ``` - - and replacing `{project-name}` with your project name. - - or if you have already renamed your key: +Example: - ```bash - export GOOGLE_CREDENTIALS=$(cat ../.keys/cdp-dev.json) - ``` - -- To clean and remove all database documents and file store objects: - - ```bash - just clean key={path-to-key} - ``` - - **Note:** Cleaning infrastructure is best practice when comparing pipeline - outputs and database models aren't changing (specifically database indices). - -- To reset infrastructure but reuse the same Google project: - - ```bash - just reset - ``` - - **Note:** Reseting infrastructure is likely required when iterating on - database models (specifically database indices). Cleaning infrastructure - should always be attempted first before reset or destroy as `just clean` - will not use any extra Google Cloud (or Firebase) projects and applications. +```bash +just init cdp-eva-dev-001 +just setup-and-deploy cdp-eva-dev-001 +``` -- To delete all Pulumi and GCloud resources entirely: +## Update an Existing Project - ```bash - just destroy project={project-name} - ``` +```bash +just deploy {project-name} +``` - **Note:** This will delete the GCP project. +Example: -Try to use the same project and infrastructure as much as possible, there are -limits for how many projects and Firestore applications a single user can have. +```bash +just deploy cdp-eva-dev-001 +``` ### All Commands -- See Justfile commands with `just`. - Or simply open the Justfile. All the commands are decently easy to follow. -- See Pulumi [CLI documentation](https://www.pulumi.com/docs/reference/cli/) - for all Pulumi commands. - -## Non-Default Parameters - -If you want to edit the default behavior of the `__main__.py` file and change the -parameters, please see the documentation for the -[CDPStack object](https://councildataproject.github.io/cdp-backend/cdp_backend.infrastructure.html#module-cdp_backend.infrastructure.cdp_stack) - -## Running Pipelines Against Dev Infra - -Once you have a dev infrastructure set up and a key downloaded (`just gen-key`) -you can run pipelines and store data in the infrastructure by moving up to the -base directory of this repository and running the following from `cdp-backend/`: - -- To run a semi-random (large permutation) event pipeline: - - ```bash - just run-rand-event-pipeline - ``` - -- To run a minimal (by definition) event pipeline: - - ```bash - just run-min-event-pipeline - ``` +- See Justfile commands with `just` or open the Justfile. diff --git a/dev-infrastructure/firebase.json b/dev-infrastructure/firebase.json new file mode 100644 index 00000000..4110c3b8 --- /dev/null +++ b/dev-infrastructure/firebase.json @@ -0,0 +1,12 @@ +{ + "firestore": { + "rules": "firestore.rules", + "indexes": "firestore.indexes.json" + }, + "storage": { + "rules": "storage.rules" + }, + "functions": { + "source": "functions" + } +} diff --git a/dev-infrastructure/firestore.rules b/dev-infrastructure/firestore.rules new file mode 100644 index 00000000..db967a33 --- /dev/null +++ b/dev-infrastructure/firestore.rules @@ -0,0 +1,7 @@ +service cloud.firestore { + match /databases/{database}/documents { + match /{document=**} { + allow read; + } + } +} \ No newline at end of file diff --git a/dev-infrastructure/storage.rules b/dev-infrastructure/storage.rules new file mode 100644 index 00000000..4eda34fd --- /dev/null +++ b/dev-infrastructure/storage.rules @@ -0,0 +1,8 @@ +rules_version = '2'; +service firebase.storage { + match /b/{bucket}/o { + match /{allPaths=**} { + allow read, write: if request.auth!=null; + } + } +} From 34261ae1ba0862ea8eafa5d5307403f6a8acdbfa Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Tue, 16 Aug 2022 16:22:46 -0700 Subject: [PATCH 05/16] Create the indexes and the metadata doc in Python --- .../bin/create_cdp_firestore_indexes_json.py | 89 ++++ .../bin/store_cdp_metadata_document.py | 88 ++++ cdp_backend/database/models.py | 144 +++--- cdp_backend/database/types.py | 15 +- dev-infrastructure/Justfile | 5 + .../dev-cookiecutter-metadata.yaml | 13 + dev-infrastructure/firestore.indexes.json | 473 ++++++++++++++++++ pyproject.toml | 3 + 8 files changed, 756 insertions(+), 74 deletions(-) create mode 100644 cdp_backend/bin/create_cdp_firestore_indexes_json.py create mode 100644 cdp_backend/bin/store_cdp_metadata_document.py create mode 100644 dev-infrastructure/dev-cookiecutter-metadata.yaml create mode 100644 dev-infrastructure/firestore.indexes.json diff --git a/cdp_backend/bin/create_cdp_firestore_indexes_json.py b/cdp_backend/bin/create_cdp_firestore_indexes_json.py new file mode 100644 index 00000000..c1dbcc44 --- /dev/null +++ b/cdp_backend/bin/create_cdp_firestore_indexes_json.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import json +import logging +import sys +import traceback +from pathlib import Path + +from cdp_backend.database import DATABASE_MODELS + +############################################################################### + +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", +) +log = logging.getLogger(__name__) + +############################################################################### + + +class Args(argparse.Namespace): + def __init__(self) -> None: + self.__parse() + + def __parse(self) -> None: + p = argparse.ArgumentParser( + prog="create_cdp_firestore_indexes_json", + description="Create the CDP Firestore indexes JSON file.", + ) + p.add_argument( + "outfile", + type=Path, + help="Path to where the indexes JSON file should be stored." + ) + p.parse_args(namespace=self) + + +############################################################################### + + +def _generate_indexes_json(outfile: Path) -> None: + # All indexes + indexes = [] + + for model_cls in DATABASE_MODELS: + for idx_field_set in model_cls._INDEXES: + + indexes.append( + { + "collectionGroup": model_cls.collection_name, + "queryScope": "COLLECTION", + "fields": idx_field_set.to_dict()["fields"], + } + ) + + # Add indexes to the normal JSON format + indexes_full_json = { + "indexes": indexes, + "fieldOverrides": [], + } + + # Write out the file + outfile = outfile.resolve() + with open(outfile, "w") as open_f: + json.dump(indexes_full_json, open_f, indent=2) + log.info(f"Wrote out CDP firestore.indexes.json to: '{outfile}'") + + +def main() -> None: + try: + args = Args() + _generate_indexes_json(outfile=args.outfile) + except Exception as e: + log.error("=============================================") + log.error("\n\n" + traceback.format_exc()) + log.error("=============================================") + log.error("\n\n" + str(e) + "\n") + log.error("=============================================") + sys.exit(1) + + +############################################################################### +# Allow caller to directly run this module (usually in development scenarios) + +if __name__ == "__main__": + main() diff --git a/cdp_backend/bin/store_cdp_metadata_document.py b/cdp_backend/bin/store_cdp_metadata_document.py new file mode 100644 index 00000000..3bd9e1ca --- /dev/null +++ b/cdp_backend/bin/store_cdp_metadata_document.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import logging +import sys +import traceback +from pathlib import Path + +from google.cloud.firestore import Client +import yaml + +from .. import __version__ + +############################################################################### + +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", +) +log = logging.getLogger(__name__) + +############################################################################### + + +class Args(argparse.Namespace): + def __init__(self) -> None: + self.__parse() + + def __parse(self) -> None: + p = argparse.ArgumentParser( + prog="store_cdp_metadata_document", + description="Store the CDP metadata document to a firestore instance.", + ) + p.add_argument( + "google_credentials_file", + type=Path, + help="Path to Google service account JSON key.", + ) + p.add_argument( + "cookiecutter_yaml", + type=Path, + help="Path to the CDP Cookiecutter YAML file to lookup metadata details.", + ) + p.parse_args(namespace=self) + + +############################################################################### + + +def _store_cdp_metadata_document(google_credentials_file: Path, cookiecutter_yaml: Path,) -> None: + # Read the cookiecutter file + with open(cookiecutter_yaml, "r") as open_f: + cookiecutter_meta = yaml.load(open_f, Loader=yaml.FullLoader)["default_context"] + + # Open client and write doc + client = Client.from_service_account_json(google_credentials_file) + collection = client.collection("metadata") + collection.document("configuration").set( + { + "infrastructure_version": __version__, + "municipality_name": cookiecutter_meta["municipality"], + "hosting_github_url": cookiecutter_meta["hosting_github_url"], + "hosting_web_app_address": cookiecutter_meta["hosting_web_app_address"], + "firestore_location": cookiecutter_meta["firestore_region"], + "governing_body": cookiecutter_meta["governing_body_type"], + } + ) + + +def main() -> None: + try: + args = Args() + _store_cdp_metadata_document(google_credentials_file=args.google_credentials_file, cookiecutter_yaml=args.cookiecutter_yaml,) + except Exception as e: + log.error("=============================================") + log.error("\n\n" + traceback.format_exc()) + log.error("=============================================") + log.error("\n\n" + str(e) + "\n") + log.error("=============================================") + sys.exit(1) + + +############################################################################### +# Allow caller to directly run this module (usually in development scenarios) + +if __name__ == "__main__": + main() diff --git a/cdp_backend/database/models.py b/cdp_backend/database/models.py index da219988..611526e3 100644 --- a/cdp_backend/database/models.py +++ b/cdp_backend/database/models.py @@ -216,67 +216,67 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="seat_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="seat_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="end_datetime", order=Order.ASCENDING), - IndexedField(name="seat_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="seat_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), ) @@ -358,8 +358,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="name", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="name", order=Order.ASCENDING), ) ), ) @@ -389,8 +389,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), ) ), ) @@ -469,14 +469,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="body_ref", order=Order.ASCENDING), - IndexedField(name="event_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="body_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="body_ref", order=Order.ASCENDING), - IndexedField(name="event_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="body_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_datetime", order=Order.DESCENDING), ) ), ) @@ -524,8 +524,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="session_index", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="session_index", order=Order.ASCENDING), ) ), ) @@ -560,14 +560,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="session_ref", order=Order.ASCENDING), - IndexedField(name="created", order=Order.DESCENDING), + IndexedField(fieldPath="session_ref", order=Order.ASCENDING), + IndexedField(fieldPath="created", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="session_ref", order=Order.ASCENDING), - IndexedField(name="confidence", order=Order.DESCENDING), + IndexedField(fieldPath="session_ref", order=Order.ASCENDING), + IndexedField(fieldPath="confidence", order=Order.DESCENDING), ) ), ) @@ -605,14 +605,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="index", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="index", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="index", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="index", order=Order.DESCENDING), ) ), ) @@ -657,14 +657,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="update_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="update_datetime", order=Order.ASCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="update_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="update_datetime", order=Order.DESCENDING), ), ), ) @@ -708,8 +708,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_minutes_item_ref", order=Order.ASCENDING), - IndexedField(name="name", order=Order.ASCENDING), + IndexedField(fieldPath="event_minutes_item_ref", order=Order.ASCENDING), + IndexedField(fieldPath="name", order=Order.ASCENDING), ) ), ) @@ -758,26 +758,26 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), ) ), ) @@ -819,38 +819,44 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), IndexedFieldSet( ( - IndexedField(name="stemmed_gram", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="stemmed_gram", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="stemmed_gram", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="stemmed_gram", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), IndexedFieldSet( ( - IndexedField(name="unstemmed_gram", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="unstemmed_gram", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="unstemmed_gram", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="unstemmed_gram", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), ) diff --git a/cdp_backend/database/types.py b/cdp_backend/database/types.py index 563eaefb..063a4fbf 100644 --- a/cdp_backend/database/types.py +++ b/cdp_backend/database/types.py @@ -1,15 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import NamedTuple, Tuple +from dataclasses import dataclass +from typing import List + +from dataclasses_json import DataClassJsonMixin ############################################################################### -class IndexedField(NamedTuple): - name: str +@dataclass +class IndexedField(DataClassJsonMixin): + fieldPath: str order: str -class IndexedFieldSet(NamedTuple): - fields: Tuple[IndexedField, ...] +@dataclass +class IndexedFieldSet(DataClassJsonMixin): + fields: List[IndexedField] diff --git a/dev-infrastructure/Justfile b/dev-infrastructure/Justfile index 2a85c49f..251e59f5 100644 --- a/dev-infrastructure/Justfile +++ b/dev-infrastructure/Justfile @@ -67,9 +67,14 @@ deploy project: just enable-services -firebase projects:addfirebase {{project}} firebase use --add {{project}} + create_cdp_firestore_indexes_json \ + {{join(justfile_directory(), "firestore.indexes.json")}} firebase deploy --only firestore:rules firebase deploy --only firestore:indexes firebase deploy --only storage + store_cdp_metadata_document \ + {{default_key}} \ + {{join(justfile_directory(), "dev-cookiecutter-metadata.yaml")}} gsutil cors set cors.json gs://{{project}}.appspot.com/ # run both setup and deploy diff --git a/dev-infrastructure/dev-cookiecutter-metadata.yaml b/dev-infrastructure/dev-cookiecutter-metadata.yaml new file mode 100644 index 00000000..36e2591a --- /dev/null +++ b/dev-infrastructure/dev-cookiecutter-metadata.yaml @@ -0,0 +1,13 @@ +default_context: + municipality: "CDP Dev Muni" + iana_timezone: "America/Los_Angeles" + governing_body_type: "other" + municipality_slug: "cdp-dev-muni" + python_municipality_slug: "cdp_dev_muni" + infrastructure_slug: "cdp_dev_muni_000001" + maintainer_or_org_full_name: "evamaxfield" + hosting_github_username_or_org: "CouncilDataProject" + hosting_github_repo_name: "cdp-dev-muni" + hosting_github_url: "https://github.com/CouncilDataProject/cdp-dev-muni" + hosting_web_app_address: "https://councildataproject.github.io/cdp-dev-muni" + firestore_region: "us-central" diff --git a/dev-infrastructure/firestore.indexes.json b/dev-infrastructure/firestore.indexes.json new file mode 100644 index 00000000..515318bd --- /dev/null +++ b/dev-infrastructure/firestore.indexes.json @@ -0,0 +1,473 @@ +{ + "indexes": [ + { + "collectionGroup": "event", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "body_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "event_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "event", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "body_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "event_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "event_minutes_item", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "index", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "event_minutes_item", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "index", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "event_minutes_item_file", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_minutes_item_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "name", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "datetime_weighted_value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "stemmed_gram", + "order": "ASCENDING" + }, + { + "fieldPath": "value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "stemmed_gram", + "order": "ASCENDING" + }, + { + "fieldPath": "datetime_weighted_value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "unstemmed_gram", + "order": "ASCENDING" + }, + { + "fieldPath": "value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "indexed_event_gram", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "unstemmed_gram", + "order": "ASCENDING" + }, + { + "fieldPath": "datetime_weighted_value", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "matter_file", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "name", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "matter_sponsor", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "matter_status", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "update_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "matter_status", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "update_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "start_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "start_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "end_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "end_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "seat_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "end_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "title", + "order": "ASCENDING" + }, + { + "fieldPath": "start_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "title", + "order": "ASCENDING" + }, + { + "fieldPath": "start_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "title", + "order": "ASCENDING" + }, + { + "fieldPath": "end_datetime", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "title", + "order": "ASCENDING" + }, + { + "fieldPath": "end_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "role", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "end_datetime", + "order": "ASCENDING" + }, + { + "fieldPath": "seat_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "start_datetime", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "session", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "session_index", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "transcript", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "session_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "created", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "transcript", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "session_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "confidence", + "order": "DESCENDING" + } + ] + }, + { + "collectionGroup": "vote", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "event_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "person_ref", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "vote", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "person_ref", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "vote", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "event_ref", + "order": "ASCENDING" + } + ] + }, + { + "collectionGroup": "vote", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "person_ref", + "order": "ASCENDING" + }, + { + "fieldPath": "matter_ref", + "order": "ASCENDING" + } + ] + } + ], + "fieldOverrides": [] +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cbb140f7..9be7b1ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "fireo>=1.5", "fsspec", # Version pin set by gcsfs "gcsfs>=2022.7.1", + "google-cloud-firestore", # Version pin set by fireo "requests>=2.26.0", ] @@ -137,6 +138,8 @@ process_cdp_event_index_chunk = "cdp_backend.bin.process_cdp_event_index_chunk:m search_cdp_events = "cdp_backend.bin.search_cdp_events:main" process_special_event = "cdp_backend.bin.process_special_event:main" add_content_hash_to_sessions = "cdp_backend.bin.add_content_hash_to_sessions:main" +create_cdp_firestore_indexes_json = "cdp_backend.bin.create_cdp_firestore_indexes_json:main" +store_cdp_metadata_document = "cdp_backend.bin.store_cdp_metadata_document:main" # build settings # https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html From ae58d76c4a4368334951316560407e9aee2c2880 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Tue, 16 Aug 2022 16:44:42 -0700 Subject: [PATCH 06/16] Remove infra requirements --- .github/workflows/ci.yml | 2 +- Justfile | 2 +- pyproject.toml | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2660bf31..4b49f3e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install .[infrastructure,pipeline,test] + pip install .[pipeline,test] - name: Run Tests run: just test - name: Upload Codecov diff --git a/Justfile b/Justfile index 57da129c..f6f3c4f7 100644 --- a/Justfile +++ b/Justfile @@ -27,7 +27,7 @@ clean: # install with all deps install: - pip install -e .[pipeline,infrastructure,lint,test,docs,dev] + pip install -e .[pipeline,lint,test,docs,dev] # lint, format, and check all files lint: diff --git a/pyproject.toml b/pyproject.toml index 9be7b1ea..f32d3b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,11 +53,6 @@ Documentation = "https://CouncilDataProject.github.io/cdp-backend" # extra dependencies # https://peps.python.org/pep-0621/#dependencies-optional-dependencies [project.optional-dependencies] -infrastructure = [ - "pulumi==3.33.2", - "pulumi-google-native==0.20", - "pulumi-gcp==6.26.0", -] pipeline = [ "dask[distributed]>=2021.7.0", "ffmpeg-python==0.2.0", From 8fba0e66c04d1d8bceb3852ae7c15d1621cfde23 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Tue, 16 Aug 2022 16:51:51 -0700 Subject: [PATCH 07/16] Lint, format, fix tests --- .pre-commit-config.yaml | 1 + .../bin/create_cdp_firestore_indexes_json.py | 4 +- .../bin/store_cdp_metadata_document.py | 14 +++-- cdp_backend/tests/database/test_models.py | 13 ++--- cdp_backend/tests/infrastructure/__init__.py | 6 --- .../tests/infrastructure/test_cdp_stack.py | 53 ------------------- pyproject.toml | 3 +- 7 files changed, 19 insertions(+), 75 deletions(-) delete mode 100644 cdp_backend/tests/infrastructure/__init__.py delete mode 100644 cdp_backend/tests/infrastructure/test_cdp_stack.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57e43824..e6d861e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,4 @@ repos: additional_dependencies: - "types-pytz" - "types-requests" + - "types-PyYAML" diff --git a/cdp_backend/bin/create_cdp_firestore_indexes_json.py b/cdp_backend/bin/create_cdp_firestore_indexes_json.py index c1dbcc44..0a585c5b 100644 --- a/cdp_backend/bin/create_cdp_firestore_indexes_json.py +++ b/cdp_backend/bin/create_cdp_firestore_indexes_json.py @@ -33,7 +33,7 @@ def __parse(self) -> None: p.add_argument( "outfile", type=Path, - help="Path to where the indexes JSON file should be stored." + help="Path to where the indexes JSON file should be stored.", ) p.parse_args(namespace=self) @@ -61,7 +61,7 @@ def _generate_indexes_json(outfile: Path) -> None: "indexes": indexes, "fieldOverrides": [], } - + # Write out the file outfile = outfile.resolve() with open(outfile, "w") as open_f: diff --git a/cdp_backend/bin/store_cdp_metadata_document.py b/cdp_backend/bin/store_cdp_metadata_document.py index 3bd9e1ca..e2740203 100644 --- a/cdp_backend/bin/store_cdp_metadata_document.py +++ b/cdp_backend/bin/store_cdp_metadata_document.py @@ -7,8 +7,8 @@ import traceback from pathlib import Path -from google.cloud.firestore import Client import yaml +from google.cloud.firestore import Client from .. import __version__ @@ -48,11 +48,14 @@ def __parse(self) -> None: ############################################################################### -def _store_cdp_metadata_document(google_credentials_file: Path, cookiecutter_yaml: Path,) -> None: +def _store_cdp_metadata_document( + google_credentials_file: Path, + cookiecutter_yaml: Path, +) -> None: # Read the cookiecutter file with open(cookiecutter_yaml, "r") as open_f: cookiecutter_meta = yaml.load(open_f, Loader=yaml.FullLoader)["default_context"] - + # Open client and write doc client = Client.from_service_account_json(google_credentials_file) collection = client.collection("metadata") @@ -71,7 +74,10 @@ def _store_cdp_metadata_document(google_credentials_file: Path, cookiecutter_yam def main() -> None: try: args = Args() - _store_cdp_metadata_document(google_credentials_file=args.google_credentials_file, cookiecutter_yaml=args.cookiecutter_yaml,) + _store_cdp_metadata_document( + google_credentials_file=args.google_credentials_file, + cookiecutter_yaml=args.cookiecutter_yaml, + ) except Exception as e: log.error("=============================================") log.error("\n\n" + traceback.format_exc()) diff --git a/cdp_backend/tests/database/test_models.py b/cdp_backend/tests/database/test_models.py index 8f7f1016..e7b2b750 100644 --- a/cdp_backend/tests/database/test_models.py +++ b/cdp_backend/tests/database/test_models.py @@ -34,16 +34,11 @@ def test_validate_model_definitions() -> None: # Check that all index fields are valid attributes of the model for idx_field_set in model_cls._INDEXES: for idx_field in idx_field_set.fields: - assert hasattr(m, idx_field.name) + assert hasattr(m, idx_field.fieldPath) - # Check that all primary keys are valid attributes of the model - for pk in model_cls._PRIMARY_KEYS: - assert hasattr(m, pk) - - # Check that all index fields are valid attributes of the model - for idx_field_set in model_cls._INDEXES: - for idx_field in idx_field_set.fields: - assert hasattr(m, idx_field.name) + # Check that all primary keys are valid attributes of the model + for pk in model_cls._PRIMARY_KEYS: + assert hasattr(m, pk) def test_cdp_database_model_has_no_cyclic_dependencies(tmpdir: Path) -> None: diff --git a/cdp_backend/tests/infrastructure/__init__.py b/cdp_backend/tests/infrastructure/__init__.py deleted file mode 100644 index c5b99e2c..00000000 --- a/cdp_backend/tests/infrastructure/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Unit test package for cdp_backend.infrastructure""" - -# Import to avoid breaking pulumi test running on asynchronous loop -import prefect # noqa: F401 diff --git a/cdp_backend/tests/infrastructure/test_cdp_stack.py b/cdp_backend/tests/infrastructure/test_cdp_stack.py deleted file mode 100644 index 3d297342..00000000 --- a/cdp_backend/tests/infrastructure/test_cdp_stack.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import unittest -from typing import Any, Dict, List, Optional, Tuple - -import pulumi - -############################################################################### - - -# Create the various mocks. -class CDPStackMocks(pulumi.runtime.Mocks): - def call( - self, args: pulumi.runtime.MockCallArgs - ) -> Tuple[Dict[Any, Any], Optional[List[Tuple[str, str]]]]: - return {}, None - - def new_resource( - self, args: pulumi.runtime.MockResourceArgs - ) -> Tuple[Optional[str], Dict[Any, Any]]: - return (args.name + "_id", args.inputs) - - -pulumi.runtime.set_mocks(CDPStackMocks()) - -from cdp_backend.infrastructure import CDPStack - -############################################################################### - - -class InfrastructureTests(unittest.TestCase): - @pulumi.runtime.test - def test_basic_run(self) -> None: - gcp_project_id = "mocked-testing-stack" - - # Write output checks - def check_firestore_app_id(args: List[Any]) -> None: - app_id = args - assert app_id == f"{gcp_project_id}.fake-appspot.io" - - def check_firestore_default_bucket(args: List[Any]) -> None: - default_bucket = args - assert default_bucket == f"gcs://{gcp_project_id}" - - # Init mocked stack - stack = CDPStack("mocked-testing-stack") - - # Check outputs - pulumi.Output.all(stack.firestore_app.app_id).apply(check_firestore_app_id) - pulumi.Output.all(stack.firestore_app.default_bucket).apply( - check_firestore_default_bucket - ) diff --git a/pyproject.toml b/pyproject.toml index f32d3b2c..5c821899 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,8 +87,9 @@ lint = [ "isort>=5.7.0", "mypy>=0.790", "pre-commit>=2.20.0", - "types-pytz>=2022.1.2", "types-requests>=2.28.5", + "types-pytz>=2022.1.2", + "types-PyYAML>=6.0.11", ] test = [ # Pytest From 9782e21185cda31c670d59dbea15a5267a3806aa Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 11:37:37 -0700 Subject: [PATCH 08/16] Function to copy stored infrastructure files to dir --- .gitignore | 6 + __init__.py | 13 + ...son.py => get_cdp_infrastructure_stack.py} | 30 +- .../infrastructure}/Justfile | 1 + cdp_backend/infrastructure/__init__.py | 13 + .../infrastructure}/cors.json | 0 .../infrastructure}/firebase.json | 0 .../infrastructure}/firestore.rules | 0 .../infrastructure}/storage.rules | 0 dev-infrastructure/README.md | 14 + dev-infrastructure/firestore.indexes.json | 473 ------------------ pyproject.toml | 11 +- 12 files changed, 81 insertions(+), 480 deletions(-) create mode 100644 __init__.py rename cdp_backend/bin/{create_cdp_firestore_indexes_json.py => get_cdp_infrastructure_stack.py} (70%) rename {dev-infrastructure => cdp_backend/infrastructure}/Justfile (96%) create mode 100644 cdp_backend/infrastructure/__init__.py rename {dev-infrastructure => cdp_backend/infrastructure}/cors.json (100%) rename {dev-infrastructure => cdp_backend/infrastructure}/firebase.json (100%) rename {dev-infrastructure => cdp_backend/infrastructure}/firestore.rules (100%) rename {dev-infrastructure => cdp_backend/infrastructure}/storage.rules (100%) delete mode 100644 dev-infrastructure/firestore.indexes.json diff --git a/.gitignore b/.gitignore index ff9b0a39..081c8210 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,12 @@ index-chunks/ abc123-cdp* *-person-picture *-seat-image +dev-infrastructure/cors.json +dev-infrastructure/firebase.json +dev-infrastructure/firestore.indexes.json +dev-infrastructure/firestore.rules +dev-infrastructure/storage.rules +dev-infrastructure/Justfile # Common credentials directories .keys/ diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..d4a69b75 --- /dev/null +++ b/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +""" +The files required for setting up an infrastructure stack. + +They are stored here in cdp-backend rather than the cookiecutter +so that all backend components are together and it makes it easy to +create dev-infrastructures for development. +""" + +from pathlib import Path + +INFRA_DIR = Path(__file__).parent \ No newline at end of file diff --git a/cdp_backend/bin/create_cdp_firestore_indexes_json.py b/cdp_backend/bin/get_cdp_infrastructure_stack.py similarity index 70% rename from cdp_backend/bin/create_cdp_firestore_indexes_json.py rename to cdp_backend/bin/get_cdp_infrastructure_stack.py index 0a585c5b..5a61ab8f 100644 --- a/cdp_backend/bin/create_cdp_firestore_indexes_json.py +++ b/cdp_backend/bin/get_cdp_infrastructure_stack.py @@ -4,11 +4,13 @@ import argparse import json import logging +import shutil import sys import traceback from pathlib import Path from cdp_backend.database import DATABASE_MODELS +from cdp_backend.infrastructure import INFRA_DIR ############################################################################### @@ -27,13 +29,18 @@ def __init__(self) -> None: def __parse(self) -> None: p = argparse.ArgumentParser( - prog="create_cdp_firestore_indexes_json", - description="Create the CDP Firestore indexes JSON file.", + prog="get_cdp_infrastructure_stack", + description=( + "Generate or copy all the files needed for a new CDP infrastructure." + ), ) p.add_argument( - "outfile", + "output_dir", type=Path, - help="Path to where the indexes JSON file should be stored.", + help=( + "Path to where the infrastructure files should be copied " + "or generated." + ), ) p.parse_args(namespace=self) @@ -69,10 +76,23 @@ def _generate_indexes_json(outfile: Path) -> None: log.info(f"Wrote out CDP firestore.indexes.json to: '{outfile}'") +def _copy_infra_files(output_dir: Path) -> None: + # Copy each file in the infra dir to the output dir + output_dir.mkdir(parents=True, exist_ok=True) + for f in INFRA_DIR.iterdir(): + if f.name not in [ + "__pycache__", + "__init__.py", + ]: + shutil.copy(f, output_dir / f.name) + + def main() -> None: try: args = Args() - _generate_indexes_json(outfile=args.outfile) + output_dir = args.output_dir.expanduser().resolve() + _copy_infra_files(output_dir=output_dir) + _generate_indexes_json(outfile=output_dir / "firestore.indexes.json") except Exception as e: log.error("=============================================") log.error("\n\n" + traceback.format_exc()) diff --git a/dev-infrastructure/Justfile b/cdp_backend/infrastructure/Justfile similarity index 96% rename from dev-infrastructure/Justfile rename to cdp_backend/infrastructure/Justfile index 251e59f5..98938f2c 100644 --- a/dev-infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -33,6 +33,7 @@ gen-key project: --iam-account "{{project}}@{{project}}.iam.gserviceaccount.com" echo "----------------------------------------------------------------------------" echo "Sleeping for one minute while resources set up" + echo "----------------------------------------------------------------------------" sleep 60 cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json diff --git a/cdp_backend/infrastructure/__init__.py b/cdp_backend/infrastructure/__init__.py new file mode 100644 index 00000000..88d4423d --- /dev/null +++ b/cdp_backend/infrastructure/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +""" +The files required for setting up an infrastructure stack. + +They are stored here in cdp-backend rather than the cookiecutter +so that all backend components are together and it makes it easy to +create dev-infrastructures for development. +""" + +from pathlib import Path + +INFRA_DIR = Path(__file__).parent diff --git a/dev-infrastructure/cors.json b/cdp_backend/infrastructure/cors.json similarity index 100% rename from dev-infrastructure/cors.json rename to cdp_backend/infrastructure/cors.json diff --git a/dev-infrastructure/firebase.json b/cdp_backend/infrastructure/firebase.json similarity index 100% rename from dev-infrastructure/firebase.json rename to cdp_backend/infrastructure/firebase.json diff --git a/dev-infrastructure/firestore.rules b/cdp_backend/infrastructure/firestore.rules similarity index 100% rename from dev-infrastructure/firestore.rules rename to cdp_backend/infrastructure/firestore.rules diff --git a/dev-infrastructure/storage.rules b/cdp_backend/infrastructure/storage.rules similarity index 100% rename from dev-infrastructure/storage.rules rename to cdp_backend/infrastructure/storage.rules diff --git a/dev-infrastructure/README.md b/dev-infrastructure/README.md index b3903265..b714c32e 100644 --- a/dev-infrastructure/README.md +++ b/dev-infrastructure/README.md @@ -51,13 +51,17 @@ following commands to log in to gcloud: `just login` ## Create a New Project and Deploy the Infrastructure ```bash +get_cdp_infrastructure_stack {OPTIONAL: dir path to store to} just init {project-name} just setup-and-deploy {project-name} {OPTIONAL: region} ``` Example: +_Assuming user is within the `dev-infrastructure` dir._ + ```bash +get_cdp_infrastructure_stack . just init cdp-eva-dev-001 just setup-and-deploy cdp-eva-dev-001 ``` @@ -77,3 +81,13 @@ just deploy cdp-eva-dev-001 ### All Commands - See Justfile commands with `just` or open the Justfile. + + +### Changing the Stack + +The actual infrastructure files live in the `cdp_backend/infrastructure` module. +To make changes to the infrastructure stack, change the files in that module and then +rerun `get-cdp-infrastructure-stack`. + +Note: the database indexes are store in the `cdp_backend/database/models.py` module +with each collection model. \ No newline at end of file diff --git a/dev-infrastructure/firestore.indexes.json b/dev-infrastructure/firestore.indexes.json deleted file mode 100644 index 515318bd..00000000 --- a/dev-infrastructure/firestore.indexes.json +++ /dev/null @@ -1,473 +0,0 @@ -{ - "indexes": [ - { - "collectionGroup": "event", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "body_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "event_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "event", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "body_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "event_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "event_minutes_item", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "index", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "event_minutes_item", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "index", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "event_minutes_item_file", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_minutes_item_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "name", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "datetime_weighted_value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "stemmed_gram", - "order": "ASCENDING" - }, - { - "fieldPath": "value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "stemmed_gram", - "order": "ASCENDING" - }, - { - "fieldPath": "datetime_weighted_value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "unstemmed_gram", - "order": "ASCENDING" - }, - { - "fieldPath": "value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "indexed_event_gram", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "unstemmed_gram", - "order": "ASCENDING" - }, - { - "fieldPath": "datetime_weighted_value", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "matter_file", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "name", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "matter_sponsor", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "matter_status", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "update_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "matter_status", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "update_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "start_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "start_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "end_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "end_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "seat_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "end_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "title", - "order": "ASCENDING" - }, - { - "fieldPath": "start_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "title", - "order": "ASCENDING" - }, - { - "fieldPath": "start_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "title", - "order": "ASCENDING" - }, - { - "fieldPath": "end_datetime", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "title", - "order": "ASCENDING" - }, - { - "fieldPath": "end_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "role", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "end_datetime", - "order": "ASCENDING" - }, - { - "fieldPath": "seat_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "start_datetime", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "session", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "session_index", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "transcript", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "session_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "created", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "transcript", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "session_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "confidence", - "order": "DESCENDING" - } - ] - }, - { - "collectionGroup": "vote", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "event_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "person_ref", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "vote", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "person_ref", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "vote", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "event_ref", - "order": "ASCENDING" - } - ] - }, - { - "collectionGroup": "vote", - "queryScope": "COLLECTION", - "fields": [ - { - "fieldPath": "person_ref", - "order": "ASCENDING" - }, - { - "fieldPath": "matter_ref", - "order": "ASCENDING" - } - ] - } - ], - "fieldOverrides": [] -} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5c821899..f848cd79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,8 +134,8 @@ process_cdp_event_index_chunk = "cdp_backend.bin.process_cdp_event_index_chunk:m search_cdp_events = "cdp_backend.bin.search_cdp_events:main" process_special_event = "cdp_backend.bin.process_special_event:main" add_content_hash_to_sessions = "cdp_backend.bin.add_content_hash_to_sessions:main" -create_cdp_firestore_indexes_json = "cdp_backend.bin.create_cdp_firestore_indexes_json:main" store_cdp_metadata_document = "cdp_backend.bin.store_cdp_metadata_document:main" +get_cdp_infrastructure_stack = "cdp_backend.bin.get_cdp_infrastructure_stack:main" # build settings # https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html @@ -147,7 +147,14 @@ include-package-data = true exclude = ["*docs/*", "*tests/*"] [tool.setuptools.package-data] -"*" = ["*.yaml", "py.typed", "*.csv"] +"*" = [ + "*.yaml", + "py.typed", + "*.csv", + "*infrastructure/*.rules", + "*infrastructure/Justfile", + "*infrastructure/*.json", +] # tools [tool.black] From ca1fc8de30368566dd2dbd3bdb1497a8ef102f81 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 11:44:57 -0700 Subject: [PATCH 09/16] Forgot to remove copied file --- Justfile | 1 + __init__.py | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 __init__.py diff --git a/Justfile b/Justfile index f6f3c4f7..4e2e1809 100644 --- a/Justfile +++ b/Justfile @@ -23,6 +23,7 @@ clean: rm -fr abc123-cdp_*-transcript.json rm -fr test.err rm -fr test.out + rm -fr *-thumbnail.* # install with all deps diff --git a/__init__.py b/__init__.py deleted file mode 100644 index d4a69b75..00000000 --- a/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -The files required for setting up an infrastructure stack. - -They are stored here in cdp-backend rather than the cookiecutter -so that all backend components are together and it makes it easy to -create dev-infrastructures for development. -""" - -from pathlib import Path - -INFRA_DIR = Path(__file__).parent \ No newline at end of file From 1de7f0e8b33f6865b04d865589a7e84e48c23aab Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 12:23:30 -0700 Subject: [PATCH 10/16] Remove extra comments in Justfile referencing pulumi --- cdp_backend/infrastructure/Justfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile index 98938f2c..829783e1 100644 --- a/cdp_backend/infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -8,9 +8,8 @@ USER := env_var("USER") # Default region for infrastructures default_region := "us-central1" default_key := clean(join(justfile_directory(), "../.keys/cdp-dev.json")) -export GOOGLE_APPLICATION_CREDENTIALS := default_key -# run gcloud and pulumi logins +# run gcloud login login: gcloud auth login gcloud auth application-default login @@ -36,8 +35,11 @@ gen-key project: echo "----------------------------------------------------------------------------" sleep 60 cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json + echo "----------------------------------------------------------------------------" + echo "Be sure to update the GOOGLE_APPLICATION_CREDENTIALS environment variable." + echo "----------------------------------------------------------------------------" -# create a new gcloud project and pulumi stack +# create a new gcloud project and generate a key init project: gcloud projects create {{project}} --set-as-default echo "----------------------------------------------------------------------------" From d3930c1930e0d1fbf1c1eddfb08cd8c05717b89d Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:25:00 -0700 Subject: [PATCH 11/16] Add governing body back and remove outdated func call in justfile --- cdp_backend/infrastructure/Justfile | 2 -- cdp_backend/infrastructure/__init__.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile index 829783e1..257e914f 100644 --- a/cdp_backend/infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -70,8 +70,6 @@ deploy project: just enable-services -firebase projects:addfirebase {{project}} firebase use --add {{project}} - create_cdp_firestore_indexes_json \ - {{join(justfile_directory(), "firestore.indexes.json")}} firebase deploy --only firestore:rules firebase deploy --only firestore:indexes firebase deploy --only storage diff --git a/cdp_backend/infrastructure/__init__.py b/cdp_backend/infrastructure/__init__.py index 88d4423d..389a7be0 100644 --- a/cdp_backend/infrastructure/__init__.py +++ b/cdp_backend/infrastructure/__init__.py @@ -11,3 +11,9 @@ from pathlib import Path INFRA_DIR = Path(__file__).parent + +class GoverningBody: + city_council = "city council" + county_council = "county council" + school_board = "school board" + other = "other" From 46c8ac52967c7d9a7be2c69e754bb9393e3101ec Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:28:17 -0700 Subject: [PATCH 12/16] Add pyyaml dep --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f848cd79..eb495f01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "fsspec", # Version pin set by gcsfs "gcsfs>=2022.7.1", "google-cloud-firestore", # Version pin set by fireo + "PyYAML>=5.4.1", "requests>=2.26.0", ] From 225643845953d71ddf5faec3c070496a0f5d2720 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:33:39 -0700 Subject: [PATCH 13/16] Parametrize cookiecutter path for deploy --- cdp_backend/infrastructure/Justfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile index 257e914f..7c013a96 100644 --- a/cdp_backend/infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -8,6 +8,7 @@ USER := env_var("USER") # Default region for infrastructures default_region := "us-central1" default_key := clean(join(justfile_directory(), "../.keys/cdp-dev.json")) +default_cookiecutter_yaml := join(justfile_directory(), "dev-cookiecutter-metadata.yaml") # run gcloud login login: @@ -66,16 +67,14 @@ setup region=default_region: gcloud alpha firestore databases create --region={{replace(region, "us-central1", "us-central")}} # deploy the CDP specific firestore and storage requirements -deploy project: +deploy project cookiecutter_yaml=default_cookiecutter_yaml: just enable-services -firebase projects:addfirebase {{project}} firebase use --add {{project}} firebase deploy --only firestore:rules firebase deploy --only firestore:indexes firebase deploy --only storage - store_cdp_metadata_document \ - {{default_key}} \ - {{join(justfile_directory(), "dev-cookiecutter-metadata.yaml")}} + store_cdp_metadata_document {{default_key}} {{cookiecutter_yaml}} gsutil cors set cors.json gs://{{project}}.appspot.com/ # run both setup and deploy From 5193d9ac93fd4cefdfb505575264e9551f44174e Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:37:51 -0700 Subject: [PATCH 14/16] Parametrize key to deploy --- cdp_backend/infrastructure/Justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile index 7c013a96..7edbe23f 100644 --- a/cdp_backend/infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -67,14 +67,14 @@ setup region=default_region: gcloud alpha firestore databases create --region={{replace(region, "us-central1", "us-central")}} # deploy the CDP specific firestore and storage requirements -deploy project cookiecutter_yaml=default_cookiecutter_yaml: +deploy project json_key=default_key cookiecutter_yaml=default_cookiecutter_yaml: just enable-services -firebase projects:addfirebase {{project}} firebase use --add {{project}} firebase deploy --only firestore:rules firebase deploy --only firestore:indexes firebase deploy --only storage - store_cdp_metadata_document {{default_key}} {{cookiecutter_yaml}} + store_cdp_metadata_document {{json_key}} {{cookiecutter_yaml}} gsutil cors set cors.json gs://{{project}}.appspot.com/ # run both setup and deploy From bda1572d2f9a7ff30161a08b93c8aaad4f4d9c06 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:45:24 -0700 Subject: [PATCH 15/16] Do not pass credentials JSON, infer from env --- cdp_backend/bin/store_cdp_metadata_document.py | 9 +-------- cdp_backend/infrastructure/Justfile | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/cdp_backend/bin/store_cdp_metadata_document.py b/cdp_backend/bin/store_cdp_metadata_document.py index e2740203..f3a9ff02 100644 --- a/cdp_backend/bin/store_cdp_metadata_document.py +++ b/cdp_backend/bin/store_cdp_metadata_document.py @@ -32,11 +32,6 @@ def __parse(self) -> None: prog="store_cdp_metadata_document", description="Store the CDP metadata document to a firestore instance.", ) - p.add_argument( - "google_credentials_file", - type=Path, - help="Path to Google service account JSON key.", - ) p.add_argument( "cookiecutter_yaml", type=Path, @@ -49,7 +44,6 @@ def __parse(self) -> None: def _store_cdp_metadata_document( - google_credentials_file: Path, cookiecutter_yaml: Path, ) -> None: # Read the cookiecutter file @@ -57,7 +51,7 @@ def _store_cdp_metadata_document( cookiecutter_meta = yaml.load(open_f, Loader=yaml.FullLoader)["default_context"] # Open client and write doc - client = Client.from_service_account_json(google_credentials_file) + client = Client() collection = client.collection("metadata") collection.document("configuration").set( { @@ -75,7 +69,6 @@ def main() -> None: try: args = Args() _store_cdp_metadata_document( - google_credentials_file=args.google_credentials_file, cookiecutter_yaml=args.cookiecutter_yaml, ) except Exception as e: diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile index 7edbe23f..89054570 100644 --- a/cdp_backend/infrastructure/Justfile +++ b/cdp_backend/infrastructure/Justfile @@ -67,14 +67,14 @@ setup region=default_region: gcloud alpha firestore databases create --region={{replace(region, "us-central1", "us-central")}} # deploy the CDP specific firestore and storage requirements -deploy project json_key=default_key cookiecutter_yaml=default_cookiecutter_yaml: +deploy project cookiecutter_yaml=default_cookiecutter_yaml: just enable-services -firebase projects:addfirebase {{project}} firebase use --add {{project}} firebase deploy --only firestore:rules firebase deploy --only firestore:indexes firebase deploy --only storage - store_cdp_metadata_document {{json_key}} {{cookiecutter_yaml}} + store_cdp_metadata_document {{cookiecutter_yaml}} gsutil cors set cors.json gs://{{project}}.appspot.com/ # run both setup and deploy From 3e151fe6877bbf8c14ee5d11392255930b7b22a1 Mon Sep 17 00:00:00 2001 From: Eva Maxfield Brown Date: Wed, 17 Aug 2022 13:45:47 -0700 Subject: [PATCH 16/16] Lint and format --- cdp_backend/infrastructure/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdp_backend/infrastructure/__init__.py b/cdp_backend/infrastructure/__init__.py index 389a7be0..2d3bd729 100644 --- a/cdp_backend/infrastructure/__init__.py +++ b/cdp_backend/infrastructure/__init__.py @@ -12,6 +12,7 @@ INFRA_DIR = Path(__file__).parent + class GoverningBody: city_council = "city council" county_council = "county council"