diff --git a/.editorconfig b/.editorconfig index 09084781..5d93397f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,6 @@ insert_final_newline = false [Makefile] indent_style = tab + +[Justfile] +indent_style = tab \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2660bf31..4b49f3e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install .[infrastructure,pipeline,test] + pip install .[pipeline,test] - name: Run Tests run: just test - name: Upload Codecov diff --git a/.gitignore b/.gitignore index ff9b0a39..081c8210 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,12 @@ index-chunks/ abc123-cdp* *-person-picture *-seat-image +dev-infrastructure/cors.json +dev-infrastructure/firebase.json +dev-infrastructure/firestore.indexes.json +dev-infrastructure/firestore.rules +dev-infrastructure/storage.rules +dev-infrastructure/Justfile # Common credentials directories .keys/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57e43824..e6d861e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,4 @@ repos: additional_dependencies: - "types-pytz" - "types-requests" + - "types-PyYAML" diff --git a/Justfile b/Justfile index 57da129c..4e2e1809 100644 --- a/Justfile +++ b/Justfile @@ -23,11 +23,12 @@ clean: rm -fr abc123-cdp_*-transcript.json rm -fr test.err rm -fr test.out + rm -fr *-thumbnail.* # install with all deps install: - pip install -e .[pipeline,infrastructure,lint,test,docs,dev] + pip install -e .[pipeline,lint,test,docs,dev] # lint, format, and check all files lint: diff --git a/cdp_backend/bin/get_cdp_infrastructure_stack.py b/cdp_backend/bin/get_cdp_infrastructure_stack.py new file mode 100644 index 00000000..5a61ab8f --- /dev/null +++ b/cdp_backend/bin/get_cdp_infrastructure_stack.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import json +import logging +import shutil +import sys +import traceback +from pathlib import Path + +from cdp_backend.database import DATABASE_MODELS +from cdp_backend.infrastructure import INFRA_DIR + +############################################################################### + +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", +) +log = logging.getLogger(__name__) + +############################################################################### + + +class Args(argparse.Namespace): + def __init__(self) -> None: + self.__parse() + + def __parse(self) -> None: + p = argparse.ArgumentParser( + prog="get_cdp_infrastructure_stack", + description=( + "Generate or copy all the files needed for a new CDP infrastructure." + ), + ) + p.add_argument( + "output_dir", + type=Path, + help=( + "Path to where the infrastructure files should be copied " + "or generated." + ), + ) + p.parse_args(namespace=self) + + +############################################################################### + + +def _generate_indexes_json(outfile: Path) -> None: + # All indexes + indexes = [] + + for model_cls in DATABASE_MODELS: + for idx_field_set in model_cls._INDEXES: + + indexes.append( + { + "collectionGroup": model_cls.collection_name, + "queryScope": "COLLECTION", + "fields": idx_field_set.to_dict()["fields"], + } + ) + + # Add indexes to the normal JSON format + indexes_full_json = { + "indexes": indexes, + "fieldOverrides": [], + } + + # Write out the file + outfile = outfile.resolve() + with open(outfile, "w") as open_f: + json.dump(indexes_full_json, open_f, indent=2) + log.info(f"Wrote out CDP firestore.indexes.json to: '{outfile}'") + + +def _copy_infra_files(output_dir: Path) -> None: + # Copy each file in the infra dir to the output dir + output_dir.mkdir(parents=True, exist_ok=True) + for f in INFRA_DIR.iterdir(): + if f.name not in [ + "__pycache__", + "__init__.py", + ]: + shutil.copy(f, output_dir / f.name) + + +def main() -> None: + try: + args = Args() + output_dir = args.output_dir.expanduser().resolve() + _copy_infra_files(output_dir=output_dir) + _generate_indexes_json(outfile=output_dir / "firestore.indexes.json") + except Exception as e: + log.error("=============================================") + log.error("\n\n" + traceback.format_exc()) + log.error("=============================================") + log.error("\n\n" + str(e) + "\n") + log.error("=============================================") + sys.exit(1) + + +############################################################################### +# Allow caller to directly run this module (usually in development scenarios) + +if __name__ == "__main__": + main() diff --git a/cdp_backend/bin/store_cdp_metadata_document.py b/cdp_backend/bin/store_cdp_metadata_document.py new file mode 100644 index 00000000..f3a9ff02 --- /dev/null +++ b/cdp_backend/bin/store_cdp_metadata_document.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse +import logging +import sys +import traceback +from pathlib import Path + +import yaml +from google.cloud.firestore import Client + +from .. import __version__ + +############################################################################### + +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", +) +log = logging.getLogger(__name__) + +############################################################################### + + +class Args(argparse.Namespace): + def __init__(self) -> None: + self.__parse() + + def __parse(self) -> None: + p = argparse.ArgumentParser( + prog="store_cdp_metadata_document", + description="Store the CDP metadata document to a firestore instance.", + ) + p.add_argument( + "cookiecutter_yaml", + type=Path, + help="Path to the CDP Cookiecutter YAML file to lookup metadata details.", + ) + p.parse_args(namespace=self) + + +############################################################################### + + +def _store_cdp_metadata_document( + cookiecutter_yaml: Path, +) -> None: + # Read the cookiecutter file + with open(cookiecutter_yaml, "r") as open_f: + cookiecutter_meta = yaml.load(open_f, Loader=yaml.FullLoader)["default_context"] + + # Open client and write doc + client = Client() + collection = client.collection("metadata") + collection.document("configuration").set( + { + "infrastructure_version": __version__, + "municipality_name": cookiecutter_meta["municipality"], + "hosting_github_url": cookiecutter_meta["hosting_github_url"], + "hosting_web_app_address": cookiecutter_meta["hosting_web_app_address"], + "firestore_location": cookiecutter_meta["firestore_region"], + "governing_body": cookiecutter_meta["governing_body_type"], + } + ) + + +def main() -> None: + try: + args = Args() + _store_cdp_metadata_document( + cookiecutter_yaml=args.cookiecutter_yaml, + ) + except Exception as e: + log.error("=============================================") + log.error("\n\n" + traceback.format_exc()) + log.error("=============================================") + log.error("\n\n" + str(e) + "\n") + log.error("=============================================") + sys.exit(1) + + +############################################################################### +# Allow caller to directly run this module (usually in development scenarios) + +if __name__ == "__main__": + main() diff --git a/cdp_backend/database/models.py b/cdp_backend/database/models.py index da219988..611526e3 100644 --- a/cdp_backend/database/models.py +++ b/cdp_backend/database/models.py @@ -216,67 +216,67 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="seat_ref", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="seat_ref", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="title", order=Order.ASCENDING), - IndexedField(name="end_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="title", order=Order.ASCENDING), + IndexedField(fieldPath="end_datetime", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="end_datetime", order=Order.ASCENDING), - IndexedField(name="seat_ref", order=Order.ASCENDING), - IndexedField(name="start_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="end_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="seat_ref", order=Order.ASCENDING), + IndexedField(fieldPath="start_datetime", order=Order.DESCENDING), ) ), ) @@ -358,8 +358,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="name", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="name", order=Order.ASCENDING), ) ), ) @@ -389,8 +389,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), ) ), ) @@ -469,14 +469,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="body_ref", order=Order.ASCENDING), - IndexedField(name="event_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="body_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_datetime", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="body_ref", order=Order.ASCENDING), - IndexedField(name="event_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="body_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_datetime", order=Order.DESCENDING), ) ), ) @@ -524,8 +524,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="session_index", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="session_index", order=Order.ASCENDING), ) ), ) @@ -560,14 +560,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="session_ref", order=Order.ASCENDING), - IndexedField(name="created", order=Order.DESCENDING), + IndexedField(fieldPath="session_ref", order=Order.ASCENDING), + IndexedField(fieldPath="created", order=Order.DESCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="session_ref", order=Order.ASCENDING), - IndexedField(name="confidence", order=Order.DESCENDING), + IndexedField(fieldPath="session_ref", order=Order.ASCENDING), + IndexedField(fieldPath="confidence", order=Order.DESCENDING), ) ), ) @@ -605,14 +605,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="index", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="index", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="index", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="index", order=Order.DESCENDING), ) ), ) @@ -657,14 +657,14 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="update_datetime", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="update_datetime", order=Order.ASCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="update_datetime", order=Order.DESCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="update_datetime", order=Order.DESCENDING), ), ), ) @@ -708,8 +708,8 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_minutes_item_ref", order=Order.ASCENDING), - IndexedField(name="name", order=Order.ASCENDING), + IndexedField(fieldPath="event_minutes_item_ref", order=Order.ASCENDING), + IndexedField(fieldPath="name", order=Order.ASCENDING), ) ), ) @@ -758,26 +758,26 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="matter_ref", order=Order.ASCENDING), - IndexedField(name="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), ) ), IndexedFieldSet( ( - IndexedField(name="person_ref", order=Order.ASCENDING), - IndexedField(name="matter_ref", order=Order.ASCENDING), + IndexedField(fieldPath="person_ref", order=Order.ASCENDING), + IndexedField(fieldPath="matter_ref", order=Order.ASCENDING), ) ), ) @@ -819,38 +819,44 @@ def Example(cls) -> Model: _INDEXES = ( IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="event_ref", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="event_ref", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), IndexedFieldSet( ( - IndexedField(name="stemmed_gram", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="stemmed_gram", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="stemmed_gram", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="stemmed_gram", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), IndexedFieldSet( ( - IndexedField(name="unstemmed_gram", order=Order.ASCENDING), - IndexedField(name="value", order=Order.DESCENDING), + IndexedField(fieldPath="unstemmed_gram", order=Order.ASCENDING), + IndexedField(fieldPath="value", order=Order.DESCENDING), ), ), IndexedFieldSet( ( - IndexedField(name="unstemmed_gram", order=Order.ASCENDING), - IndexedField(name="datetime_weighted_value", order=Order.DESCENDING), + IndexedField(fieldPath="unstemmed_gram", order=Order.ASCENDING), + IndexedField( + fieldPath="datetime_weighted_value", order=Order.DESCENDING + ), ), ), ) diff --git a/cdp_backend/database/types.py b/cdp_backend/database/types.py index 563eaefb..063a4fbf 100644 --- a/cdp_backend/database/types.py +++ b/cdp_backend/database/types.py @@ -1,15 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import NamedTuple, Tuple +from dataclasses import dataclass +from typing import List + +from dataclasses_json import DataClassJsonMixin ############################################################################### -class IndexedField(NamedTuple): - name: str +@dataclass +class IndexedField(DataClassJsonMixin): + fieldPath: str order: str -class IndexedFieldSet(NamedTuple): - fields: Tuple[IndexedField, ...] +@dataclass +class IndexedFieldSet(DataClassJsonMixin): + fields: List[IndexedField] diff --git a/cdp_backend/infrastructure/Justfile b/cdp_backend/infrastructure/Justfile new file mode 100644 index 00000000..89054570 --- /dev/null +++ b/cdp_backend/infrastructure/Justfile @@ -0,0 +1,98 @@ +# list all available commands +default: + just --list + +# get and store user +USER := env_var("USER") + +# Default region for infrastructures +default_region := "us-central1" +default_key := clean(join(justfile_directory(), "../.keys/cdp-dev.json")) +default_cookiecutter_yaml := join(justfile_directory(), "dev-cookiecutter-metadata.yaml") + +# run gcloud login +login: + gcloud auth login + gcloud auth application-default login + +# switch active gcloud project +switch-project project: + gcloud config set project {{project}} + +# generate a service account JSON +gen-key project: + mkdir -p ../.keys/ + rm -rf ../.keys/{{project}}.json + gcloud iam service-accounts create {{project}} \ + --description="CDP Dev Service Account for {{USER}}" \ + --display-name="{{project}}" + gcloud projects add-iam-policy-binding {{project}} \ + --member="serviceAccount:{{project}}@{{project}}.iam.gserviceaccount.com" \ + --role="roles/owner" + gcloud iam service-accounts keys create ../.keys/{{project}}.json \ + --iam-account "{{project}}@{{project}}.iam.gserviceaccount.com" + echo "----------------------------------------------------------------------------" + echo "Sleeping for one minute while resources set up" + echo "----------------------------------------------------------------------------" + sleep 60 + cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json + echo "----------------------------------------------------------------------------" + echo "Be sure to update the GOOGLE_APPLICATION_CREDENTIALS environment variable." + echo "----------------------------------------------------------------------------" + +# create a new gcloud project and generate a key +init project: + gcloud projects create {{project}} --set-as-default + echo "----------------------------------------------------------------------------" + echo "Follow the link to setup billing for the created GCloud account." + echo "https://console.cloud.google.com/billing/linkedaccount?project={{project}}" + echo "----------------------------------------------------------------------------" + just gen-key {{project}} + +# enable gcloud services +enable-services: + gcloud services enable cloudresourcemanager.googleapis.com + gcloud services enable speech.googleapis.com \ + appengine.googleapis.com \ + storage.googleapis.com \ + firebase.googleapis.com \ + firestore.googleapis.com \ + firebaserules.googleapis.com \ + firebasestorage.googleapis.com + +# setup the basic gcloud app and firestore connection +setup region=default_region: + just enable-services + gcloud app create --region={{replace(region, "us-central1", "us-central")}} + gcloud alpha firestore databases create --region={{replace(region, "us-central1", "us-central")}} + +# deploy the CDP specific firestore and storage requirements +deploy project cookiecutter_yaml=default_cookiecutter_yaml: + just enable-services + -firebase projects:addfirebase {{project}} + firebase use --add {{project}} + firebase deploy --only firestore:rules + firebase deploy --only firestore:indexes + firebase deploy --only storage + store_cdp_metadata_document {{cookiecutter_yaml}} + gsutil cors set cors.json gs://{{project}}.appspot.com/ + +# run both setup and deploy +setup-and-deploy project region=default_region: + just setup {{region}} + just deploy {{project}} + +# fully teardown project +destroy project: + gcloud projects delete {{project}} + rm -f ../.keys/{{project}}.json + +# remove all database documents (except search index) and filestore objects +clean key=default_key: + clean_cdp_database {{key}} + clean_cdp_filestore {{key}} + +# remove all database documents (including search index) and filestore objects +clean-full key=default_key: + clean_cdp_database {{key}} --clean-index + clean_cdp_filestore {{key}} \ No newline at end of file diff --git a/cdp_backend/infrastructure/__init__.py b/cdp_backend/infrastructure/__init__.py index 5d73d761..2d3bd729 100644 --- a/cdp_backend/infrastructure/__init__.py +++ b/cdp_backend/infrastructure/__init__.py @@ -1,6 +1,20 @@ # -*- coding: utf-8 -*- -"""Infrastructure package for cdp_backend.""" +""" +The files required for setting up an infrastructure stack. +They are stored here in cdp-backend rather than the cookiecutter +so that all backend components are together and it makes it easy to +create dev-infrastructures for development. +""" -from .cdp_stack import CDPStack # noqa: F401 +from pathlib import Path + +INFRA_DIR = Path(__file__).parent + + +class GoverningBody: + city_council = "city council" + county_council = "county council" + school_board = "school board" + other = "other" diff --git a/cdp_backend/infrastructure/cdp_stack.py b/cdp_backend/infrastructure/cdp_stack.py deleted file mode 100644 index 412566f4..00000000 --- a/cdp_backend/infrastructure/cdp_stack.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import json -from typing import List - -import pulumi -import pulumi_gcp as gcp -from pulumi_google_native.firestore import v1 as firestore - -from ..database import DATABASE_MODELS -from ..version import __version__ - -############################################################################### - - -class GoverningBody: - city_council = "city council" - county_council = "county council" - school_board = "school board" - other = "other" - - -class CDPStack(pulumi.ComponentResource): - def __init__( - self, - gcp_project_id: str, - municipality_name: str = "dev_municipality", - hosting_github_url: str = "no_github_host", - hosting_web_app_address: str = "no_web_app_address", - firestore_location: str = "us-west2", - governing_body: str = GoverningBody.other, - opts: pulumi.ResourceOptions = None, - ): - """ - Creates all required infrastructure and enables all required services for CDP - backend stacks. - - Parameters - ---------- - gcp_project_id: str - The id of the gcp_project, the Pulumi stack, and any other required - names for resources created during infrastructure creation will use this id - as a prefix. - I.E. `cdp-seattle` would create a Cloud Firestore instance called - `cdp-seattle`, a GCP bucket called `cdp-seattle`, etc. - - municipality_name: str - The name of the municipality this instance will be for. - I.E. "Seattle", "King County", etc. - - hosting_github_url: str - The GitHub repo this instance configuration details will be stored. - I.E. https://github.com/councildataproject/seattle-staging - - hosting_web_app_address: str - The web address for this instances web portal. - I.E. https://councildataproject.org/seattle-staging - - firestore_location: str - The location for the Cloud Firestore database and file storage servers - to be hosted. - List of locations: https://firebase.google.com/docs/firestore/locations - Default: "us-west2" - - governing_body: str - What governing body this instance will archive. - Default: GoverningBody.other - - opts: pulumi.ResourceOptions - Extra resource options to initialize the entire stack with. - Default: None - - Notes - ----- - When using this resource it is recommended to run set Pulumi parallel resource - creation to five (5) max. GCP has limits on how many resources you can create - in parallel. - - The default values for many parameters are set to fake values as this object - should primarily be used with the cookiecutter / automated scripts. The default - values in this case are set because when using this object outside of - the cookiecutter it is likely for dev infrastructures. - """ - super().__init__("CDPStack", gcp_project_id, None, opts) - - # Store parameters - self.gcp_project_id = gcp_project_id - self.firestore_location = firestore_location - - # Enable all required services - self.cloudresourcemanager = gcp.projects.Service( - f"{self.gcp_project_id}-cloudresourcemanager-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="cloudresourcemanager.googleapis.com", - opts=pulumi.ResourceOptions(parent=self), - ) - self.speech_service = gcp.projects.Service( - f"{self.gcp_project_id}-speech-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="speech.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firebase_service = gcp.projects.Service( - f"{self.gcp_project_id}-firebase-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firebase.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.app_engine_service = gcp.projects.Service( - f"{self.gcp_project_id}-app-engine-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="appengine.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firestore_service = gcp.projects.Service( - f"{self.gcp_project_id}-firestore-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firestore.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - self.firebase_rules_service = gcp.projects.Service( - f"{self.gcp_project_id}-firebase-rules-service", - disable_dependent_services=True, - project=self.gcp_project_id, - service="firebaserules.googleapis.com", - opts=pulumi.ResourceOptions( - parent=self, depends_on=[self.cloudresourcemanager] - ), - ) - - # Create the firestore application - self.firestore_app = gcp.appengine.Application( - f"{self.gcp_project_id}-firestore-app", - project=self.gcp_project_id, - location_id=self.firestore_location, - database_type="CLOUD_FIRESTORE", - opts=pulumi.ResourceOptions( - parent=self, - depends_on=[ - self.firebase_service, - self.app_engine_service, - self.firestore_service, - self.firebase_rules_service, - ], - ), - ) - - # Init firebase project - self.firebase_init = gcp.firebase.Project( - resource_name=f"{self.gcp_project_id}-firebase-init", - project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self, depends_on=[self.firestore_app]), - ) - - # Connect app engine (firestore) + bucket - self.firebase_project = gcp.firebase.ProjectLocation( - resource_name=f"{self.gcp_project_id}-firebase-project", - project=self.gcp_project_id, - location_id=self.firestore_location, - opts=pulumi.ResourceOptions(parent=self.firebase_init), - ) - - # Set full public read on bucket - self.storage_public_read = gcp.storage.DefaultObjectAccessControl( - f"{self.gcp_project_id}-storage-acl-public-viewer", - bucket=self.firestore_app.default_bucket, - entity="allUsers", - role="READER", - opts=pulumi.ResourceOptions(parent=self.firestore_app), - ) - - # Create all firestore indexes - prior_index = None - for model_cls in DATABASE_MODELS: - for idx_field_set in model_cls._INDEXES: - - # Add fields to field list - idx_set_name_parts = [] - idx_set_fields = [] - for idx_field in idx_field_set.fields: - idx_set_name_parts += [idx_field.name, idx_field.order] - idx_set_fields.append( - firestore.GoogleFirestoreAdminV1IndexFieldArgs( - field_path=idx_field.name, - order=idx_field.order, - ) - ) - - # Finish creating the index set name - idx_set_name = "_".join(idx_set_name_parts) - fq_idx_set_name = f"{model_cls.collection_name}-{idx_set_name}" - - # Create depends on list - # We don't want to create a ton of indexes in parallel - if prior_index is None: - depends_on: List[pulumi.Resource] = [] - else: - depends_on = [prior_index] - - # Create - prior_index = firestore.Index( - fq_idx_set_name, - project=self.gcp_project_id, - database_id="(default)", - collection_group_id=model_cls.collection_name, - fields=idx_set_fields, - query_scope="COLLECTION", - opts=pulumi.ResourceOptions( - parent=self.firestore_app, - depends_on=depends_on, - ), - ) - - # Add metadata document - gcp.firestore.Document( - f"{self.gcp_project_id}-metadata-doc", - collection="metadata", - document_id="configuration", - fields=json.dumps( - { - "infrastructure_version": {"stringValue": __version__}, - "municipality_name": {"stringValue": municipality_name}, - "hosting_github_url": {"stringValue": hosting_github_url}, - "hosting_web_app_address": {"stringValue": hosting_web_app_address}, - "firestore_location": {"stringValue": firestore_location}, - "governing_body": {"stringValue": governing_body}, - } - ), - project=self.gcp_project_id, - opts=pulumi.ResourceOptions(parent=self.firestore_app), - ) - - super().register_outputs({}) diff --git a/dev-infrastructure/cors.json b/cdp_backend/infrastructure/cors.json similarity index 100% rename from dev-infrastructure/cors.json rename to cdp_backend/infrastructure/cors.json diff --git a/cdp_backend/infrastructure/firebase.json b/cdp_backend/infrastructure/firebase.json new file mode 100644 index 00000000..4110c3b8 --- /dev/null +++ b/cdp_backend/infrastructure/firebase.json @@ -0,0 +1,12 @@ +{ + "firestore": { + "rules": "firestore.rules", + "indexes": "firestore.indexes.json" + }, + "storage": { + "rules": "storage.rules" + }, + "functions": { + "source": "functions" + } +} diff --git a/cdp_backend/infrastructure/firestore.rules b/cdp_backend/infrastructure/firestore.rules new file mode 100644 index 00000000..db967a33 --- /dev/null +++ b/cdp_backend/infrastructure/firestore.rules @@ -0,0 +1,7 @@ +service cloud.firestore { + match /databases/{database}/documents { + match /{document=**} { + allow read; + } + } +} \ No newline at end of file diff --git a/cdp_backend/infrastructure/storage.rules b/cdp_backend/infrastructure/storage.rules new file mode 100644 index 00000000..4eda34fd --- /dev/null +++ b/cdp_backend/infrastructure/storage.rules @@ -0,0 +1,8 @@ +rules_version = '2'; +service firebase.storage { + match /b/{bucket}/o { + match /{allPaths=**} { + allow read, write: if request.auth!=null; + } + } +} diff --git a/cdp_backend/tests/database/test_models.py b/cdp_backend/tests/database/test_models.py index 8f7f1016..e7b2b750 100644 --- a/cdp_backend/tests/database/test_models.py +++ b/cdp_backend/tests/database/test_models.py @@ -34,16 +34,11 @@ def test_validate_model_definitions() -> None: # Check that all index fields are valid attributes of the model for idx_field_set in model_cls._INDEXES: for idx_field in idx_field_set.fields: - assert hasattr(m, idx_field.name) + assert hasattr(m, idx_field.fieldPath) - # Check that all primary keys are valid attributes of the model - for pk in model_cls._PRIMARY_KEYS: - assert hasattr(m, pk) - - # Check that all index fields are valid attributes of the model - for idx_field_set in model_cls._INDEXES: - for idx_field in idx_field_set.fields: - assert hasattr(m, idx_field.name) + # Check that all primary keys are valid attributes of the model + for pk in model_cls._PRIMARY_KEYS: + assert hasattr(m, pk) def test_cdp_database_model_has_no_cyclic_dependencies(tmpdir: Path) -> None: diff --git a/cdp_backend/tests/infrastructure/__init__.py b/cdp_backend/tests/infrastructure/__init__.py deleted file mode 100644 index c5b99e2c..00000000 --- a/cdp_backend/tests/infrastructure/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Unit test package for cdp_backend.infrastructure""" - -# Import to avoid breaking pulumi test running on asynchronous loop -import prefect # noqa: F401 diff --git a/cdp_backend/tests/infrastructure/test_cdp_stack.py b/cdp_backend/tests/infrastructure/test_cdp_stack.py deleted file mode 100644 index 3d297342..00000000 --- a/cdp_backend/tests/infrastructure/test_cdp_stack.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import unittest -from typing import Any, Dict, List, Optional, Tuple - -import pulumi - -############################################################################### - - -# Create the various mocks. -class CDPStackMocks(pulumi.runtime.Mocks): - def call( - self, args: pulumi.runtime.MockCallArgs - ) -> Tuple[Dict[Any, Any], Optional[List[Tuple[str, str]]]]: - return {}, None - - def new_resource( - self, args: pulumi.runtime.MockResourceArgs - ) -> Tuple[Optional[str], Dict[Any, Any]]: - return (args.name + "_id", args.inputs) - - -pulumi.runtime.set_mocks(CDPStackMocks()) - -from cdp_backend.infrastructure import CDPStack - -############################################################################### - - -class InfrastructureTests(unittest.TestCase): - @pulumi.runtime.test - def test_basic_run(self) -> None: - gcp_project_id = "mocked-testing-stack" - - # Write output checks - def check_firestore_app_id(args: List[Any]) -> None: - app_id = args - assert app_id == f"{gcp_project_id}.fake-appspot.io" - - def check_firestore_default_bucket(args: List[Any]) -> None: - default_bucket = args - assert default_bucket == f"gcs://{gcp_project_id}" - - # Init mocked stack - stack = CDPStack("mocked-testing-stack") - - # Check outputs - pulumi.Output.all(stack.firestore_app.app_id).apply(check_firestore_app_id) - pulumi.Output.all(stack.firestore_app.default_bucket).apply( - check_firestore_default_bucket - ) diff --git a/dev-infrastructure/Justfile b/dev-infrastructure/Justfile deleted file mode 100644 index 5074f829..00000000 --- a/dev-infrastructure/Justfile +++ /dev/null @@ -1,75 +0,0 @@ -# list all available commands -default: - just --list - -# get and store user -USER := env_var("USER") - -# run gcloud and pulumi logins -login: - pulumi logout - pulumi login - gcloud auth login - gcloud auth application-default login - -# create a new gcloud project and pulumi stack -init project: - gcloud projects create {{project}} --set-as-default - pulumi stack init {{project}} - echo "----------------------------------------------------------------------------" - echo "Follow the link to setup billing for the created GCloud account." - echo "https://console.cloud.google.com/billing/linkedaccount?project={{project}}" - -# set a CORS policy for the bucket -set-cors project: - gsutil cors set cors.json gs://{{project}}.appspot.com/ - -# run pulumi up for infra deploy -build: - pulumi up -p 4 - -# switch active gcloud project -switch-project project: - gcloud config set project {{project}} - -# remove all database documents (except search index) and filestore objects -clean key: - clean_cdp_database {{key}} - clean_cdp_filestore {{key}} - -# remove all database documents (including search index) and filestore objects -clean-full key: - clean_cdp_database {{key}} --clean-index - clean_cdp_filestore {{key}} - -# destroy just the stack (not the gcloud project) and rebuild -reset: - pulumi destroy -p 4 - echo "----------------------------------------------------------------------------" - echo "Sleeping for three minutes while resources clean up" - sleep 180 - make build - -# fully teardown project -destroy project: - pulumi stack rm {{project}} --force - gcloud projects delete {{project}} - rm -f ../.keys/{{project}}.json - -# generate a service account JSON -gen-key project: - mkdir ../.keys/ -p - rm -rf ../.keys/{{project}}.json - gcloud iam service-accounts create {{project}} \ - --description="CDP Dev Service Account for {{USER}}" \ - --display-name="{{project}}" - gcloud projects add-iam-policy-binding {{project}} \ - --member="serviceAccount:{{project}}@{{project}}.iam.gserviceaccount.com" \ - --role="roles/owner" - gcloud iam service-accounts keys create ../.keys/{{project}}.json \ - --iam-account "{{project}}@{{project}}.iam.gserviceaccount.com" - echo "----------------------------------------------------------------------------" - echo "Sleeping for one minute while resources set up" - sleep 60 - cp -rf ../.keys/{{project}}.json ../.keys/cdp-dev.json - echo "Key generation complete, updated ../.keys/cdp-dev.json to {{project}} key. Be sure to update GOOGLE_CREDENTIALS." \ No newline at end of file diff --git a/dev-infrastructure/Pulumi.yaml b/dev-infrastructure/Pulumi.yaml deleted file mode 100644 index 3a72b5f6..00000000 --- a/dev-infrastructure/Pulumi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: cdp-infrastructure -description: Dev CDP Pulumi stack -runtime: python diff --git a/dev-infrastructure/README.md b/dev-infrastructure/README.md index 0a1de564..b714c32e 100644 --- a/dev-infrastructure/README.md +++ b/dev-infrastructure/README.md @@ -27,10 +27,6 @@ found in our Deploying the CDP infrastructure requires having `cdp-backend` installed. -```bash -pip install -e ../[dev] -``` - For more detailed information please see the [project installation details](https://github.com/CouncilDataProject/cdp-backend#installation). @@ -40,138 +36,58 @@ For more detailed information please see the ([Google Cloud Console Home](https://console.cloud.google.com/)) 2. Create (or re-use) a [billing account](https://console.cloud.google.com/billing) and attach it to your GCP account. -3. Create (or sign in to) a - [Pulumi account](https://app.pulumi.com/signup). ## Environment Setup -The only environment setup needed to run this deployment is to make sure `pulumi` itself -and the `gcloud` SDK are both installed. - -- [pulumi](https://www.pulumi.com/docs/get-started/install/) -- [gcloud](https://cloud.google.com/sdk/install) +The only environment setup needed to run this deployment is to make +sure the [`gcloud` SDK](https://cloud.google.com/sdk/install) is installed. _If this was the first time installing either of those packages, it is recommended to restart your terminal after installation._ -After `pulumi` and `gcloud` have both been installed and terminal restarted, run the -following commands to setup your local machine with credentials to both services. +After `gcloud` has both been installed and terminal restarted, run the +following commands to log in to gcloud: `just login` + +## Create a New Project and Deploy the Infrastructure ```bash -cd cdp-backend/dev-infrastructure -make login -make init project={project-name} -make gen-key project={project-name} -make build +get_cdp_infrastructure_stack {OPTIONAL: dir path to store to} +just init {project-name} +just setup-and-deploy {project-name} {OPTIONAL: region} ``` -After generating the key, name your `key` file in `cdp-backend/.keys` to `cdp-dev.json`. In case you have many keys, note that by default, the random and minimal event pipelines use the key named `cdp-dev.json`. - -## Infrastructure Management Commands - -All of these commands should be run from within the `cdp-backend/dev-infrastructure` directory. - -- To log in to GCloud and Pulumi: - - ```bash - make login - ``` - -- To create a new service account JSON key: - - ```bash - make gen-key project={project-name} - ``` - -- To create a new dev infrastructure: - - ```bash - make init project={project-name} - ``` - - And follow the link logged to link a billing account to the created project. - - **Note:** This will create a new GCP project. - -- To set up infrastructure: - - ```bash - make build - ``` - - **Note:** You should run `make gen-key` prior to build and ensure you have - `GOOGLE_CREDENTIALS` set in your environment variables using: +Example: - ```bash - export GOOGLE_CREDENTIALS=$(cat ../.keys/{project-name}-sa-dev.json) - ``` +_Assuming user is within the `dev-infrastructure` dir._ - and replacing `{project-name}` with your project name. - - or if you have already renamed your key: - - ```bash - export GOOGLE_CREDENTIALS=$(cat ../.keys/cdp-dev.json) - ``` - -- To clean and remove all database documents and file store objects: - - ```bash - make clean key={path-to-key} - ``` - - **Note:** Cleaning infrastructure is best practice when comparing pipeline - outputs and database models aren't changing (specifically database indices). - -- To reset infrastructure but reuse the same Google project: - - ```bash - make reset - ``` - - **Note:** Reseting infrastructure is likely required when iterating on - database models (specifically database indices). Cleaning infrastructure - should always be attempted first before reset or destroy as `make clean` - will not use any extra Google Cloud (or Firebase) projects and applications. +```bash +get_cdp_infrastructure_stack . +just init cdp-eva-dev-001 +just setup-and-deploy cdp-eva-dev-001 +``` -- To delete all Pulumi and GCloud resources entirely: +## Update an Existing Project - ```bash - make destroy project={project-name} - ``` +```bash +just deploy {project-name} +``` - **Note:** This will delete the GCP project. +Example: -Try to use the same project and infrastructure as much as possible, there are -limits for how many projects and Firestore applications a single user can have. +```bash +just deploy cdp-eva-dev-001 +``` ### All Commands -- See Makefile commands with `make help`. - Or simply open the Makefile. All the commands are decently easy to follow. -- See Pulumi [CLI documentation](https://www.pulumi.com/docs/reference/cli/) - for all Pulumi commands. - -## Non-Default Parameters - -If you want to edit the default behavior of the `__main__.py` file and change the -parameters, please see the documentation for the -[CDPStack object](https://councildataproject.github.io/cdp-backend/cdp_backend.infrastructure.html#module-cdp_backend.infrastructure.cdp_stack) - -## Running Pipelines Against Dev Infra - -Once you have a dev infrastructure set up and a key downloaded (`make gen-key`) -you can run pipelines and store data in the infrastructure by moving up to the -base directory of this repository and running the following from `cdp-backend/`: +- See Justfile commands with `just` or open the Justfile. -- To run a semi-random (large permutation) event pipeline: - ```bash - make run-rand-event-pipeline - ``` +### Changing the Stack -- To run a minimal (by definition) event pipeline: +The actual infrastructure files live in the `cdp_backend/infrastructure` module. +To make changes to the infrastructure stack, change the files in that module and then +rerun `get-cdp-infrastructure-stack`. - ```bash - make run-min-event-pipeline - ``` +Note: the database indexes are store in the `cdp_backend/database/models.py` module +with each collection model. \ No newline at end of file diff --git a/dev-infrastructure/__main__.py b/dev-infrastructure/__main__.py deleted file mode 100644 index f100a2ba..00000000 --- a/dev-infrastructure/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" -This file is just used for dev setup and CI stack preview. -It is recommended to use the cookiecutter-cdp-deployment as it -""" - -from pulumi import export, get_stack - -from cdp_backend.infrastructure import CDPStack - -############################################################################### - -cdp_stack = CDPStack(get_stack()) - -export("firestore_address", cdp_stack.firestore_app.app_id) -export("gcp_bucket_name", cdp_stack.firestore_app.default_bucket) diff --git a/dev-infrastructure/dev-cookiecutter-metadata.yaml b/dev-infrastructure/dev-cookiecutter-metadata.yaml new file mode 100644 index 00000000..36e2591a --- /dev/null +++ b/dev-infrastructure/dev-cookiecutter-metadata.yaml @@ -0,0 +1,13 @@ +default_context: + municipality: "CDP Dev Muni" + iana_timezone: "America/Los_Angeles" + governing_body_type: "other" + municipality_slug: "cdp-dev-muni" + python_municipality_slug: "cdp_dev_muni" + infrastructure_slug: "cdp_dev_muni_000001" + maintainer_or_org_full_name: "evamaxfield" + hosting_github_username_or_org: "CouncilDataProject" + hosting_github_repo_name: "cdp-dev-muni" + hosting_github_url: "https://github.com/CouncilDataProject/cdp-dev-muni" + hosting_web_app_address: "https://councildataproject.github.io/cdp-dev-muni" + firestore_region: "us-central" diff --git a/dev-infrastructure/requirements.txt b/dev-infrastructure/requirements.txt deleted file mode 100644 index f3229c5b..00000000 --- a/dev-infrastructure/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -.. diff --git a/pyproject.toml b/pyproject.toml index 46e7512e..eb495f01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,8 @@ dependencies = [ "fireo>=1.5", "fsspec", # Version pin set by gcsfs "gcsfs>=2022.7.1", + "google-cloud-firestore", # Version pin set by fireo + "PyYAML>=5.4.1", "requests>=2.26.0", ] @@ -52,11 +54,6 @@ Documentation = "https://CouncilDataProject.github.io/cdp-backend" # extra dependencies # https://peps.python.org/pep-0621/#dependencies-optional-dependencies [project.optional-dependencies] -infrastructure = [ - "pulumi~=3.31", - "pulumi-google-native~=0.18", - "pulumi-gcp~=6.0", -] pipeline = [ "dask[distributed]>=2021.7.0", "ffmpeg-python==0.2.0", @@ -91,8 +88,9 @@ lint = [ "isort>=5.7.0", "mypy>=0.790", "pre-commit>=2.20.0", - "types-pytz>=2022.1.2", "types-requests>=2.28.5", + "types-pytz>=2022.1.2", + "types-PyYAML>=6.0.11", ] test = [ # Pytest @@ -137,6 +135,8 @@ process_cdp_event_index_chunk = "cdp_backend.bin.process_cdp_event_index_chunk:m search_cdp_events = "cdp_backend.bin.search_cdp_events:main" process_special_event = "cdp_backend.bin.process_special_event:main" add_content_hash_to_sessions = "cdp_backend.bin.add_content_hash_to_sessions:main" +store_cdp_metadata_document = "cdp_backend.bin.store_cdp_metadata_document:main" +get_cdp_infrastructure_stack = "cdp_backend.bin.get_cdp_infrastructure_stack:main" # build settings # https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html @@ -148,7 +148,14 @@ include-package-data = true exclude = ["*docs/*", "*tests/*"] [tool.setuptools.package-data] -"*" = ["*.yaml", "py.typed", "*.csv"] +"*" = [ + "*.yaml", + "py.typed", + "*.csv", + "*infrastructure/*.rules", + "*infrastructure/Justfile", + "*infrastructure/*.json", +] # tools [tool.black]