bugfix/infra-stack (#202)

* Minor dev-infra README updates

* Start on stack debugging

* Remove pulumi

* Everything but indexes

* Create the indexes and the metadata doc in Python

* Remove infra requirements

* Lint, format, fix tests

* Function to copy stored infrastructure files to dir

* Forgot to remove copied file

* Remove extra comments in Justfile referencing pulumi

* Add governing body back and remove outdated func call in justfile

* Add pyyaml dep

* Parametrize cookiecutter path for deploy

* Parametrize key to deploy

* Do not pass credentials JSON, infer from env

* Lint and format

.editorconfig

@@ -22,3 +22,6 @@ insert_final_newline = false
 
 [Makefile]
 indent_style = tab
+
+[Justfile]
+indent_style = tab

.github/workflows/ci.yml

@@ -68,7 +68,7 @@ jobs:
     - name: Install Dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .[infrastructure,pipeline,test]
+        pip install .[pipeline,test]
     - name: Run Tests
       run: just test
     - name: Upload Codecov

.gitignore

@@ -130,6 +130,12 @@ index-chunks/
 abc123-cdp*
 *-person-picture
 *-seat-image
+dev-infrastructure/cors.json
+dev-infrastructure/firebase.json
+dev-infrastructure/firestore.indexes.json
+dev-infrastructure/firestore.rules
+dev-infrastructure/storage.rules
+dev-infrastructure/Justfile
 
 # Common credentials directories
 .keys/

.pre-commit-config.yaml

@@ -32,3 +32,4 @@ repos:
         additional_dependencies:
               - "types-pytz"
               - "types-requests"
+              - "types-PyYAML"

Justfile

@@ -23,11 +23,12 @@ clean:
 	rm -fr abc123-cdp_*-transcript.json
 	rm -fr test.err
 	rm -fr test.out
+	rm -fr *-thumbnail.*
 
 
 # install with all deps
 install:
-	pip install -e .[pipeline,infrastructure,lint,test,docs,dev]
+	pip install -e .[pipeline,lint,test,docs,dev]
 
 # lint, format, and check all files
 lint:

cdp_backend/bin/get_cdp_infrastructure_stack.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+import json
+import logging
+import shutil
+import sys
+import traceback
+from pathlib import Path
+
+from cdp_backend.database import DATABASE_MODELS
+from cdp_backend.infrastructure import INFRA_DIR
+
+###############################################################################
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s",
+)
+log = logging.getLogger(__name__)
+
+###############################################################################
+
+
+class Args(argparse.Namespace):
+    def __init__(self) -> None:
+        self.__parse()
+
+    def __parse(self) -> None:
+        p = argparse.ArgumentParser(
+            prog="get_cdp_infrastructure_stack",
+            description=(
+                "Generate or copy all the files needed for a new CDP infrastructure."
+            ),
+        )
+        p.add_argument(
+            "output_dir",
+            type=Path,
+            help=(
+                "Path to where the infrastructure files should be copied "
+                "or generated."
+            ),
+        )
+        p.parse_args(namespace=self)
+
+
+###############################################################################
+
+
+def _generate_indexes_json(outfile: Path) -> None:
+    # All indexes
+    indexes = []
+
+    for model_cls in DATABASE_MODELS:
+        for idx_field_set in model_cls._INDEXES:
+
+            indexes.append(
+                {
+                    "collectionGroup": model_cls.collection_name,
+                    "queryScope": "COLLECTION",
+                    "fields": idx_field_set.to_dict()["fields"],
+                }
+            )
+
+    # Add indexes to the normal JSON format
+    indexes_full_json = {
+        "indexes": indexes,
+        "fieldOverrides": [],
+    }
+
+    # Write out the file
+    outfile = outfile.resolve()
+    with open(outfile, "w") as open_f:
+        json.dump(indexes_full_json, open_f, indent=2)
+    log.info(f"Wrote out CDP firestore.indexes.json to: '{outfile}'")
+
+
+def _copy_infra_files(output_dir: Path) -> None:
+    # Copy each file in the infra dir to the output dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    for f in INFRA_DIR.iterdir():
+        if f.name not in [
+            "__pycache__",
+            "__init__.py",
+        ]:
+            shutil.copy(f, output_dir / f.name)
+
+
+def main() -> None:
+    try:
+        args = Args()
+        output_dir = args.output_dir.expanduser().resolve()
+        _copy_infra_files(output_dir=output_dir)
+        _generate_indexes_json(outfile=output_dir / "firestore.indexes.json")
+    except Exception as e:
+        log.error("=============================================")
+        log.error("\n\n" + traceback.format_exc())
+        log.error("=============================================")
+        log.error("\n\n" + str(e) + "\n")
+        log.error("=============================================")
+        sys.exit(1)
+
+
+###############################################################################
+# Allow caller to directly run this module (usually in development scenarios)
+
+if __name__ == "__main__":
+    main()

cdp_backend/bin/store_cdp_metadata_document.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+import logging
+import sys
+import traceback
+from pathlib import Path
+
+import yaml
+from google.cloud.firestore import Client
+
+from .. import __version__
+
+###############################################################################
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s",
+)
+log = logging.getLogger(__name__)
+
+###############################################################################
+
+
+class Args(argparse.Namespace):
+    def __init__(self) -> None:
+        self.__parse()
+
+    def __parse(self) -> None:
+        p = argparse.ArgumentParser(
+            prog="store_cdp_metadata_document",
+            description="Store the CDP metadata document to a firestore instance.",
+        )
+        p.add_argument(
+            "cookiecutter_yaml",
+            type=Path,
+            help="Path to the CDP Cookiecutter YAML file to lookup metadata details.",
+        )
+        p.parse_args(namespace=self)
+
+
+###############################################################################
+
+
+def _store_cdp_metadata_document(
+    cookiecutter_yaml: Path,
+) -> None:
+    # Read the cookiecutter file
+    with open(cookiecutter_yaml, "r") as open_f:
+        cookiecutter_meta = yaml.load(open_f, Loader=yaml.FullLoader)["default_context"]
+
+    # Open client and write doc
+    client = Client()
+    collection = client.collection("metadata")
+    collection.document("configuration").set(
+        {
+            "infrastructure_version": __version__,
+            "municipality_name": cookiecutter_meta["municipality"],
+            "hosting_github_url": cookiecutter_meta["hosting_github_url"],
+            "hosting_web_app_address": cookiecutter_meta["hosting_web_app_address"],
+            "firestore_location": cookiecutter_meta["firestore_region"],
+            "governing_body": cookiecutter_meta["governing_body_type"],
+        }
+    )
+
+
+def main() -> None:
+    try:
+        args = Args()
+        _store_cdp_metadata_document(
+            cookiecutter_yaml=args.cookiecutter_yaml,
+        )
+    except Exception as e:
+        log.error("=============================================")
+        log.error("\n\n" + traceback.format_exc())
+        log.error("=============================================")
+        log.error("\n\n" + str(e) + "\n")
+        log.error("=============================================")
+        sys.exit(1)
+
+
+###############################################################################
+# Allow caller to directly run this module (usually in development scenarios)
+
+if __name__ == "__main__":
+    main()