bugfix/infra-stack (#111)

* Remove pulumi, use new infra stack, update docs

* Minor fixes

* More dependabot configs

.github/dependabot.yml

@@ -0,0 +1,26 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "ci(dependabot):"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "cdp-backend"
+    commit-message:
+      prefix: "ci(dependabot):"
+
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "@councildataproject/cdp-frontend"
+    commit-message:
+      prefix: "ci(dependabot):"

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: Setup Node
       uses: actions/setup-node@v1
       with:
@@ -59,11 +59,7 @@ jobs:
         npm run build
 
     # Publish the generated repo
-    - name: Publish Generated Repo
-      uses: JamesIves/github-pages-deploy-action@releases/v3
+    - name: Publish Docs
+      uses: JamesIves/github-pages-deploy-action@v4
       with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        BASE_BRANCH: main # The branch the action should deploy from.
-        BRANCH: 'example-build' # The branch the action should deploy to.
-        FOLDER: example/ # The folder the action should deploy
-        CLEAN: true
+        folder: example/

.github/workflows/deployment-management-bot.yml

@@ -67,7 +67,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - name: Install Bot Scripts Dependencies
         run: |
           pip install --upgrade pip
@@ -233,7 +233,6 @@ jobs:
       - name: Remove SETUP
         run: |
           cd ${{ steps.municipality-slug.outputs.slug }}
-          rm -Rf SETUP/
           git add -A
           git commit -m "Remove SETUP dir"
 
@@ -309,41 +308,17 @@ jobs:
 
             ##### Final Setup
 
-            * [ ] Run `make gen-key project=${{ steps.infrastructure-slug.outputs.slug }}`
+            * [ ] Copy the key generated from the prior `just init` process
             * [ ] Use the generated key as the [repository secret](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/secrets/actions) for `GOOGLE_CREDENTIALS`
             * [ ] Rerun the [infrastructure setup job](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/actions/workflows/deploy-infra.yml), then:
-              * [ ] Run `make set-cors project=${{ steps.infrastructure-slug.outputs.slug }}`
               * [ ] Enable [Google Speech-to-Text Data Logging](https://console.cloud.google.com/apis/api/speech.googleapis.com/data_logging?project=${{ steps.infrastructure-slug.outputs.slug }})
-              * [ ] Navigate to [Firebase Console](https://console.firebase.google.com), login to the CDP Google account, select the `${{ steps.infrastructure-slug.outputs.slug }}` Firebase project
-                * [ ] Navigate to "Firestore Database", select the "Rules" tab, paste the following in:
-                  ```
-                  rules_version = '2';
-                  service cloud.firestore {
-                    match /databases/{database}/documents {
-                      match /{document=**} {
-                        allow read;
-                      }
-                    }
-                  }
-                  ```
-                * [ ] Navigate to "Storage", select the "Rules" tab, paste the following in:
-                  ```
-                  rules_version = '2';
-                  service firebase.storage {
-                    match /b/{bucket}/o {
-                      match /{allPaths=**} {
-                        allow read;
-                      }
-                    }
-                  }
-                  ```
             * [ ] Enable [GitHub Pages](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/pages)
             * [ ] Comment on this issue with "Deployment Status - Complete" and the instance URL
 
             ##### Deletion Steps (Future Reference)
             
             * [ ] Delete the [instance repository](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings)
-            * [ ] Run `make login` and login to the CDP gcloud and pulumi accounts
-            * [ ] Run `make destroy project=${{ steps.infrastructure-slug.outputs.slug }}`
+            * [ ] Run `just login` and login to the CDP gcloud
+            * [ ] Run `just destroy project=${{ steps.infrastructure-slug.outputs.slug }}`
 
-            More details on the `make` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure).
+            More details on the `just` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure).

.github/workflows/instance-configuration-validation-bot.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - name: Install Bot Scripts Dependencies
         run: |
           pip install --upgrade pip
@@ -116,13 +116,15 @@ jobs:
           body="${body//$'\r'/'%0D'}" 
           echo ::set-output name=body::$body
 
-      - name: Get Municipality Slug
-        id: municipality-slug
+      - name: Get Infrastructure Metadata
+        id: infra-meta
         run: |
           slug=$(jq -r '.municipality_slug' planned-cookiecutter.json)
           infra_slug=$(jq -r '.infrastructure_slug' planned-cookiecutter.json)
+          region=$(jq -r '.firestore_region' planned-cookiecutter.json)
           echo "::set-output name=slug::$slug"
           echo "::set-output name=infra_slug::$infra_slug"
+          echo "::set-output name=region::$region"
 
       - name: Find Validation Results Comment - Post Run
         uses: peter-evans/find-comment@v1
@@ -161,11 +163,13 @@ jobs:
 
             To proceed with the deployment process, please do the following:
 
-            * [ ] Run `make login` in cdp-backend/dev-infrastructure and login to the CDP gcloud and pulumi accounts
-            * [ ] Run `make init project=${{ steps.municipality-slug.outputs.infra_slug }}` in cdp-backend/dev-infrastructure
+            * [ ] Run `get_cdp_infrastructure_stack dev-infrastructure/`
+            * [ ] Run `just login` in cdp-backend/dev-infrastructure and login to the CDP gcloud account
+            * [ ] Run `just init ${{ steps.infra-meta.outputs.infra_slug }}` in cdp-backend/dev-infrastructure
+            * [ ] Run `just setup ${{ steps.infra-meta.outputs.region }}` in cdp-backend/dev-infrastructure
             * [ ] Comment "/cdp-deploy" on this issue and follow the rest of the instructions
 
-            More details on the `make` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure).
+            More details on the `just` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure).
             
             _This comment was written by a bot!_
 

.github/workflows/scripts/requirements.txt

@@ -1,4 +1,4 @@
-cdp-backend[infrastructure]>=3.1.1
+cdp-backend>=3.2.0.dev0
 cdp-scrapers~=0.4
 cookiecutter~=1.7
 requests~=2.25

.github/workflows/scripts/validate_form.py

@@ -8,7 +8,7 @@
 import sys
 import traceback
 
-from cdp_backend.infrastructure.cdp_stack import GoverningBody
+from cdp_backend.infrastructure import GoverningBody
 from cdp_scrapers.legistar_utils import LegistarScraper
 from cdp_scrapers import instances
 import requests

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: Setup Node
       uses: actions/setup-node@v1
       with:

README.md

@@ -79,7 +79,6 @@ you will need to install `cookiecutter` and use this template.
 Before you begin, please note that you will need to install or have available the following:
 
 -   [gcloud](https://cloud.google.com/sdk/docs/install)
--   [pulumi](https://www.pulumi.com/docs/get-started/install/)
 -   [gsutil](https://cloud.google.com/storage/docs/gsutil_install)
 -   [Python 3.6+](https://www.python.org/downloads/) (Any Python version greater than or equal to 3.6)
 
@@ -109,7 +108,7 @@ see [Cookiecutter Repo Generation](#cookiecutter-repo-generation).
 The short summary of setup tasks remaining are:
 
 -   The creation of a new GitHub repository for the instance.
--   Logging in or creating accounts for Google Cloud and Pulumi.
+-   Logging in or creating an account for Google Cloud.
 -   Initialize the basic infrastructure.
 -   Assign a billing account to the created Google Cloud project.
 -   Generate credentials for the Google Project for use in automated scripts.
@@ -152,12 +151,6 @@ file storage, and (if needed) [speech-to-text](#speech-to-text) for transcriptio
 You can see more about the average monthly cost of running a
 CDP Instance in [Cost](#cost).
 
-[Pulumi](#pulumi) is a service to manage and track infrastructure deployment state.
-For those familiar with [Terraform](https://www.terraform.io/),
-the two are quite similar. Pulumi's purpose is to ensure that we can move from
-infrastructure upgrade to infrastructure upgrade without breaking anything
-(and skipping things that don't need to be done).
-
 For more details see [Cookiecutter Repo Generation](#cookiecutter-repo-generation).
 _After creating the repo, the following steps will have instructions and links specific_
 _to your deployment in the generated repository's README._
@@ -186,7 +179,7 @@ cookiecutter gh:CouncilDataProject/cookiecutter-cdp-deployment
 ```
 
 _Note: This will only create the basic repository._
-_You will still need to setup Google Cloud and Pulumi accounts._
+_You will still need to set up a Google Cloud account._
 
 ### Google Cloud
 
@@ -197,25 +190,6 @@ Google Cloud Platform (GCP).
 -   Your deployment's generated files (audio clips, transcripts, etc.) will live in [Filestore](https://cloud.google.com/filestore).
 -   When provided a video without closed captions, the audio from the provided video will be processed using [Speech-to-Text](https://cloud.google.com/speech-to-text).
 
-All of these resources will be set up for you using [Pulumi](#pulumi) but
-you will need to create both Google Cloud and Pulumi accounts. More information
-on these services and the steps for account creation can be found in the
-generated repository's README.
-
-### Pulumi
-
-Pulumi allows CDP developers and Instance maintainers to create, deploy, and manage
-infrastructure on any cloud using familiar programming languages and tools.
-It additionally, stores and tracks the _state_ of the CDP infrastructure,
-i.e. how many and which file storage, database, and processing resources are available.
-
-For CDP Instance maintainers, this simply means, the infrastructure management is
-packaged up as a part of `cdp-backend`, _and_ the infrastructure will never be
-incompatible with the pipelines as they are versioned together.
-
-Pulumi is free, and generally, you as an instance maintainer should never have to
-interact with Pulumi other than during the CDP Instance creation and setup process.
-
 ## Cost
 
 CDP was created and maintained by a group of people working on it in their free time.
@@ -229,7 +203,6 @@ Free Resources and Infrastructure:
 -   Event Processing (GitHub Actions)
 -   Event and Legislation Indexing (GitHub Actions)
 -   Web Hosting (GitHub Pages)
--   Infrastructure State Management (Pulumi)
 
 The backend resources and processing are the only real costs and depend on usage.
 The more users that use your web application, the more the database and

{{ cookiecutter.hosting_github_repo_name }}/.github/dependabot.yml

@@ -1,22 +1,19 @@
 version: 2
 updates:
-  - package-ecosystem: "pip"
-    directory: "/infra"
-    schedule:
-      interval: "daily"
-    allow:
-      - dependency-name: "cdp-backend"
-
   - package-ecosystem: "pip"
     directory: "/python"
     schedule:
       interval: "daily"
     allow:
       - dependency-name: "cdp-backend"
+    commit-message:
+      prefix: "ci(dependabot):"
 
   - package-ecosystem: "npm"
     directory: "/web"
     schedule:
       interval: "daily"
     allow:
       - dependency-name: "@councildataproject/cdp-frontend"
+    commit-message:
+      prefix: "ci(dependabot):"

{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/build-main.yml

@@ -19,7 +19,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: '3.10'
 
     - name: Install Packages
       run: |

{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/check-pr.yml

@@ -13,7 +13,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: '3.10'
 
     - name: Install Packages
       run: |

{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/deploy-infra.yml

@@ -17,20 +17,43 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
+    # Install OS Deps
     - uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: '3.10'
+    - uses: extractions/setup-just@v1
+      env:
+        GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %}
+    - name: Install firebase-cli
+      run: |
+        curl -sL https://firebase.tools | bash
+
+    # Install cdp-backend
     - name: Install Dependencies
       run: |
-        cd infra/
-        pip install -r requirements.txt
-    - name: Run Infrastructure Deploy
-      uses: pulumi/actions@v3
+        cd python/
+        pip install .
+    
+    # Setup gcloud
+    - uses: 'google-github-actions/auth@v0'
       with:
-        command: up
-        parallel: 4
-        stack-name: {{ cookiecutter.infrastructure_slug }}
-        work-dir: infra/
+        credentials_json: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v0
+
+    # Generate / copy infrastructure files
+    - name: Copy and Generate Infrastructure Files
+      run: |
+        get_cdp_infrastructure_stack infrastructure
+    
+    # Run infrastructure deploy
+    - name: Run Infrastructure Deploy
+      run: |
+        cd infrastructure
+        echo "$GOOGLE_CREDENTIALS" > google-creds.json
+        export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/google-creds.json
+        just deploy {{ cookiecutter.infrastructure_slug }} ../cookiecutter.yaml
       env:
+        FIREBASE_TOKEN: {% raw %}${{ secrets.FIREBASE_TOKEN }}{% endraw %}
         GOOGLE_CREDENTIALS: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %}
-        PULUMI_ACCESS_TOKEN: {% raw %}${{ secrets.PULUMI_ACCESS_TOKEN }}{% endraw %}

{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/deploy-web.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     # Setup Node
     - name: Setup Node
@@ -35,10 +35,7 @@ jobs:
         npm run build
 
     # Deploy Web
-    - name: Deploy Web
-      uses: JamesIves/github-pages-deploy-action@3.7.1
-      with:
-        GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %}
-        BASE_BRANCH: main # The branch the action should deploy from.
-        BRANCH: gh-pages # The branch the action should deploy to.
-        FOLDER: web/build/ # The folder the action should deploy.
+    - name: Publish Docs
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: web/build/