deployment-management-bot.yml

name: Instance Deployment Bot

on:
  issue_comment:
    types:
      - created

jobs:
  deploy-instance:
    runs-on: ubuntu-latest
    if: |
      contains(github.event.comment.html_url, '/issues/') &&
      contains(github.event.comment.body, '/cdp-deploy')

    steps:
      #########################################################################
      # Check initiator is a member of CDP

      - name: Get CDP Organization Members
        id: cdp-members
        env:
          GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
        run: |
          members="$(gh api -X GET 'orgs/CouncilDataProject/members' -F per_page=100 --paginate --cache 1h --jq '[.[].login] | join("---")')"
          echo "::set-output name=members::$members"

      - name: Generate Safe Username Check
        id: safe-username
        run: |
          username=${{ github.event.comment.user.login }}
          username="---$username---"
          echo "::set-output name=username::$username"

      - name: Check Job Initiator - Message
        if: |
          !contains(
            steps.cdp-members.outputs.members,
            steps.safe-username.outputs.username
          )
        uses: peter-evans/create-or-update-comment@v1
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ## Deployment Status

            ❌ ❌ **Rejected** ❌ ❌
            
            User (${{ github.event.comment.user.login }}) attempted to deploy without permissions.

            _Only users which are members of the CouncilDataProject organization can deploy new instances with this bot._

            **Stopping Deployment Procedure**
      
      - name: Check Job Initiator - Exit
        if: |
          !contains(
            steps.cdp-members.outputs.members,
            steps.safe-username.outputs.username
          )
        run: |
          exit 1

      #########################################################################
      # Workflow Setup

      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: '3.10'
      - name: Install Bot Scripts Dependencies
        run: |
          pip install --upgrade pip
          pip install -r .github/workflows/scripts/requirements.txt
          pip install cookiecutter

      # Run Validation Bot to get Configuration Options
      - name: Dump Issue Body to File
        run: |
          echo "${{ github.event.issue.body }}" > issue-body.md
      - name: Validate Form and Generate Configuration Files
        run: |
          python .github/workflows/scripts/validate_form.py issue-body.md

      # Store Cookiecutter Options
      - name: Store Cookiecutter Options
        id: cookiecutter-options
        run: |
          content=$(cat planned-cookiecutter.json)
          content="${content//'%'/'%25'}"
          content="${content//$'\n'/'%0A'}"
          content="${content//$'\r'/'%0D'}" 
          echo ::set-output name=options::$content

      #########################################################################
      # Run fast checks

      - name: Read generation-options JSON
        id: generation-options
        run: |
          content="$(cat generation-options.json)"
          echo "::set-output name=content::$content"

      - name: Error Early - Message
        uses: peter-evans/create-or-update-comment@v1
        if: |
          fromJSON(steps.generation-options.outputs.content).scraper_options == null ||
          fromJSON(steps.generation-options.outputs.content).maintainer_name == null ||
          fromJSON(steps.generation-options.outputs.content).repository_path == null
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ## Deployment Status

            :warning: :warning: **Configuration Error** :warning: :warning:

            Not all configuration options are present or some options have errors.

            #### Configuration Options

            ```json
            ${{ steps.generation-options.outputs.content }}
            ```

            **Stopping Deployment Procedure**
      
      - name: Error Early - Exit
        if: |
          fromJSON(steps.generation-options.outputs.content).scraper_options == null ||
          fromJSON(steps.generation-options.outputs.content).maintainer_name == null ||
          fromJSON(steps.generation-options.outputs.content).repository_path == null
        run: |
          exit 1
    
      #########################################################################
      # Proceed with deployment

      - name: Get Prior Infrastructure Slug
        id: get-infra-slug-comment
        uses: peter-evans/find-comment@v1
        with:
          issue-number: ${{ github.event.issue.number }}
          comment-author: 'github-actions[bot]'
          body-includes: 'Generated Infrastructure Slug'

      - name: Parse Prior Infrastructure Slug
        id: infrastructure-slug
        run: |
          slug=$(python -c 'content = """${{ steps.get-infra-slug-comment.outputs.comment-body }}"""; print(content[content.index("`") + 1:content.index("`", content.index("`") + 1)]);')
          echo "::set-output name=slug::$slug"

      - name: Update Infrastructure Slug in Cookiecutter Options
        run: |
          replacement=$(jq '.infrastructure_slug = "${{ steps.infrastructure-slug.outputs.slug }}"' planned-cookiecutter.json)
          echo "$replacement" > planned-cookiecutter.json
          cat planned-cookiecutter.json

      - name: Run Cookiecutter
        run: |
          mv planned-cookiecutter.json cookiecutter.json
          cookiecutter . --no-input

      - name: Get Municipality Slugs
        id: municipality-slug
        run: |
          slug=$(jq -r '.municipality_slug' cookiecutter.json)
          python_slug=$(jq -r '.python_municipality_slug' cookiecutter.json)
          echo "::set-output name=slug::$slug"
          echo "::set-output name=python_slug::$python_slug"

      - name: Init Git
        run: |
          cd ${{ steps.municipality-slug.outputs.slug }}
          git config --global user.email "councildataproject@gmail.com"
          git config --global user.name "CouncilDataProjectServiceAccount"
          git init
          git checkout -b main
          git add -A
          git commit -m "Initial commit"

      # Replace scraper with provided
      - name: Add cdp-scrapers as dependency
        run: sed -i '21 i \    "cdp-scrapers[${{ steps.municipality-slug.outputs.python_slug }}]",' ${{ steps.municipality-slug.outputs.slug }}/python/setup.py

      # Get scraper path and update scraper function
      # The sed after param2 extract is to replace `/` with `\/` to escape the char
      - name: Get Scraper Path
        id: selected-scraper
        run: |
          scraper_options=$(jq -r '.scraper_options' generation-options.json)
          scraper_choice=$(echo $scraper_options | cut -f1 -d %)
          param1=$(echo $scraper_options | cut -f2 -d %)
          param2=$(echo $scraper_options | cut -f3 -d %)
          param2=$(sed 's/\//\\\//g' <<< $param2)
          echo "::set-output name=choice::$scraper_choice"
          echo "::set-output name=param1::$param1"
          echo "::set-output name=param2::$param2"

      # Use base legistar for scraper
      - name: Add Base Legistar Scraper
        if: |
          contains(steps.selected-scraper.outputs.choice, 'USE_BASE_LEGISTAR')
        run: |
          sed -i \
            's/REPLACE_LEGISTAR_CLIENT/${{ steps.selected-scraper.outputs.param1 }}/g' \
            .github/workflows/resources/base_legistar_scraper.py
          sed -i \
            's/REPLACE_IANA_CLIENT_TZ/${{ steps.selected-scraper.outputs.param2 }}/g' \
            .github/workflows/resources/base_legistar_scraper.py
          mv \
            .github/workflows/resources/base_legistar_scraper.py \
            ${{ steps.municipality-slug.outputs.slug }}/python/cdp_${{ steps.municipality-slug.outputs.python_slug }}_backend/scraper.py

      # Use custom scraper
      - name: Add Custom Scraper
        if: |
          contains(steps.selected-scraper.outputs.choice, 'USE_FOUND_SCRAPER')
        run: |
          sed -i \
            's/REPLACE_CUSTOM_SCRAPER/${{ steps.selected-scraper.outputs.param1 }}/g' \
            .github/workflows/resources/custom_scraper.py
          mv \
            .github/workflows/resources/custom_scraper.py \
            ${{ steps.municipality-slug.outputs.slug }}/python/cdp_${{ steps.municipality-slug.outputs.python_slug }}_backend/scraper.py
      
      # Commit scraper changes
      - name: Commit Scraper Dep and Usage
        run: |
          cd ${{ steps.municipality-slug.outputs.slug }}
          git add -A
          git commit -m "Update scraper dependency and implementation"

      - name: Remove SETUP
        run: |
          cd ${{ steps.municipality-slug.outputs.slug }}
          git add -A
          git commit -m "Remove SETUP dir"

      - name: Setup Git SSH
        run: |
          eval "$(ssh-agent -s)"
          mkdir ~/.ssh/
          echo "${{ secrets.SA_SSH_KEY }}" > ~/.ssh/id_cdp_sa
          chmod 600 ~/.ssh/id_cdp_sa
          ssh-add ~/.ssh/id_cdp_sa
          echo "Host github.com
            HostName github.com
            IdentityFile ~/.ssh/id_cdp_sa" > ~/.ssh/config

      - name: Create New Instance Repo and Push
        run: |
          cd ${{ fromJSON(steps.cookiecutter-options.outputs.options ).hosting_github_repo_name }}
          gh repo create \
            ${{ fromJSON(steps.generation-options.outputs.content).repository_path }} \
            --public \
            --description "CDP Instance for ${{ fromJSON(steps.cookiecutter-options.outputs.options).municipality }}" \
            --homepage "${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_web_app_address }}" \
            --disable-wiki
          sleep 10s
          git remote add origin git@github.com:${{ fromJSON(steps.generation-options.outputs.content).repository_path }}.git
          sleep 10s
          git push -u origin main
        env:
          GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}

      - name: Generate Safe Maintainer Name Check
        id: safe-maintainer
        run: |
          username=$(jq -r '.maintainer_name' generation-options.json)
          username="---$username---"
          echo "::set-output name=username::$username"

      # Add external collaborator to the repo if the maintainer isn't in CDP org
      - name: Add External Collaborator
        if: |
          !contains(
            steps.cdp-members.outputs.members,
            steps.safe-maintainer.outputs.username
          )
        run: |
          gh api \
            repos/CouncilDataProject/${{ steps.municipality-slug.outputs.slug }}/collaborators/$(jq -r '.maintainer_name' generation-options.json) \
            -X PUT \
            -f permission='maintain'
        env:
          GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}

      - name: Log Success
        uses: peter-evans/create-or-update-comment@v1
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            ## Deployment Status

            :tada: :tada: **Repository Created** :tada: :tada:
            
            A new CouncilDataProject Instance Repository was created ([${{ fromJSON(steps.generation-options.outputs.content).repository_path }}](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }})), external collaborator added (@${{ fromJSON(steps.generation-options.outputs.content).maintainer_name }}), and cookiecutter files generated and pushed to repository.

            The instance is setting itself up right now and the process will take around 10 minutes to complete. Once completed, a CDP maintainer will comment on this issue with your instance's website link. See [the instance's GitHub Action job history](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/actions) for more details on the deployment setup progress.

            Your CDP instance will be populated with data within 6 hours of website creation.

            At any point in the future if you would like to destroy this instance, please just add a comment to this thread and a maintainer will help you.

            ---

            #### Steps for Internal CDP Team

            ##### Final Setup

            * [ ] Copy the key generated from the prior `just init` process
            * [ ] Use the generated key as the [repository secret](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/secrets/actions) for `GOOGLE_CREDENTIALS`
            * [ ] Rerun the [infrastructure setup job](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/actions/workflows/deploy-infra.yml), then:
              * [ ] Enable [Google Speech-to-Text Data Logging](https://console.cloud.google.com/apis/api/speech.googleapis.com/data_logging?project=${{ steps.infrastructure-slug.outputs.slug }})
            * [ ] Enable [GitHub Pages](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/pages)
            * [ ] Comment on this issue with "Deployment Status - Complete" and the instance URL

            ##### Deletion Steps (Future Reference)
            
            * [ ] Delete the [instance repository](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings)
            * [ ] Run `just login` and login to the CDP gcloud
            * [ ] Run `just destroy project=${{ steps.infrastructure-slug.outputs.slug }}`

            More details on the `just` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure).