Merge pull request #219 from City-of-Helsinki/release/2.5.0

Release/2.5.0
City-of-Helsinki · May 5, 2021 · 800249d · 800249d
2 parents 5a8e254 + b232855
commit 800249d
Show file tree

Hide file tree

Showing 41 changed files with 4,618 additions and 324 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,62 @@
+name: Continuous integration
+
+on:
+  push:
+    branches: [master, develop]
+  pull_request:
+
+
+env:
+  SECRET_KEY: topsecret123
+
+
+jobs:
+  test:
+    name: Tests
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:10
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_USER: kuvaselaamo
+          POSTGRES_PASSWORD: kuvaselaamo
+          POSTGRES_DB: kuvaselaamo
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python 2.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.7'
+
+      - name: Cache pip packages
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-pip-modules
+        with:
+          path: ~/.pip-cache
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-dev.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt -r requirements-dev.txt codecov
+      - name: Run tests
+        run: pytest -ra -vv --cov=.
+        env:
+          DATABASE_URL: postgres://kuvaselaamo:kuvaselaamo@localhost:5432/kuvaselaamo
+
+      - name: Coverage
+        run: codecov
diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml
@@ -2,7 +2,7 @@ name: Build & Production
 on:
   push:
     tags:
-      - 'release-*'
+      - "release-*"
 
 env:
   CONTAINER_REGISTRY: ghcr.io
@@ -11,15 +11,18 @@ env:
   CONTAINER_REGISTRY_REPO: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
   REPO_NAME: ${{ github.event.repository.name }}
   KUBECONFIG_RAW: ${{ secrets.KUBECONFIG_RAW_STABLE }}
-  BUILD_ARTIFACT_FOLDER: 'build_artifacts'
-  SERVICE_ARTIFACT_FOLDER: 'service_artifacts'
+  BUILD_ARTIFACT_FOLDER: "build_artifacts"
+  SERVICE_ARTIFACT_FOLDER: "service_artifacts"
   APP_MIGRATE_COMMAND: /app/.prod/on_deploy.sh
   SERVICE_PORT: "8080"
   K8S_REQUEST_CPU: 100m
   K8S_REQUEST_RAM: 200Mi
   K8S_LIMIT_CPU: 1000m
   K8S_LIMIT_RAM: 1Gi
   K8S_REPLICACOUNT: 2
+  VAULT_JWT_PRIVATE_KEY: ${{ secrets.VAULT_ACCESS_PRIVATE_KEY_STABLE }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+  VAULT_KV_VERSION: "2"
 
 jobs:
   build:
@@ -41,7 +44,7 @@ jobs:
       - name: Deploy
         uses: andersinno/kolga-deploy-action@v2
         with:
-          track: 'stable'
+          track: "stable"
         env:
           K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE_STABLE }}
           K8S_ADDITIONAL_HOSTNAMES: ${{ secrets.K8S_ADDITIONAL_HOSTNAMES }}
@@ -61,7 +64,7 @@ jobs:
           K8S_SECRET_HKM_PBW_API_ENDPOINT: "https://payform.bambora.com/pbwapi"
           K8S_SECRET_HKM_PBW_API_KEY: ${{ secrets.GH_STABLE_PBW_API_KEY }}
           K8S_SECRET_HKM_PBW_SECRET_KEY: ${{ secrets.GH_STABLE_PBW_SECRET_KEY }}
-          K8S_SECRET_HKM_MY_DOMAIN: "https://helsinkikuvia.prod.kuva.hel.ninja"
+          K8S_SECRET_HKM_MY_DOMAIN: "https://helsinkikuvia.fi"
           K8S_SECRET_HKM_PRINTMOTOR_USERNAME: ${{ secrets.GH_STABLE_PRINTMOTOR_USERNAME }}
           K8S_SECRET_HKM_PRINTMOTOR_PASSWORD: ${{ secrets.GH_STABLE_PRINTMOTOR_PASSWORD }}
           K8S_SECRET_HKM_PRINTMOTOR_API_KEY: ${{ secrets.GH_STABLE_PRINTMOTOR_API_KEY }}
@@ -78,3 +81,19 @@ jobs:
           K8S_SECRET_DATABASE_DB: ${{ secrets.K8S_SECRET_DATABASE_DB_STABLE }}
           K8S_SECRET_DATABASE_USERNAME: ${{ secrets.K8S_SECRET_DATABASE_USERNAME_STABLE }}
           K8S_SECRET_DATABASE_PASSWORD: ${{ secrets.K8S_SECRET_DATABASE_PASSWORD_STABLE }}
+          VAULT_JWT_AUTH_PATH: ${{ github.event.repository.name }}-stable
+          VAULT_KV_SECRET_MOUNT_POINT: stable
+
+      - name: Deploy Cronjob to send removal notifications and clean old data
+        uses: City-of-Helsinki/setup-cronjob-action@main
+        with:
+          name: send-reminders-and-clean-old-data-cronjob
+          image_repository: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
+          image_tag: ${{ github.sha }}
+          kubeconfig_raw: ${{ env.KUBECONFIG_RAW }}
+          target_namespace: ${{ secrets.K8S_NAMESPACE_STABLE }}
+          schedule: "15 0 * * *" # Daily at quarter past midnight
+          secret_name: "project-kuvaselaamo-secret"
+          command: "{/bin/sh}"
+          args: "{-c,cd /app && python manage.py send_removal_notifications && python manage.py clean_unused_data && python manage.py clearsessions}"
+          max_duration: 900 # 15 minutes
diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml
@@ -1,4 +1,4 @@
-name: Build & Review
+name: Build & Review & Accept
 on: [pull_request]
 
 env:
@@ -8,8 +8,8 @@ env:
   CONTAINER_REGISTRY_REPO: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
   REPO_NAME: ${{ github.event.repository.name }}
   KUBECONFIG_RAW: ${{ secrets.KUBECONFIG_RAW }}
-  BUILD_ARTIFACT_FOLDER: 'build_artifacts'
-  SERVICE_ARTIFACT_FOLDER: 'service_artifacts'
+  BUILD_ARTIFACT_FOLDER: "build_artifacts"
+  SERVICE_ARTIFACT_FOLDER: "service_artifacts"
   BASE_DOMAIN: ${{ secrets.BASE_DOMAIN_STAGING }}
   DATABASE_USER: user
   DATABASE_PASSWORD: testing-password
@@ -19,6 +19,9 @@ env:
   K8S_REQUEST_RAM: 200Mi
   K8S_LIMIT_CPU: 800m
   K8S_LIMIT_RAM: 1Gi
+  VAULT_JWT_PRIVATE_KEY: ${{ secrets.VAULT_ACCESS_PRIVATE_KEY_REVIEW }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+  VAULT_KV_VERSION: "2"
 
 jobs:
   build:
@@ -32,7 +35,7 @@ jobs:
   review:
     runs-on: ubuntu-latest
     needs: build
-    name: Review
+    name: Review and Accept
     steps:
       - uses: actions/checkout@v2
       - uses: andersinno/kolga-setup-action@v2
@@ -68,3 +71,45 @@ jobs:
           K8S_SECRET_HKM_PRINTMOTOR_API_ENDPOINT: "https://test-api.printmotor.com/api/v1/order"
           K8S_SECRET_HKM_POSTAL_FEES: ${{ secrets.HKM_POSTAL_FEES }}
           K8S_SECRET_LOG_LEVEL: ${{ secrets.LOG_LEVEL }}
+          VAULT_JWT_AUTH_PATH: ${{ github.event.repository.name }}-review
+          VAULT_KV_SECRET_MOUNT_POINT: review
+
+      - name: Setup kubectl
+        run: |
+          echo "${{ env.KUBECONFIG_RAW }}" > $(pwd)/kubeconfig
+          echo "KUBECONFIG=$(pwd)/kubeconfig" >> $GITHUB_ENV
+        shell: bash
+      - name: Get Review Deploy URL
+        id: deploy-url
+        run: |
+          DEPLOY_URL=$(kubectl get ingress -n "${{ env.K8S_NAMESPACE }}" -o jsonpath='{.items[0].spec.rules[0].host}')
+          echo "BROWSER_TESTING_BASE_URL=https://$DEPLOY_URL" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Setup Node
+        uses: actions/setup-node@v2.1.2
+        with:
+          node-version: 12.x
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install Dependencies
+        run: npm ci --prefer-offline
+
+      - name: Run Acceptance Tests
+        run: npm run browser-test:ci
+      - name: Upload screenshots of failed tests
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: screenshots
+          path: screenshots/
diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml
@@ -1,4 +1,4 @@
-name: Build & Staging
+name: Build & Staging & Accept
 on:
   push:
     branches:
@@ -11,15 +11,18 @@ env:
   CONTAINER_REGISTRY_REPO: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
   REPO_NAME: ${{ github.event.repository.name }}
   KUBECONFIG_RAW: ${{ secrets.KUBECONFIG_RAW_STAGING }}
-  BUILD_ARTIFACT_FOLDER: 'build_artifacts'
-  SERVICE_ARTIFACT_FOLDER: 'service_artifacts'
+  BUILD_ARTIFACT_FOLDER: "build_artifacts"
+  SERVICE_ARTIFACT_FOLDER: "service_artifacts"
   APP_MIGRATE_COMMAND: /app/.prod/on_deploy.sh
   SERVICE_PORT: "8080"
   K8S_REQUEST_CPU: 100m
   K8S_REQUEST_RAM: 200Mi
   K8S_LIMIT_CPU: 800m
   K8S_LIMIT_RAM: 1Gi
   K8S_REPLICACOUNT: 2
+  VAULT_JWT_PRIVATE_KEY: ${{ secrets.VAULT_ACCESS_PRIVATE_KEY_STAGING }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+  VAULT_KV_VERSION: "2"
 
 jobs:
   build:
@@ -33,15 +36,15 @@ jobs:
   staging:
     runs-on: ubuntu-latest
     needs: build
-    name: Staging
+    name: Staging and Accept
     steps:
       - uses: actions/checkout@v2
       - uses: andersinno/kolga-setup-action@v2
 
       - name: Deploy
         uses: andersinno/kolga-deploy-action@v2
         with:
-          track: 'staging'
+          track: "staging"
         env:
           K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE_STAGING }}
           ENVIRONMENT_URL: https://${{ secrets.ENVIRONMENT_URL_STAGING }}
@@ -76,17 +79,50 @@ jobs:
           K8S_SECRET_DATABASE_DB: ${{ secrets.K8S_SECRET_DATABASE_DB_STAGING }}
           K8S_SECRET_DATABASE_USERNAME: ${{ secrets.K8S_SECRET_DATABASE_USERNAME_STAGING }}
           K8S_SECRET_DATABASE_PASSWORD: ${{ secrets.K8S_SECRET_DATABASE_PASSWORD_STAGING }}
+          VAULT_JWT_AUTH_PATH: ${{ github.event.repository.name }}-staging
+          VAULT_KV_SECRET_MOUNT_POINT: staging
+
+      - name: Setup Node
+        uses: actions/setup-node@v2.1.2
+        with:
+          node-version: 12.x
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install Dependencies
+        run: npm ci --prefer-offline
+
+      - name: Run Acceptance Tests
+        run: npm run browser-test:ci
+        env:
+          BROWSER_TESTING_BASE_URL: https://${{ secrets.ENVIRONMENT_URL_STAGING }}
+      - name: Upload screenshots of failed tests
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: screenshots
+          path: screenshots/
 
       - name: Deploy Cronjob
         uses: City-of-Helsinki/setup-cronjob-action@main
         with:
           image_repository: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
-          image_tag:  ${{ github.sha }}
+          image_tag: ${{ github.sha }}
           secret_name: project-staging-kuvaselaamo-secret
           file_secret_name: project-staging-kuvaselaamo-file-secret
           file_secret_mount_path: /tmp/secrets
           kubeconfig_raw: ${{ env.KUBECONFIG_RAW }}
           target_namespace: ${{ secrets.K8S_NAMESPACE_STAGING }}
-          schedule: '0 0 * * *'
+          schedule: "0 0 * * *"
           command: "{/bin/sh}"
-          args: "{-c,cd /app && python manage.py clean_unused_data --days 365 && python manage.py clearsessions}"
+          args: "{-c,cd /app && python manage.py send_removal_notifications && python manage.py clean_unused_data && python manage.py clearsessions}"
+          max_duration: 900 # 15 minutes
diff --git a/.github/workflows/stop_review.yml b/.github/workflows/stop_review.yml
@@ -16,4 +16,4 @@ jobs:
       - name: Stop Review
         uses: andersinno/kolga-review-cleanup-action@v2
         with:
-          namespace: ${{ env.K8S_NAMESPACE }}
+          namespace: ${{ env.K8S_NAMESPACE }}
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,4 @@ docker-compose.env.yaml
 /venv/
 media/*
 static/*
+browser-tests/.env