(E2E) OCP Tests #633
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: (E2E) OCP Tests | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| schedule: | |
| - cron: "0 9 * * *" | |
| env: | |
| AWS_REGION: us-east-1 | |
| RESTART_OCP_NODE_TIMEOUT: 15 | |
| GRAFANA_NAMESPACE: grafana | |
| GAR_REGION: us-east1 | |
| GAR_PROJECT: hazelcast-33 | |
| GAR_REPO: hazelcast-platform-operator | |
| jobs: | |
| check-and-restart-nodes: | |
| name: Check OCP nodes state | |
| env: | |
| NR_CLUSTER_NAME: ocp-operator | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| OCP_CLUSTER_URL,CN/OCP_CLUSTER_URL | |
| OCP_USERNAME,CN/OCP_USERNAME | |
| OCP_PASSWORD,CN/OCP_PASSWORD | |
| - name: Restart Non-Ready Instances | |
| run: | | |
| source .github/scripts/utils.sh | |
| oc login ${{ env.OCP_CLUSTER_URL }} -u=${{ env.OCP_USERNAME }} -p=${{ env.OCP_PASSWORD }} --insecure-skip-tls-verify | |
| wait_for_instance_restarted $RESTART_OCP_NODE_TIMEOUT | |
| get-image: | |
| name: Get Image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| IMG: ${{ steps.get-image-tag.outputs.IMG }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set Up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3.3.0 | |
| with: | |
| install: true | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Generate Image Name | |
| id: get-image-tag | |
| run: | | |
| echo "IMG=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/$(uuidgen):5d" >> $GITHUB_OUTPUT | |
| - name: Build Image | |
| uses: docker/build-push-action@v5.3.0 | |
| with: | |
| context: . | |
| builder: ${{ steps.buildx.outputs.name }} | |
| build-args: | | |
| version=${{github.sha}} | |
| pardotID=dockerhub | |
| file: Dockerfile | |
| push: true | |
| tags: ${{ steps.get-image-tag.outputs.IMG }} | |
| cache-from: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| cache-to: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| grafana-setup: | |
| name: Setup Grafana agent | |
| env: | |
| GRAFANA_NS: "grafana-${{ github.run_id }}" | |
| CLUSTER_TYPE: ocp | |
| CLUSTER_NAME: api-demo-ocp4-hazelcast-com | |
| GH_RUN_ID: ${{ github.run_id }} | |
| GH_RUN_NUMBER: ${{ github.run_number }} | |
| GH_SHA: ${{ github.sha }} | |
| NAMESPACES: "oc-e2e-test-operator-os-${{ github.run_id }}, oc-e2e-test-operator-ee-${{ github.run_id }}" | |
| runs-on: ubuntu-latest | |
| needs: check-and-restart-nodes | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| OCP_CLUSTER_URL,CN/OCP_CLUSTER_URL | |
| OCP_USERNAME,CN/OCP_USERNAME | |
| OCP_PASSWORD,CN/OCP_PASSWORD | |
| GRAFANA_REMOTE_WRITE_URL,CN/GRAFANA_REMOTE_WRITE_URL | |
| GRAFANA_PROM_REMOTE_WRITE_URL,CN/GRAFANA_PROM_REMOTE_WRITE_URL | |
| GRAFANA_PROM_USERNAME,CN/GRAFANA_PROM_USERNAME | |
| GRAFANA_PROM_PASSWORD,CN/GRAFANA_PROM_PASSWORD | |
| LOKI_USERNAME,CN/GRAFANA_LOKI_USERNAME | |
| LOKI_PASSWORD,CN/GRAFANA_LOKI_PASSWORD | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Update kubeconfig | |
| run: |- | |
| oc login ${{ env.OCP_CLUSTER_URL }} -u=${{ env.OCP_USERNAME }} -p=${{ env.OCP_PASSWORD }} --insecure-skip-tls-verify | |
| - name: Install Grafana Agent metrics | |
| env: | |
| MANIFEST_URL: "https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/agent-bare.yaml" | |
| run: | | |
| kubectl create namespace ${GRAFANA_NS} | |
| NAMESPACE=${GRAFANA_NS} envsubst < .github/grafana/metrics-cm.yaml | /bin/sh -c 'kubectl apply -f -' | |
| oc adm policy add-scc-to-user privileged system:serviceaccount:${GRAFANA_NS}:grafana-agent | |
| oc adm policy add-scc-to-user privileged system:serviceaccount:${GRAFANA_NS}:default | |
| NAMESPACE=${GRAFANA_NS} /bin/sh -c "$(curl -fsSL https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/install-bare.sh)" | kubectl apply -f - | |
| - name: Deploy kube-state-metrics | |
| run: | | |
| oc adm policy add-scc-to-user privileged system:serviceaccount:${GRAFANA_NS}:ksm-kube-state-metrics | |
| helm repo add prometheus-community https://prometheus-community.github.io/helm-charts && \ | |
| helm repo update && \ | |
| helm install ksm prometheus-community/kube-state-metrics --version 5.3.0 --set image.tag=v2.8.2 -n ${GRAFANA_NS} | |
| - name: Install Grafana Agent logs | |
| env: | |
| MANIFEST_URL: "https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/agent-loki.yaml" | |
| run: | | |
| NAMESPACE=${GRAFANA_NS} envsubst < .github/grafana/logs-cm.yaml | /bin/sh -c 'kubectl apply -f -' | |
| oc adm policy add-scc-to-user privileged system:serviceaccount:${GRAFANA_NS}:grafana-agent-logs | |
| NAMESPACE=${GRAFANA_NS} /bin/sh -c "$(curl -fsSL https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/install-bare.sh)" | kubectl apply -f - | |
| ocp-e2e-tests: | |
| name: Run e2e test on OCP | |
| runs-on: ubuntu-latest | |
| needs: [grafana-setup, get-image] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| edition: [ 'os', 'ee' ] | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| NAMESPACE: oc-e2e-test-operator-${{ matrix.edition }}-${{ github.run_id }} | |
| RELEASE_NAME: hp-${{ matrix.edition }}-${{ github.run_id }} | |
| IMG: ${{ needs.get-image.outputs.IMG }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| HZ_LICENSE_KEY,CN/HZ_LICENSE_KEY | |
| AZURE_STORAGE_KEY,CN/AZURE_STORAGE_KEY | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| OCP_CLUSTER_URL,CN/OCP_CLUSTER_URL | |
| OCP_USERNAME,CN/OCP_USERNAME | |
| OCP_PASSWORD,CN/OCP_PASSWORD | |
| - name: Checkout to hazelcast-operator | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.21.0" | |
| - name: Update kubeconfig | |
| run: |- | |
| oc login ${{ env.OCP_CLUSTER_URL }} -u=${{ env.OCP_USERNAME }} -p=${{ env.OCP_PASSWORD }} --insecure-skip-tls-verify | |
| - name: Authenticate to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Install CRD's | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 1 | |
| max_attempts: 2 | |
| retry_on: error | |
| command: | | |
| make install-crds | |
| - name: Deploy Hazelcast-Platform-Operator to OCP | |
| run: | | |
| oc new-project $NAMESPACE | |
| DEPLOY_NAME=${RELEASE_NAME}-hazelcast-platform-operator | |
| make install-operator IMG=$IMG NAMESPACE=$NAMESPACE RELEASE_NAME=$RELEASE_NAME | |
| oc rollout status deployment ${DEPLOY_NAME} | |
| - name: Create secrets | |
| if: matrix.edition == 'ee' | |
| run: | | |
| kubectl create secret generic hazelcast-license-key \ | |
| --namespace ${{ env.NAMESPACE }} \ | |
| --from-literal=license-key=${{ env.HZ_LICENSE_KEY }} | |
| kubectl create secret generic br-secret-s3 --namespace ${NAMESPACE} \ | |
| --from-literal=region=us-east-1 \ | |
| --from-literal=access-key-id=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --from-literal=secret-access-key=${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| kubectl create secret generic br-secret-az --namespace ${NAMESPACE} \ | |
| --from-literal=storage-account=operatortest \ | |
| --from-literal=storage-key=${{ env.AZURE_STORAGE_KEY }} | |
| - name: Create secret for both OS and EE tests | |
| run: | | |
| kubectl create secret generic br-secret-gcp --namespace ${NAMESPACE} --from-literal=google-credentials-path='${{ env.GKE_SA_KEY }}' | |
| - name: Run Hazelcast E2E tests at OCP | |
| id: e2e-test | |
| run: | | |
| case ${{ matrix.edition }} in | |
| os) GO_TEST_FLAGS=-ee=false;; | |
| ee) GO_TEST_FLAGS=-ee=true;; | |
| *) echo Unexpected edition: ${{ matrix.edition }} && exit 1;; | |
| esac | |
| make test-e2e GO_TEST_FLAGS="${GO_TEST_FLAGS}" NAMESPACE=$NAMESPACE RELEASE_NAME=$RELEASE_NAME REPORT_SUFFIX=${{ matrix.edition }}_01 WORKFLOW_ID=ocp | |
| echo "RUNNING TESTS in $NAMESPACE" | |
| - name: Clean up after Tests | |
| if: always() | |
| run: | | |
| make clean-up-namespace KUBECTL=oc NAMESPACE=${NAMESPACE} | |
| - name: Upload Test Report | |
| if: always() && github.event_name != 'workflow_dispatch' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-report-ocp | |
| path: allure-results/ocp/ | |
| cleanup-unused-images-and-grafana: | |
| name: Remove Unused Images From The Local Store | |
| runs-on: ubuntu-latest | |
| needs: ocp-e2e-tests | |
| timeout-minutes: 10 | |
| if: always() | |
| env: | |
| GRAFANA_NS: "grafana-${{ github.run_id }}" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| OCP_CLUSTER_URL,CN/OCP_CLUSTER_URL | |
| OCP_USERNAME,CN/OCP_USERNAME | |
| OCP_PASSWORD,CN/OCP_PASSWORD | |
| - name: Image Prune And 'ksm-kube-state-metrics' Cleanup | |
| run: | | |
| oc login ${{ env.OCP_CLUSTER_URL }} -u=${{ env.OCP_USERNAME }} -p=${{ env.OCP_PASSWORD }} --insecure-skip-tls-verify | |
| helm uninstall ksm --namespace ${GRAFANA_NS} | |
| kubectl delete namespace ${GRAFANA_NS} --grace-period=0 --force | |
| for NODE in $(oc get nodes --no-headers -o name); do | |
| oc debug ${NODE} -- chroot /host sh -c "sleep 5; podman image prune -a -f" | |
| done | |
| update-test-run-status-badge: | |
| runs-on: ubuntu-latest | |
| needs: ocp-e2e-tests | |
| if: always() && github.event_name != 'workflow_dispatch' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| TEST_STATUSES_GIST_ID,CN/TEST_STATUSES_GIST_ID | |
| GIST_CREATION_TOKEN,CN/GIST_CREATION_TOKEN | |
| - name: Update Status Badge | |
| run: | | |
| source .github/scripts/utils.sh | |
| update_status_badges ${{ github.run_id }} ${{ env.TEST_STATUSES_GIST_ID }} ${{ env.GIST_CREATION_TOKEN }} | |
| report-generation: | |
| needs: ocp-e2e-tests | |
| if: always() && (needs.ocp-e2e-tests.result == 'success' || needs.ocp-e2e-tests.result == 'failure') && github.event_name != 'workflow_dispatch' | |
| uses: ./.github/workflows/generate-test-report.yaml | |
| secrets: inherit | |
| with: | |
| WORKFLOW_ID: ocp | |
| CLUSTER_NAME: api-demo-ocp4-hazelcast-com | |
| slack_notify: | |
| name: Slack Notify | |
| needs: [ 'ocp-e2e-tests' ] | |
| if: always() && needs.ocp-e2e-tests.result != 'success' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| SLACK_WEBHOOK_URL,CN/SLACK_WEBHOOK_URL | |
| - uses: 8398a7/action-slack@v3 | |
| with: | |
| fields: repo,commit,author,action,eventName,workflow | |
| status: failure | |
| channel: "#github-actions-log" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} |