(E2E) Platform Wan Replication Tests #193
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: (E2E) Platform Wan Replication Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 9 * * *" | |
| env: | |
| GCP_PROJECT_ID: hazelcast-33 | |
| GKE_ZONE: us-east1-b | |
| NUMBER_OF_NODES: 3 | |
| GCP_NETWORK: operator-test-network | |
| GAR_REGION: us-east1 | |
| GAR_PROJECT: hazelcast-33 | |
| GAR_REPO: hazelcast-platform-operator | |
| AWS_REGION: us-east-1 | |
| CHAOS_MESH_NAMESPACE: chaos-testing | |
| jobs: | |
| get-image: | |
| name: Get Image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| IMG: ${{ steps.get-image-tag.outputs.IMG }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set Up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3.3.0 | |
| with: | |
| install: true | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Generate Image Name | |
| id: get-image-tag | |
| run: | | |
| if [[ "${{github.event_name}}" == "workflow_dispatch" ]]; then | |
| echo "IMG=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/$(uuidgen):5d" >> $GITHUB_OUTPUT | |
| else | |
| IMG=hazelcast/hazelcast-platform-operator:latest-snapshot | |
| echo "IMG=${IMG}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build Image | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: docker/build-push-action@v5.3.0 | |
| with: | |
| context: . | |
| builder: ${{ steps.buildx.outputs.name }} | |
| build-args: | | |
| version=${{github.sha}} | |
| pardotID=dockerhub | |
| file: Dockerfile | |
| push: true | |
| tags: ${{ steps.get-image-tag.outputs.IMG }} | |
| cache-from: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| cache-to: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| create-gke-cluster: | |
| name: Create GKE Cluster | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| cluster: ["src", "trg"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate To ${{ matrix.cluster }} GCP Cluster | |
| uses: 'google-github-actions/auth@v2.1.2' | |
| with: | |
| credentials_json: ${{ env.GKE_SA_KEY }} | |
| - name: Create ${{ matrix.cluster }} GKE Cluster | |
| id: set-cluster-data | |
| run: |- | |
| gcloud container clusters create ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} \ | |
| --zone=${{ env.GKE_ZONE }} \ | |
| --project=${{ env.GCP_PROJECT_ID }} \ | |
| --network=${{ env.GCP_NETWORK }} \ | |
| --machine-type=n1-standard-16 \ | |
| --workload-pool=${{ env.GCP_PROJECT_ID }}.svc.id.goog \ | |
| --num-nodes=${{ env.NUMBER_OF_NODES }} | |
| sleep 20 | |
| grafana-setup: | |
| needs: [ create-gke-cluster ] | |
| uses: ./.github/workflows/grafana-agent.yaml | |
| secrets: inherit | |
| strategy: | |
| matrix: | |
| cluster: [ "src", "trg" ] | |
| with: | |
| cluster_name: ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} | |
| cluster_type: gke-wan | |
| gh_run_id: ${{ github.run_id }} | |
| gh_run_number: ${{ github.run_number }} | |
| gh_sha: ${{ github.sha }} | |
| namespaces: ${{ matrix.cluster }}-ns | |
| deploy-operator: | |
| name: Deploy Hazelcast Operator | |
| needs: [create-gke-cluster, grafana-setup, get-image] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| cluster: ["src", "trg"] | |
| namespace: ["src-ns", "trg-ns"] | |
| exclude: | |
| - cluster: trg | |
| namespace: src-ns | |
| env: | |
| IMG: ${{ needs.get-image.outputs.IMG }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.21.0" | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| HZ_LICENSE_KEY,CN/HZ_LICENSE_KEY | |
| - name: Authenticate To ${{ matrix.cluster }} GCP Cluster | |
| uses: 'google-github-actions/auth@v2.1.2' | |
| with: | |
| credentials_json: ${{ env.GKE_SA_KEY }} | |
| - name: Connect To The ${{ matrix.cluster }} GKE Cluster | |
| uses: 'google-github-actions/get-gke-credentials@v2' | |
| with: | |
| cluster_name: ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| location: ${{ env.GKE_ZONE }} | |
| - name: Authenticate to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Deploy CRDs | |
| if: matrix.namespace == 'trg-ns' | |
| run: make install-crds | |
| - name: Deploy Operator To ${{ matrix.cluster }} GKE Cluster and ${{ matrix.namespace }} namespace | |
| run: | | |
| kubectl create namespace ${{ matrix.namespace }} | |
| kubectl config set-context --current --namespace=${{ matrix.namespace }} | |
| DEPLOY_NAME=${{ matrix.namespace }}-hazelcast-platform-operator | |
| echo "DEPLOY_NAME=${DEPLOY_NAME}" >> $GITHUB_ENV | |
| make install-operator IMG=$IMG NAMESPACE=${{ matrix.namespace }} RELEASE_NAME=${{ matrix.namespace }} | |
| kubectl rollout status deployment $DEPLOY_NAME | |
| - name: Install Chaos Mesh | |
| if: matrix.namespace == 'trg-ns' | |
| run: | | |
| helm repo add chaos-mesh https://charts.chaos-mesh.org | |
| helm upgrade \ | |
| --install \ | |
| chaos-mesh \ | |
| chaos-mesh/chaos-mesh \ | |
| -n ${CHAOS_MESH_NAMESPACE} \ | |
| --create-namespace \ | |
| --set chaosDaemon.runtime=containerd \ | |
| --set chaosDaemon.socketPath=/run/containerd/containerd.sock \ | |
| --version v2.6.2 \ | |
| --wait | |
| kubectl delete validatingwebhookconfigurations.admissionregistration.k8s.io chaos-mesh-validation-auth | |
| - name: Create Secret On ${{ matrix.cluster }} Cluster and ${{ matrix.namespace }} namespace | |
| run: | | |
| kubectl create secret generic hazelcast-license-key \ | |
| --namespace ${{ matrix.namespace }} \ | |
| --from-literal=license-key=${{ env.HZ_LICENSE_KEY }} | |
| wan-gke-tests: | |
| name: Run Wan Test | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy-operator] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.21.0" | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate to GCP | |
| uses: 'google-github-actions/auth@v2.1.2' | |
| with: | |
| credentials_json: ${{ env.GKE_SA_KEY }} | |
| - name: Set Cluster And Context Name Variables | |
| run: | | |
| FIRST_CLUSTER_NAME="src-wan-cluster-${{ github.run_number }}" | |
| echo "FIRST_CLUSTER_NAME=${FIRST_CLUSTER_NAME}" >> $GITHUB_ENV | |
| SECOND_CLUSTER_NAME="trg-wan-cluster-${{ github.run_number }}" | |
| echo "SECOND_CLUSTER_NAME=${SECOND_CLUSTER_NAME}" >> $GITHUB_ENV | |
| FIRST_CONTEXT_NAME="gke_${{ env.GCP_PROJECT_ID }}_${{ env.GKE_ZONE }}_${FIRST_CLUSTER_NAME}" | |
| echo "FIRST_CONTEXT_NAME=${FIRST_CONTEXT_NAME}" >> $GITHUB_ENV | |
| SECOND_CONTEXT_NAME="gke_${{ env.GCP_PROJECT_ID }}_${{ env.GKE_ZONE }}_${SECOND_CLUSTER_NAME}" | |
| echo "SECOND_CONTEXT_NAME=${SECOND_CONTEXT_NAME}" >> $GITHUB_ENV | |
| echo "sourceNamespace="src-ns"" >> $GITHUB_ENV | |
| echo "targetNamespace="trg-ns"" >> $GITHUB_ENV | |
| - name: Connect To The Second GKE Cluster | |
| uses: 'google-github-actions/get-gke-credentials@v2' | |
| with: | |
| cluster_name: trg-wan-cluster-${{ github.run_number }} | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| location: ${{ env.GKE_ZONE }} | |
| - name: Get Second KUBECONFIG | |
| run: | | |
| echo "SECOND_KUBECONFIG=${{ env.KUBECONFIG }}" >> $GITHUB_ENV | |
| - name: Connect To The First GKE Cluster | |
| uses: 'google-github-actions/get-gke-credentials@v2' | |
| with: | |
| cluster_name: src-wan-cluster-${{ github.run_number }} | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| location: ${{ env.GKE_ZONE }} | |
| - name: Get First KUBECONFIG | |
| run: | | |
| echo "FIRST_KUBECONFIG=${{ env.KUBECONFIG }}" >> $GITHUB_ENV | |
| - name: Merge KUBECONFIG Files | |
| run: | | |
| KUBECONFIG="${{ env.FIRST_KUBECONFIG }}:${{ env.SECOND_KUBECONFIG }}" kubectl config view --raw > ${GITHUB_WORKSPACE}/kubeconfig | |
| echo "KUBECONFIG="${GITHUB_WORKSPACE}/kubeconfig"" >> $GITHUB_ENV | |
| - name: Run Wan Test | |
| run: |- | |
| make test-e2e NAMESPACE=src-ns RELEASE_NAME=src-ns E2E_TEST_LABELS=platform_wan REPORT_SUFFIX=ee_01 WORKFLOW_ID=wan | |
| - name: Upload Test Report | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-report-wan | |
| path: allure-results/wan/ | |
| report-generation: | |
| needs: ["create-gke-cluster", "wan-gke-tests"] | |
| if: always() && (needs.wan-gke-tests.result == 'success' || needs.wan-gke-tests.result == 'failure') | |
| uses: ./.github/workflows/generate-test-report.yaml | |
| secrets: inherit | |
| with: | |
| WORKFLOW_ID: wan | |
| CLUSTER_NAME: src-wan-cluster-${{ github.run_number }} | |
| cleanup-namespaces: | |
| name: Cleanup Namespaces | |
| if: always() | |
| needs: [create-gke-cluster, wan-gke-tests] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cluster: ["src", "trg"] | |
| namespace: ["src-ns", "trg-ns"] | |
| exclude: | |
| - cluster: trg | |
| namespace: src-ns | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate To GCP | |
| uses: 'google-github-actions/auth@v2.1.2' | |
| with: | |
| credentials_json: ${{ env.GKE_SA_KEY }} | |
| - name: Connect To ${{ matrix.cluster }} GCP Cluster | |
| uses: 'google-github-actions/get-gke-credentials@v2' | |
| with: | |
| cluster_name: ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| location: ${{ env.GKE_ZONE }} | |
| - name: Clean up after Tests on ${{ matrix.cluster }} Cluster and ${{ matrix.namespace }} Namespace | |
| if: always() | |
| run: | | |
| make clean-up-namespace NAMESPACE=${{ matrix.namespace }} RELEASE_NAME=${{ matrix.namespace }} | |
| - name: Clean up Grafana namespace | |
| if: always() | |
| timeout-minutes: 10 | |
| run: | | |
| source .github/scripts/utils.sh | |
| cleanup_namespace grafana | |
| cleanup_namespace ${{ matrix.namespace }} | |
| delete-gke-cluster: | |
| name: Delete GKE Cluster | |
| if: always() | |
| needs: [create-gke-cluster, wan-gke-tests, cleanup-namespaces] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cluster: ["src", "trg"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Authenticate To GCP | |
| uses: 'google-github-actions/auth@v2.1.2' | |
| with: | |
| credentials_json: ${{ env.GKE_SA_KEY }} | |
| - name: Connect To ${{ matrix.cluster }} GCP Cluster | |
| uses: 'google-github-actions/get-gke-credentials@v2' | |
| with: | |
| cluster_name: ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| location: ${{ env.GKE_ZONE }} | |
| - name: Delete ${{ matrix.cluster }} GKE Cluster | |
| if: always() | |
| run: |- | |
| gcloud container clusters delete ${{ matrix.cluster }}-wan-cluster-${{ github.run_number }} \ | |
| --zone ${{ env.GKE_ZONE }} \ | |
| --quiet | |
| slack_notify: | |
| name: Slack Notify | |
| needs: ["wan-gke-tests", "delete-gke-cluster"] | |
| if: always() && ( needs.wan-gke-tests.result != 'success' || needs.delete-gke-cluster.result != 'success') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| SLACK_WEBHOOK_URL,CN/SLACK_WEBHOOK_URL | |
| - uses: 8398a7/action-slack@v3 | |
| with: | |
| fields: repo,commit,author,action,eventName,workflow | |
| status: failure | |
| channel: "#github-actions-log" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} |