(E2E) Restricted Setup Tests #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: (E2E) Restricted Setup Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 9 * * *" | |
| env: | |
| GO_VERSION: '1.21.0' | |
| GOBIN: ${{ github.workspace }}/bin | |
| GAR_REGION: us-east1 | |
| GAR_PROJECT: hazelcast-33 | |
| GAR_REPO: hazelcast-platform-operator | |
| AWS_REGION: us-east-1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| get-image: | |
| name: Get Image | |
| runs-on: ubuntu-latest | |
| permissions: {} | |
| timeout-minutes: 10 | |
| outputs: | |
| IMG: ${{ steps.get-image-tag.outputs.IMG }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set Up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3.3.0 | |
| with: | |
| install: true | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Login to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Generate Image Name | |
| id: get-image-tag | |
| run: | | |
| echo "IMG=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/$(uuidgen):5d" >> $GITHUB_OUTPUT | |
| - name: Build Image | |
| uses: docker/build-push-action@v5.3.0 | |
| with: | |
| context: . | |
| builder: ${{ steps.buildx.outputs.name }} | |
| build-args: | | |
| version=${{github.sha}} | |
| pardotID=dockerhub | |
| file: Dockerfile | |
| push: true | |
| tags: ${{ steps.get-image-tag.outputs.IMG }} | |
| cache-from: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| cache-to: type=registry,ref=${{ env.GAR_REGION }}-docker.pkg.dev/${{ env.GAR_PROJECT }}/${{ env.GAR_REPO }}/${{ hashFiles('Dockerfile','main.go','api/**','controllers/**','internal/**','licenses/**','**/go.mod','**/go.sum') }}:14d | |
| kind-e2e-tests: | |
| name: Run E2E tests | |
| runs-on: ubuntu-latest | |
| needs: get-image | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node_number: | |
| [ "01", "02", "03", "04", "05", "06", "07", "08", "09", "10" ] | |
| env: | |
| NAMESPACE: kind-test-operator-ee | |
| WATCHED_NS: ns-1 | |
| RELEASE_NAME: hp-ee-${{ github.run_id }} | |
| CLUSTER_NAME: kind | |
| IMG: ${{ needs.get-image.outputs.IMG }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{steps.decide-ref.outputs.ref}} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Create Kind Config | |
| run: | | |
| cat <<EOT >> ./kind-config.yaml | |
| kind: Cluster | |
| apiVersion: kind.x-k8s.io/v1alpha4 | |
| nodes: | |
| - role: control-plane | |
| - role: worker | |
| - role: worker | |
| - role: worker | |
| EOT | |
| - name: Create Kind Cluster | |
| uses: helm/kind-action@v1.9.0 | |
| with: | |
| config: ./kind-config.yaml | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | |
| with: | |
| secret-ids: | | |
| GRAFANA_REMOTE_WRITE_URL,CN/GRAFANA_REMOTE_WRITE_URL | |
| GRAFANA_PROM_REMOTE_WRITE_URL,CN/GRAFANA_PROM_REMOTE_WRITE_URL | |
| GRAFANA_PROM_USERNAME,CN/GRAFANA_PROM_USERNAME | |
| GRAFANA_PROM_PASSWORD,CN/GRAFANA_PROM_PASSWORD | |
| LOKI_USERNAME,CN/GRAFANA_LOKI_USERNAME | |
| LOKI_PASSWORD,CN/GRAFANA_LOKI_PASSWORD | |
| HZ_LICENSE_KEY,CN/HZ_LICENSE_KEY | |
| AZURE_STORAGE_KEY,CN/AZURE_STORAGE_KEY | |
| GKE_SA_KEY,CN/GKE_SA_KEY | |
| - name: Install Grafana Agent | |
| env: | |
| CLUSTER_TYPE: kind | |
| GH_RUN_ID: ${{ github.run_id }} | |
| GH_RUN_NUMBER: ${{ github.run_number }} | |
| GH_SHA: ${{ github.sha }} | |
| NAMESPACES: ${{ env.NAMESPACE }} | |
| GRAFANA_NS: grafana | |
| run: | | |
| kubectl create namespace ${GRAFANA_NS} | |
| NAMESPACE=${GRAFANA_NS} envsubst < .github/grafana/metrics-cm.yaml | /bin/sh -c 'kubectl apply -f -' | |
| NAMESPACE=${GRAFANA_NS} envsubst < .github/grafana/logs-cm.yaml | /bin/sh -c 'kubectl apply -f -' | |
| MANIFEST_URL=https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/agent-bare.yaml NAMESPACE=${GRAFANA_NS} /bin/sh -c "$(curl -fsSL https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/install-bare.sh)" | kubectl apply -f - | |
| helm repo add prometheus-community https://prometheus-community.github.io/helm-charts && helm repo update && \ | |
| helm install ksm prometheus-community/kube-state-metrics --version 5.3.0 --set image.tag=v2.8.2 -n ${GRAFANA_NS} | |
| MANIFEST_URL=https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/agent-loki.yaml NAMESPACE=${GRAFANA_NS} /bin/sh -c "$(curl -fsSL https://raw.githubusercontent.com/grafana/agent/v0.27.1/production/kubernetes/install-bare.sh)" | kubectl apply -f - | |
| - name: Install metallb | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 5 | |
| max_attempts: 3 | |
| retry_on: error | |
| command: | | |
| kubectl create namespace ${NAMESPACE} | |
| kubectl config set-context --current --namespace=$NAMESPACE | |
| kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml | |
| kubectl wait --for=condition=ready --timeout=60s -n metallb-system pod -l app=metallb | |
| HOST_MIN=$(docker network inspect -f '{{json .IPAM.Config}}' ${{ env.CLUSTER_NAME }} | jq -r '.[0].Subnet' | sed -n 's/.0.0.16/.255.200/p') | |
| HOST_MAX=$(docker network inspect -f '{{json .IPAM.Config}}' ${{ env.CLUSTER_NAME }} | jq -r '.[0].Subnet' | sed -n 's/.0.0.16/.255.250/p') | |
| IP_RANGE=$HOST_MIN-$HOST_MAX | |
| cat <<EOF | kubectl apply -f - | |
| apiVersion: metallb.io/v1beta1 | |
| kind: IPAddressPool | |
| metadata: | |
| name: kind-pool | |
| namespace: metallb-system | |
| spec: | |
| addresses: | |
| - $IP_RANGE | |
| --- | |
| apiVersion: metallb.io/v1beta1 | |
| kind: L2Advertisement | |
| metadata: | |
| name: l2 | |
| namespace: metallb-system | |
| spec: | |
| ipAddressPools: | |
| - kind-pool | |
| EOF | |
| - name: Create Namespaces | |
| run: | | |
| kubectl create namespace ${{ env.WATCHED_NS }} | |
| - name: Create Secrets | |
| run: | | |
| kubectl create secret generic hazelcast-license-key \ | |
| --namespace ${{ env.NAMESPACE }} \ | |
| --from-literal=license-key=${{ env.HZ_LICENSE_KEY }} | |
| kubectl create secret generic hazelcast-license-key \ | |
| --namespace ${{ env.WATCHED_NS }} \ | |
| --from-literal=license-key=${{ env.HZ_LICENSE_KEY }} | |
| kubectl create secret generic br-secret-s3 \ | |
| --namespace ${{ env.WATCHED_NS }} \ | |
| --from-literal=region=us-east-1 \ | |
| --from-literal=access-key-id=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --from-literal=secret-access-key=${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| kubectl create secret generic br-secret-az \ | |
| --namespace ${{ env.WATCHED_NS }} \ | |
| --from-literal=storage-account=operatortest \ | |
| --from-literal=storage-key=${{ env.AZURE_STORAGE_KEY }} | |
| kubectl create secret generic br-secret-gcp \ | |
| --namespace ${{ env.WATCHED_NS }} \ | |
| --from-literal=google-credentials-path='${{ env.GKE_SA_KEY }}' | |
| - name: Authenticate to GAR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ env.GKE_SA_KEY }} | |
| - name: Load Image | |
| run: | | |
| docker pull $IMG | |
| kind load docker-image $IMG --name ${{ env.CLUSTER_NAME }} | |
| - name: Deploy Operator to Kind Cluster | |
| run: | | |
| DEPLOY_NAME=${RELEASE_NAME}-hazelcast-platform-operator | |
| make deploy IMG=$IMG RELEASE_NAME=$RELEASE_NAME NAMESPACE=${{ env.NAMESPACE }} WATCHED_NAMESPACES=${{ env.WATCHED_NS }} CREATE_CLUSTER_SCOPE_RESOURCES=false ENABLED_HZ_NODE_DISCOVERY=false WEBHOOK_ENABLED=false ENABLED_HZ_NODE_DISCOVERY=false | |
| kubectl rollout status deployment $DEPLOY_NAME | |
| - name: Run Tests | |
| id: e2e-test | |
| run: | | |
| make test-e2e-split-kind REPORT_SUFFIX=ee_${{ matrix.node_number }} GO_TEST_FLAGS=${GO_TEST_FLAGS} NAMESPACE=${{ env.WATCHED_NS }} WATCHED_NAMESPACES=${{ env.WATCHED_NS }} RELEASE_NAME=${RELEASE_NAME} WORKFLOW_ID=restricted SHARD_ID="${{ matrix.node_number }}" | |
| - name: Upload Test Report | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-report-restricted | |
| path: allure-results/restricted | |
| retention-days: 5 | |
| report-generation: | |
| needs: ["kind-e2e-tests"] | |
| if: always() && (needs.kind-e2e-tests.result == 'success' || needs.kind-e2e-tests.result == 'failure') | |
| uses: ./.github/workflows/generate-test-report.yaml | |
| secrets: inherit | |
| with: | |
| WORKFLOW_ID: restricted | |
| CLUSTER_NAME: kind |