From 0ec58369bb70a9897376b9f345708734a9f066ef Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 21 Feb 2022 11:32:39 +0200 Subject: [PATCH] Add Azure-hosted benchmarking workflows for Linux and Windows. This patch leverages the new benchmarking features added in cri-tools in [this PR](https://github.com/kubernetes-sigs/cri-tools/pull/894) to add GitHub workflows for automatically running the benchmarks on Azure-based VMs for both Linux and Windows, as well as adding a Python script which generates plot graphs for the results. Signed-off-by: Nashwan Azhari --- .github/workflows/linux-benchmarks.yml | 314 +++++++++++++++++ .github/workflows/windows-benchmarks.yml | 333 ++++++++++++++++++ script/benchmark/process_benchmark_results.py | 242 +++++++++++++ 3 files changed, 889 insertions(+) create mode 100644 .github/workflows/linux-benchmarks.yml create mode 100644 .github/workflows/windows-benchmarks.yml create mode 100644 script/benchmark/process_benchmark_results.py diff --git a/.github/workflows/linux-benchmarks.yml b/.github/workflows/linux-benchmarks.yml new file mode 100644 index 0000000000000..39ee7bd7c62a0 --- /dev/null +++ b/.github/workflows/linux-benchmarks.yml @@ -0,0 +1,314 @@ +# Workflow intended to run CRI benchmarks on Linux. + +name: Linux Benchmarks + +on: + workflow_dispatch: + workflow_call: + secrets: + AZURE_SUB_ID: + required: true + AZURE_CREDS: + required: true + GCP_SERVICE_ACCOUNT: + required: true + GCP_WORKLOAD_IDENTITY_PROVIDER: + required: true + +env: + # Benchmarking-related options: + BENCHMARK_TYPE_PODS: "pods" + BENCHMARK_TYPE_CONTAINERS: "containers" + + # Test image options: + BUSYBOX_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/busybox:1.29-2" + RESOURCE_CONSUMER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/resource-consumer:1.10" + WEBSERVER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/nginx:1.14-2" + + # Azure-related options: + AZURE_DEFAULT_LOCATION: "westeurope" + AZURE_SUBSCRIPTION_ID: "${{ secrets.AZURE_SUB_ID }}" + AZURE_DEFAULT_VM_SIZE: "Standard_D2s_v3" + AZURE_DEFAULT_PASSWORD: "Passw0rdAdmin" + + # General options: + GOLANG_RELEASE_URL: "https://go.dev/dl/go1.17.6.linux-amd64.tar.gz" + ADMIN_USERNAME: "azureuser" + DEFAULT_ADMIN_PASSWORD: "Passw0rdAdmin" + REMOTE_VM_BIN_PATH: "/home/azureuser/containerd/bin" + SSH_OPTS: "-o ServerAliveInterval=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + GOOGLE_BUCKET_ROOT: "containerd-benchmarking" + + # Options related to the remote VM: + VM_HOME: "/home/azureuser" + VM_GOROOT: "/usr/local/go" + VM_GOPATH: "/home/azureuser/gopath" + VM_CRITOOLS_PATH: "/home/azureuser/cri-tools" + VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH: "/home/azureuser/cri-benchmark-settings.yaml" + VM_CRITEST_IMAGE_OPTIONS_FILEPATH: "/home/azureuser/cri-test-images.yaml" + VM_CRITEST_BENCHMARK_OUTPUT_DIR: "/home/azureuser/benchmarks" + VM_CRITEST_REPORT_DIR: "/home/azureuser/critest-logs" + VM_CONTAINERD_PATH: "/home/azureuser/containerd" + VM_CONTAINERD_LOGFILE: "/home/azureuser/critest-logs/containerd.log" + +jobs: + linuxBenchmarking: + runs-on: ubuntu-latest + + # NOTE: the following permissions are required by `google-github-actions/auth`: + permissions: + contents: 'read' + id-token: 'write' + + strategy: + matrix: + benchmark_params: [ + { + "os_distro": "ubuntu", + "os_release": "20.04", + "azure_vm_size": "Standard_D32s_v3", + "azure_vm_image": "Canonical:0001-com-ubuntu-server-focal:20_04-lts:20.04.202201180", + } + ] + + steps: + - uses: actions/checkout@v2 + + - name: Install required packages + run: | + sudo apt-get install xmlstarlet -y + + - name: DefineRunVariables + run: | + WORKFLOW_STARTED_TIME=$(date +%s) + echo "WORKFLOW_STARTED_TIME=$WORKFLOW_STARTED_TIME" >> $GITHUB_ENV + + # Azure-related vars: + AZURE_RESOURCE_GROUP_NAME="ctrd-benchmarking-${{ matrix.benchmark_params.os_distro }}-${{ matrix.benchmark_params.os_release }}-$WORKFLOW_STARTED_TIME" + echo "AZURE_RESOURCE_GROUP_NAME=$AZURE_RESOURCE_GROUP_NAME" >> $GITHUB_ENV + + # Local runner vars: + RUNNER_BENCHMARKS_DIR=$HOME/benchmarks/$WORKFLOW_STARTED_TIME + mkdir -p "$RUNNER_BENCHMARKS_DIR" + echo "RUNNER_BENCHMARKS_DIR=$RUNNER_BENCHMARKS_DIR" >> $GITHUB_ENV + jq -n --arg node temp --arg timestamp $WORKFLOW_STARTED_TIME '$timestamp|tonumber|{timestamp:.,$node}' > "$RUNNER_BENCHMARKS_DIR/started.json" + + # Google Cloud-related vars: + BENCHMARK_GOOGLE_BUCKET="${{ env.GOOGLE_BUCKET_ROOT }}/${{ matrix.benchmark_params.os_distro }}/${{ matrix.benchmark_params.os_release }}/$WORKFLOW_STARTED_TIME" + echo "BENCHMARK_GOOGLE_BUCKET=$BENCHMARK_GOOGLE_BUCKET" >> $GITHUB_ENV + + - name: Generate ssh key pair + run: | + mkdir -p $HOME/.ssh/ + ssh-keygen -t rsa -b 4096 -C "ci@containerd.com" -f $HOME/.ssh/id_rsa -q -N "" + echo "SSH_PUB_KEY=$(cat $HOME/.ssh/id_rsa.pub)" >> $GITHUB_ENV + echo "SSH_PUB_KEY_PATH=$HOME/.ssh/id_rsa.pub" >> $GITHUB_ENV + + - name: AZLogin + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDS }} + + - name: AZResourceGroupCreate + uses: azure/CLI@v1 + with: + inlinescript: | + az group create -n ${{ env.AZURE_RESOURCE_GROUP_NAME }} -l ${{ env.AZURE_DEFAULT_LOCATION }} --tags creationTimestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + + - name: AZTestVMCreate + uses: azure/CLI@v1 + with: + inlinescript: | + az vm create -n "${{ matrix.benchmark_params.os_distro }}-${{ matrix.benchmark_params.os_release }}-benchmarks" --admin-username ${{ env.ADMIN_USERNAME }} --admin-password ${{ env.DEFAULT_ADMIN_PASSWORD }} --image ${{ matrix.benchmark_params.azure_vm_image }} -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} --nsg-rule SSH --size ${{ matrix.benchmark_params.azure_vm_size }} --ssh-key-value "${{ env.SSH_PUB_KEY }}" + + - name: GetAZVMPublicIP + uses: azure/CLI@v1 + with: + inlinescript: | + echo "VM_PUB_IP=$(az network public-ip list -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} | jq '.[0]["ipAddress"]' | tr -d '\"')" >> $GITHUB_ENV + + - name: TestSSHConnection + run: | + if ! ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "hostname"; then + exit 1 + fi + + - name: CloneContainerDRepo + run: | + # Create directories: + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "mkdir -p ${{ env.VM_GOPATH }}/bin" + + # Clone containerd: + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "git clone http://github.com/containerd/containerd ${{ env.VM_CONTAINERD_PATH }}" + CONTAINERD_COMMIT=`ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cd ${{ env.VM_CONTAINERD_PATH }} && git log -1 --format=%H'"` + echo "CONTAINERD_COMMIT=$CONTAINERD_COMMIT" >> $GITHUB_ENV + + - name: PrepareTestingEnvUbuntu + if: ${{ matrix.benchmark_params.os_distro }} == "ubuntu" + run: | + # Install deps: + # - pk-config required by `containerd/script/install-cni` + # - unzip: required by `containerd/script/install-protobuf` + # - libbtrfs-dev btrfs-progs: containerd + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'sudo apt-get update -y && sudo apt-get install -y gcc make gperf pkg-config unzip libbtrfs-dev btrfs-progs'" + + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cat > /tmp/setup-containerd-deps.sh'" <<'EOF' + set -x + set -e + + # Latest Golang: + wget ${{ env.GOLANG_RELEASE_URL }} -O /tmp/go-release.tgz + tar -xzvf /tmp/go-release.tgz -C /usr/local + echo "export GOROOT=/usr/local/go" >> /tmp/gorc + echo "export GOPATH=${{ env.VM_GOPATH }}" >> /tmp/gorc + echo "export PATH=/usr/local/go/bin:${{ env.VM_GOPATH }}/bin:${{ env.VM_CONTAINERD_PATH }}/bin:$PATH" >> /tmp/gorc + source /tmp/gorc + cat /tmp/gorc >> /root/.bashrc + cat /tmp/gorc >> /home/azureuser/.bashrc + # chown -R azureuser:azureuser $GOPATH/pkg + + # ContainerD deps: + ${{ env.VM_CONTAINERD_PATH }}/script/setup/install-seccomp + PATH=$PATH GOPATH=$GOPATH bash ${{ env.VM_CONTAINERD_PATH }}/script/setup/install-runc + + # NOTE(aznashwan): the `install-cni` script expects containerd to have been pulled in `$GOPATH/src`: + sed -i.bak -E 's#"\$GOPATH"/src/github.com/containerd/containerd/go.mod#${{ env.VM_CONTAINERD_PATH }}/go.mod#' ${{ env.VM_CONTAINERD_PATH }}/script/setup/install-cni + PATH=$PATH GOPATH=$GOPATH bash ${{ env.VM_CONTAINERD_PATH }}/script/setup/install-cni + + # Protobuf: + PATH=$PATH GOPATH=$GOPATH bash ${{ env.VM_CONTAINERD_PATH }}/script/setup/install-protobuf + chmod +x /usr/local/bin/protoc + chmod og+rx /usr/local/include/google /usr/local/include/google/protobuf /usr/local/include/google/protobuf/compiler + chmod -R og+r /usr/local/include/google/protobuf/ + protoc --version + + # Make containerd: + PATH=$PATH GOPATH=$GOPATH bash -c 'cd ${{ env.VM_CONTAINERD_PATH }} && make binaries && make install' + + # Add ContainerD config: + mkdir -p /etc/containerd + printf "[plugins.cri.containerd.default_runtime]\nruntime_type = \"io.containerd.runc.v2\"\n" > /etc/containerd/config.toml + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sudo bash /tmp/setup-containerd-deps.sh" + + - name: PrepareBenchmarkParamFiles + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} azureuser@${{ env.VM_PUB_IP }} "sh -c 'cat > ${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }}'" <<'EOF' + defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }} + webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }} + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cat > ${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }}'" <<'EOF' + containersNumber: 500 + containersNumberParallel: 1 + podsNumber: 500 + podsNumberParallel: 1 + imagesNumber: 500 + imagesNumberParallel: 1 + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'mkdir -p ${{ env.VM_CRITEST_REPORT_DIR }}'" + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'mkdir -p ${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }}'" + + - name: GetCritestRepo + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "git clone https://github.com/kubernetes-sigs/cri-tools ${{ env.VM_CRITOOLS_PATH }}" + + - name: BuildCritest + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "bash -c 'source /tmp/gorc && cd ${{ env.VM_CRITOOLS_PATH }} && make install -e BINDIR=\$GOPATH/bin'" + + - name: RunCritestBenchmarks + run: | + BENCHMARK_STARTED_TIME=$(date +%s) + echo "BENCHMARK_STARTED_TIME=$BENCHMARK_STARTED_TIME" >> $GITHUB_ENV + + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cat > /tmp/run-containerd-benchmarks.sh'" <<'EOF' + set -x + set -e + + source /tmp/gorc + + # Start ContainerD: + ${{ env.VM_CONTAINERD_PATH }}/bin/containerd -log-level debug &> ${{ env.VM_CONTAINERD_LOGFILE }} & + ctr version > ${{ env.VM_CRITEST_REPORT_DIR }}/containerd-version.yaml + + # Run critest: + critest --runtime-endpoint="/var/run/containerd/containerd.sock" --test-images-file="${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }}" --report-dir="${{ env.VM_CRITEST_REPORT_DIR }}" --benchmark --benchmarking-params-file="${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }}" --benchmarking-output-dir="${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }}" + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sudo bash /tmp/run-containerd-benchmarks.sh" + + BENCHMARK_ENDED_TIME=$(date +%s) + echo "BENCHMARK_ENDED_TIME=$BENCHMARK_ENDED_TIME" >> $GITHUB_ENV + + - name: PullArtifactsFromVm + run: | + # Pull all logs: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} -r ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_REPORT_DIR }} "$RUNNER_BENCHMARKS_DIR/" + + # Pull benchmarks: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} -r ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }} "$RUNNER_BENCHMARKS_DIR/" + + # Pull config files for later reference: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }} "$RUNNER_BENCHMARKS_DIR/" + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }} "$RUNNER_BENCHMARKS_DIR/" + + - name: LogRunParams + run: | + # Write a file detailing the options used for the job: + cat > "$RUNNER_BENCHMARKS_DIR/benchmark-run-params.yaml" <<'EOF' + workflowRunId: ${{ env.WORKFLOW_STARTED_TIME }} + benchmarkStartedTime: ${{ env.BENCHMARK_STARTED_TIME }} + benchmarkEndedTime: ${{ env.BENCHMARK_ENDED_TIME }} + osDistro: ${{ matrix.benchmark_params.os_distro }} + osRelease: "${{ matrix.benchmark_params.os_release }}" + azureImage: ${{ matrix.benchmark_params.azure_vm_image }} + azureVmSize: ${{ matrix.benchmark_params.azure_vm_size }} + containerdCommit: ${{ env.CONTAINERD_COMMIT }} + runtimeTag: "runc-v2" + EOF + + - name: SetUpPython + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: ProcessBenchmarkResults + continue-on-error: true + run: | + # Install deps: + apt-get update && apt-get install -y libyaml-dev + pip install numpy matplotlib pyyaml + + # Prepare output dir: + OUTDIR=${{ env.RUNNER_BENCHMARKS_DIR }}/plots + mkdir $OUTDIR + + # Run script: + python $GITHUB_WORKSPACE/script/benchmark/process_benchmark_results.py --output-dir $OUTDIR ${{ env.RUNNER_BENCHMARKS_DIR }} + + - name: AssignGcpCreds + id: AssignGcpCreds + run: | + echo '::set-output name=GCP_SERVICE_ACCOUNT::${{ secrets.GCP_SERVICE_ACCOUNT }}' + echo '::set-output name=GCP_WORKLOAD_IDENTITY_PROVIDER::${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}' + + - name: AuthGcp + uses: google-github-actions/auth@v0 + if: steps.AssignGcpCreds.outputs.GCP_SERVICE_ACCOUNT && steps.AssignGcpCreds.outputs.GCP_WORKLOAD_IDENTITY_PROVIDER + with: + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: UploadBenchmarksData + uses: google-github-actions/upload-cloud-storage@v0.8.0 + if: steps.AssignGcpCreds.outputs.GCP_SERVICE_ACCOUNT && steps.AssignGcpCreds.outputs.GCP_WORKLOAD_IDENTITY_PROVIDER + with: + path: ${{ env.RUNNER_BENCHMARKS_DIR }} + destination: ${{ env.BENCHMARK_GOOGLE_BUCKET }} + parent: false + + - name: ResourceCleanup + uses: azure/CLI@v1 + if: always() + with: + inlinescript: | + az group delete -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} --yes diff --git a/.github/workflows/windows-benchmarks.yml b/.github/workflows/windows-benchmarks.yml new file mode 100644 index 0000000000000..14f7d80722e05 --- /dev/null +++ b/.github/workflows/windows-benchmarks.yml @@ -0,0 +1,333 @@ +# Workflow intended to run CRI benchmarks on Windows. + +name: Windows Benchmarks + +on: + workflow_dispatch: + workflow_call: + secrets: + AZURE_SUB_ID: + required: true + AZURE_CREDS: + required: true + GCP_SERVICE_ACCOUNT: + required: true + GCP_WORKLOAD_IDENTITY_PROVIDER: + required: true + +env: + # Benchmarking-related options: + BENCHMARK_TYPE_PODS: "pods" + BENCHMARK_TYPE_CONTAINERS: "containers" + + # NOTE: setting this to 'auto' will have the job auto-detect the version + # of hcsshim found in containerd's `go.mod` file. + HCSSHIM_VERSION_AUTO: "auto" + + # Test image options: + BUSYBOX_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/busybox:1.29-2" + RESOURCE_CONSUMER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/resource-consumer:1.10" + WEBSERVER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/nginx:1.14-2" + + # Azure-related options: + AZURE_DEFAULT_LOCATION: "westeurope" + AZURE_SUBSCRIPTION_ID: "${{ secrets.AZURE_SUB_ID }}" + AZURE_DEFAULT_VM_SIZE: "Standard_D8s_v3" + AZURE_DEFAULT_2019_IMAGE_ID: "MicrosoftWindowsServer:WindowsServer:2019-Datacenter-with-Containers-smalldisk:17763.1935.2105080716" + AZURE_DEFAULT_2022_IMAGE_ID: "MicrosoftWindowsServer:WindowsServer:2022-datacenter-smalldisk-g2:20348.169.2108120020" + + # General options: + DEFAULT_ADMIN_USERNAME: "azureuser" + DEFAULT_ADMIN_PASSWORD: "Passw0rdAdmin" + REMOTE_VM_BIN_PATH: "C:\\containerd\\bin" + SSH_OPTS: "-o ServerAliveInterval=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + GOOGLE_BUCKET_ROOT: "containerd-benchmarking" + + # Options related to the remote VM: + VM_CRITEST_IMAGE_OPTIONS_FILEPATH: "C:/cri-test-images.yaml" + VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH: "C:/cri-benchmark-settings.yaml" + VM_CRITEST_BENCHMARK_OUTPUT_DIR: "C:/Benchmarks" + VM_CRITEST_REPORT_DIR: "C:/Logs" + VM_CONTAINERD_LOGFILE: "C:/Logs/containerd.logs" + +jobs: + winBenchmarking: + runs-on: ubuntu-latest + + # NOTE: the following permissions are required by `google-github-actions/auth`: + permissions: + contents: 'read' + id-token: 'write' + + strategy: + matrix: + benchmark_params: [ + { + "windows_version": "ltsc2019", + "azure_vm_size": "Standard_D8s_v3", + "azure_vm_image": "MicrosoftWindowsServer:WindowsServer:2019-Datacenter-with-Containers-smalldisk:17763.1935.2105080716", + "hcsshim_version": "auto", + }, + { + "windows_version": "ltsc2022", + "azure_vm_size": "Standard_D8s_v3", + "azure_vm_image": "MicrosoftWindowsServer:WindowsServer:2022-datacenter-smalldisk-g2:20348.169.2108120020", + "hcsshim_version": "auto", + }, + ] + + steps: + - uses: actions/checkout@v2 + + - name: Install required packages + run: | + sudo apt-get install xmlstarlet -y + + - name: DefineRunVariables + run: | + WORKFLOW_STARTED_TIME=$(date +%s) + echo "WORKFLOW_STARTED_TIME=$WORKFLOW_STARTED_TIME" >> $GITHUB_ENV + + # Azure-related vars: + AZURE_RESOURCE_GROUP_NAME="ctrd-benchmarking-${{ matrix.benchmark_params.windows_version }}-$WORKFLOW_STARTED_TIME" + echo "AZURE_RESOURCE_GROUP_NAME=$AZURE_RESOURCE_GROUP_NAME" >> $GITHUB_ENV + + # Local runner vars: + RUNNER_BENCHMARKS_DIR=$HOME/benchmarks/$WORKFLOW_STARTED_TIME + mkdir -p "$RUNNER_BENCHMARKS_DIR" + echo "RUNNER_BENCHMARKS_DIR=$RUNNER_BENCHMARKS_DIR" >> $GITHUB_ENV + jq -n --arg node temp --arg timestamp $WORKFLOW_STARTED_TIME '$timestamp|tonumber|{timestamp:.,$node}' > "$RUNNER_BENCHMARKS_DIR/started.json" + + - name: Generate ssh key pair + run: | + mkdir -p $HOME/.ssh/ + ssh-keygen -t rsa -b 4096 -C "ci@containerd.com" -f $HOME/.ssh/id_rsa -q -N "" + echo "SSH_PUB_KEY=$(cat ~/.ssh/id_rsa.pub)" >> $GITHUB_ENV + + - name: AZLogin + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDS }} + + - name: AZResourceGroupCreate + uses: azure/CLI@v1 + with: + inlinescript: | + az group create -n ${{ env.AZURE_RESOURCE_GROUP_NAME }} -l ${{ env.AZURE_DEFAULT_LOCATION }} --tags creationTimestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + + - name: AZTestVMCreate + uses: azure/CLI@v1 + with: + inlinescript: | + DETAILS=$(az vm create -n winTestVM --admin-username ${{ env.DEFAULT_ADMIN_USERNAME }} --admin-password ${{ env.DEFAULT_ADMIN_PASSWORD }} --image ${{ matrix.benchmark_params.azure_vm_image }} -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} --nsg-rule SSH --size ${{ matrix.benchmark_params.azure_vm_size }} --public-ip-sku Standard -o json) + PUB_IP=$(echo $DETAILS | jq -r .publicIpAddress) + if [ "$PUB_IP" == "null" ] + then + RETRY=0 + while [ "$PUB_IP" == "null" ] || [ $RETRY -le 5 ] + do + sleep 5 + PUB_IP=$(az vm show -d -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} -n winTestVM -o json --query publicIps | jq -r) + RETRY=$(( $RETRY + 1 )) + done + fi + + if [ "$PUB_IP" == "null" ] + then + echo "failed to fetch public IP" + exit 1 + fi + echo "VM_PUB_IP=$PUB_IP" >> $GITHUB_ENV + + - name: GetAZVMPublicIP + uses: azure/CLI@v1 + with: + inlinescript: | + echo "VM_PUB_IP=$(az network public-ip list -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} | jq '.[0]["ipAddress"]' | tr -d '\"')" >> $GITHUB_ENV + + - name: EnableAZVMSSH + uses: azure/CLI@v1 + with: + inlinescript: | + az vm run-command invoke --command-id RunPowerShellScript -n winTestVM -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} --scripts @$GITHUB_WORKSPACE/script/setup/enable_ssh_windows.ps1 --parameters 'SSHPublicKey=${{ env.SSH_PUB_KEY }}' + + - name: TestSSHConnection + run: | + if ! ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "hostname"; + then + exit 1 + fi + + - name: InstallContainerFeatureWS2022 + if: ${{ matrix.benchmark_params.windows_version == 'ltsc2022' }} + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "powershell.exe -command { Install-WindowsFeature -Name 'Containers' -Restart }" + + - name: WaitForVMToRestart + if: ${{ matrix.benchmark_params.windows_version == 'ltsc2022' }} + timeout-minutes: 5 + run: | + # give the vm 30 seconds to actually stop. SSH server might actually respond while server is shutting down. + sleep 30 + while [ ! $( ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "hostname") ]; + do + echo "Unable to connect to azurevm" + done + echo "Connection reestablished. VM restarted succesfully." + + - name: CreateNatNetworkWS2022 + if: ${{ matrix.benchmark_params.windows_version == 'ltsc2022' }} + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "powershell.exe -command { curl.exe -L 'https://raw.githubusercontent.com/microsoft/SDN/master/Kubernetes/windows/hns.psm1' -o hns.psm1 }" + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "powershell.exe -command { Import-Module .\hns.psm1 ; New-HnsNetwork -Type NAT -Name nat -AddressPrefix 172.19.208.0/20 -Gateway 172.19.208.1 }" + + - name: PrepareTestingEnv + run: | + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} $GITHUB_WORKSPACE/script/setup/prepare_env_windows.ps1 ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:/prepare_env_windows.ps1 + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "c:\\prepare_env_windows.ps1" + + - name: MakeContainerDBins + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "git clone http://github.com/containerd/containerd c:\\containerd" + CONTAINERD_COMMIT=`ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh.exe -c 'cd c:\\containerd && git log -1 --format=%h'"` + echo "CONTAINERD_COMMIT=$CONTAINERD_COMMIT" >> $GITHUB_ENV + + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "cd c:\containerd ; make binaries" + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "git clone http://github.com/Microsoft/hcsshim c:\containerd\hcsshim " + + - name: SetupCni + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh.exe -c 'cd c:/containerd && ./script/setup/install-cni-windows'" + + - name: BuildHcsshim + run: | + if [ "${{ matrix.benchmark_params.hcsshim_version }}" = "${{ env.HCSSHIM_VERSION_AUTO }}" ]; then + # Get shim commit from containerd local repo + HCSSHIM_TAG=$(grep 'Microsoft/hcsshim' go.mod | awk '{ print $2 }'); + else + HCSSHIM_TAG=${{ matrix.benchmark_params.hcsshim_version }} + fi + echo "HCSSHIM_TAG=$HCSSHIM_TAG" >> $GITHUB_ENV + + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "cd c:\containerd\hcsshim; git fetch --tags origin $HCSSHIM_TAG ; \ + git checkout $HCSSHIM_TAG ; go build -mod=vendor -o ${{ env.REMOTE_VM_BIN_PATH }}\containerd-shim-runhcs-v1.exe .\cmd\containerd-shim-runhcs-v1" + + - name: PrepareBenchmarkParamFiles + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cat > ${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }}'" <<'EOF' + defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }} + webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }} + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'cat > ${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }}'" <<'EOF' + containersNumber: 500 + containersNumberParallel: 1 + podsNumber: 500 + podsNumberParallel: 1 + imagesNumber: 500 + imagesNumberParallel: 1 + imagePullingBenchmarkImage: "${{ env.BUSYBOX_TESTING_IMAGE_REF }}" + # NOTE(aznashwan): timeouts on Windows must be slightly more lenient + # than the default timeouts used on Linux: + podBenchmarkTimeoutSeconds: 120 + imageBenchmarkTimeoutSeconds: 60 + containerBenchmarkTimeoutSeconds: 120 + podContainerStartBenchmarkTimeoutSeconds: 120 + EOF + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'mkdir -p ${{ env.VM_CRITEST_REPORT_DIR }}'" + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh -c 'mkdir -p ${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }}'" + + - name: GetCritestRepo + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "git clone https://github.com/kubernetes-sigs/cri-tools c:/cri-tools" + + - name: BuildCritest + run: | + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "sh.exe -c 'cd /c/cri-tools && make critest'" + + - name: RunCritestBenchmarks + run: | + BENCHMARK_STARTED_TIME=$(date +%s) + echo "BENCHMARK_STARTED_TIME=$BENCHMARK_STARTED_TIME" >> $GITHUB_ENV + + ssh -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }} "powershell.exe -command { Start-Process -FilePath C:\containerd\bin\containerd.exe -NoNewWindow -RedirectStandardError true -PassThru ; get-process | sls containerd ; start-sleep 5 ; c:\cri-tools\build\bin\critest.exe --runtime-endpoint=\"npipe:\\\\.\\pipe\\containerd-containerd\" --test-images-file='${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }}' --report-dir='${{ env.VM_CRITEST_REPORT_DIR }}' --benchmark --benchmarking-params-file '${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }}' --benchmarking-output-dir '${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }}' }" + + BENCHMARK_ENDED_TIME=$(date +%s) + echo "BENCHMARK_ENDED_TIME=$BENCHMARK_ENDED_TIME" >> $GITHUB_ENV + + - name: PullArtifactsFromWindowsVm + run: | + # Pull all logs: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} -r ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_REPORT_DIR }} "$RUNNER_BENCHMARKS_DIR/" + + # Pull benchmarks: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} -r ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_BENCHMARK_OUTPUT_DIR }} "$RUNNER_BENCHMARKS_DIR/" + + # Pull config files for later reference: + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_IMAGE_OPTIONS_FILEPATH }} "$RUNNER_BENCHMARKS_DIR/" + scp -i $HOME/.ssh/id_rsa ${{ env.SSH_OPTS }} ${{ env.DEFAULT_ADMIN_USERNAME }}@${{ env.VM_PUB_IP }}:${{ env.VM_CRITEST_BENCHMARK_OPTIONS_FILEPATH }} "$RUNNER_BENCHMARKS_DIR/" + + - name: LogRunParams + run: | + # Write a file detailing the options used for the job: + cat > "$RUNNER_BENCHMARKS_DIR/benchmark-run-params.yaml" <<'EOF' + workflowRunId: ${{ env.WORKFLOW_STARTED_TIME }} + benchmarkStartedTime: ${{ env.BENCHMARK_STARTED_TIME }} + benchmarkEndedTime: ${{ env.BENCHMARK_ENDED_TIME }} + osDistro: "windows" + osRelease: ${{ matrix.benchmark_params.windows_version }} + azureImage: ${{ matrix.benchmark_params.azure_vm_image }} + azureVmSize: ${{ matrix.benchmark_params.azure_vm_size }} + containerdCommit: ${{ env.CONTAINERD_COMMIT }} + runtimeTag: "hscshim-${{ env.HCSSHIM_TAG }}" + EOF + + - name: SetUpPython + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: ProcessBenchmarkResults + continue-on-error: true + run: | + # Install deps: + apt-get update && apt-get install -y libyaml-dev + pip install numpy matplotlib pyyaml + + # Prepare output dir: + OUTDIR=${{ env.RUNNER_BENCHMARKS_DIR }}/plots + mkdir $OUTDIR + + # Run script: + python $GITHUB_WORKSPACE/script/benchmark/process_benchmark_results.py --output-dir $OUTDIR ${{ env.RUNNER_BENCHMARKS_DIR }} + + - name: AssignGcpCreds + id: AssignGcpCreds + run: | + # Format: $BUCKET_ROOT/containerd_COMMIT/OS_TYPE/OS_RELEASE/runtime_RUNTIME_COMMIT/WORKFLOW_STARTED_TIME + BENCHMARK_GOOGLE_BUCKET="${{ env.GOOGLE_BUCKET_ROOT }}/containerd_${{ env.CONTAINERD_COMMIT }}/windows/${{ matrix.benchmark_params.windows_version }}/hcsshim_${{ env.HCSSHIM_TAG }}/$WORKFLOW_STARTED_TIME" + echo "BENCHMARK_GOOGLE_BUCKET=$BENCHMARK_GOOGLE_BUCKET" >> $GITHUB_ENV + + echo '::set-output name=GCP_SERVICE_ACCOUNT::${{ secrets.GCP_SERVICE_ACCOUNT }}' + echo '::set-output name=GCP_WORKLOAD_IDENTITY_PROVIDER::${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}' + + - name: AuthGcp + uses: google-github-actions/auth@v0 + if: steps.AssignGcpCreds.outputs.GCP_SERVICE_ACCOUNT && steps.AssignGcpCreds.outputs.GCP_WORKLOAD_IDENTITY_PROVIDER + with: + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: UploadBenchmarksData + uses: google-github-actions/upload-cloud-storage@v0.8.0 + if: steps.AssignGcpCreds.outputs.GCP_SERVICE_ACCOUNT && steps.AssignGcpCreds.outputs.GCP_WORKLOAD_IDENTITY_PROVIDER + with: + path: ${{ env.RUNNER_BENCHMARKS_DIR }} + destination: ${{ env.BENCHMARK_GOOGLE_BUCKET }} + parent: false + + - name: ResourceCleanup + uses: azure/CLI@v1 + if: always() + with: + inlinescript: | + az group delete -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} --yes diff --git a/script/benchmark/process_benchmark_results.py b/script/benchmark/process_benchmark_results.py new file mode 100644 index 0000000000000..e32a763a5b35f --- /dev/null +++ b/script/benchmark/process_benchmark_results.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +# Copyright The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Processes containerd benchmark results from the periodic benchmarking workflow. +""" + +import argparse +import enum +import os +import re + +from matplotlib import pyplot +import numpy +import yaml + + +DEFAULT_BENCHMARKS_PARAMS_FILENAME = "benchmark-run-params.yaml" +DEFAULT_BENCHMARKS_METRICS_DIRNAME = "benchmarks" + +MARKDOWN_TABLE_HEADER_ROW = ( + "| Metric | Fastest | Average | Slowest | 95th% | 50th% |") +MARKDOWN_TABLE_ROW_OUTPUT_FORMAT = ( + "| %(metric_name)s | %(fastest).6fs | %(mean).6fs " + "| %(slowest).6fs | %(95th).6fs | %(50th).6fs |") + +BENCHMARK_LABEL_NAME_FORMAT = ( + "%(metric_name)s: %(os_distro)s %(os_release)s; %(azure_vm_size)s\n" + "ContainerD %(containerd_tag)s; %(runtime_tag)s; (RunID %(run_id)s)" +) + + +class GraphYAxisResolution(enum.Enum): + """ Labels various Y axis sizes (in miliseconds). """ + AUTO = 0 + XS = 500 + S = 1000 + M = 2500 + L = 5000 + XL = 10000 + XXL = 60000 + + +def _add_args(parser): + def _check_dir_arg(argname, value): + if not os.path.isdir(value): + raise argparse.ArgumentTypeError( + f"provided '{argname}' is not a directory: '{value}'") + return value + parser.add_argument( + "benchmarks_root_dir", metavar="/path/to/benchmarks", + type=lambda p: _check_dir_arg("benchmarks_root_dir", p), + help="String path to the directory containing the output(s) of " + "the benchmarking workflow(s).") + parser.add_argument( + "--output-dir", metavar="/path/to/output_dir", default=".", + type=lambda p: _check_dir_arg("--output-dir", p), + help="String path to the directory to output benchmark plots to. " + "Defaults to current working dir.") + return parser + + +def _get_bechmark_graph_label(metric_name, benchmark_run_params): + return BENCHMARK_LABEL_NAME_FORMAT % { + "metric_name": metric_name, + "os_distro": benchmark_run_params.get("osDistro", "unknown OS").capitalize(), + "os_release": benchmark_run_params.get("osRelease", "unknown Release"), + "containerd_tag": benchmark_run_params.get("containerdCommit", "unknown")[:9], + "runtime_tag": benchmark_run_params.get("runtimeTag", "unknown Runtime"), + "azure_vm_size": benchmark_run_params.get("azureVmSize", "unknown"), + "run_id": str(benchmark_run_params.get("workflowRunId", "unknown"))} + + +def _load_yaml_file(filepath): + with open(filepath, 'r') as fin: + return yaml.safe_load(fin) + + +def _convert_to_ms(string_val): + """ Converts the provided string benchmark metric to float in milliseconds. """ + if string_val.endswith("ms"): + # do nothing, is ms + return float(string_val.strip("ms")) + if string_val.endswith("µs"): + return float(string_val.strip("µs")) / 1000 + if string_val.endswith("s"): + return float(string_val.strip("s")) * 1000 + + +def _get_stats_for_metric_values(values): + return { + "mean": numpy.mean(values), + "slowest": numpy.max(values), + "fastest": numpy.min(values), + "95th": numpy.percentile(values, 95), + "50th": numpy.percentile(values, 50)} + + +def _generate_markdown_table(metrics_stats, outpath): + """ Outputs a markdown with the given stats to the given filepath. """ + table = MARKDOWN_TABLE_HEADER_ROW + for metric_name, stats in metrics_stats.items(): + stats.update({"metric_name": metric_name}) + print(f"{stats}") + table = "\n".join([table, MARKDOWN_TABLE_ROW_OUTPUT_FORMAT % stats]) + + with open(outpath, "w") as fout: + fout.write(table) + + +def _generate_plot_for_metric_values( + metric_values, outdir, plot_title, dpi=120, figsize=(10, 10), marker=".", + pad_inches=1, ylabel="time: ms", y_axis_max=None, color50th="green", + color95th="red"): + # Define plot object: + pyplot.figure(figsize=figsize, dpi=dpi) + pyplot.title(plot_title) + pyplot.ylabel(ylabel) + + # Define axes: + axes = pyplot.axes() + if y_axis_max: + pyplot.ylim((0, y_axis_max)) + + # X axis is simply the index (aka sequential number) of each datapoint: + xaxis = range(1, len(metric_values)+1) + axes.scatter(xaxis, metric_values, marker=marker) + + # Plot percentile lines: + stats = _get_stats_for_metric_values(metric_values) + axes.plot([1, 25, 50, len(metric_values)], [stats['50th']]*4, color=color50th) + axes.plot([1, 25, 50, len(metric_values)], [stats['95th']]*4, color=color95th) + + # Save figure: + outfile = os.path.join( + outdir, "%s.png" % re.sub("_+", "_", re.sub("[.:; \n]{1}", "_", plot_title))) + pyplot.savefig(outfile, dpi=dpi, pad_inches=pad_inches, format="png") + print(f"Generated: {outfile}") + + +def _normalize_results_data(results_data): + """Normalizes the provided results data to a simple mapping between the + names of operations and its respective datapoints in an ordered list. + """ + datapoints = results_data.get("datapoints") + if not datapoints: + # using v1 version of the output which is already properly-formatted: + return results_data + + operation_names = results_data.get("operationsNames") + if not operation_names: + raise ValueError( + f"Missing 'operationsNames' from results set: {results_data}", results_data) + + result = {} + for i, operation_name in enumerate(operation_names): + result[operation_name] = [ + point.get("operationsDurationsNs")[i] for point in datapoints] + + return result + + +def _process_metrics_file( + metrics_file_path, benchmark_run_params, outdir, generate_markdown=True): + """ Processes the given metrics data sets, outputting all relevant files + in the provided `outdir`. + """ + # NOTE(aznashwan): the actual metrics data is in JSON but since YAML 1.2+ + # is a superset of JSON we can simply parse it as YAML: + metrics_data = _load_yaml_file(metrics_file_path) + metrics_data = _normalize_results_data(metrics_data) + + # Generate graph for each metric: + all_y_limits = [ + GraphYAxisResolution.AUTO, GraphYAxisResolution.XS, GraphYAxisResolution.S, + GraphYAxisResolution.M, GraphYAxisResolution.L, GraphYAxisResolution.XL, + GraphYAxisResolution.XXL] + for metric_name, metric_values in metrics_data.items(): + for ylimit in all_y_limits: + title = _get_bechmark_graph_label( + "%s %s" % (metric_name, ylimit.name), benchmark_run_params) + _generate_plot_for_metric_values( + metric_values, outdir, title, y_axis_max=ylimit.value) + + # Generate markdown table for all metrics in the file: + if generate_markdown: + stats = { + metric_name: _get_stats_for_metric_values(metric_values) + for metric_name, metric_values in metrics_data.items()} + _generate_markdown_table( + stats, os.path.join(outdir, "stats-%s.md" % ( + os.path.basename(metrics_file_path)))) + + +def _process_benchmarks_directory(input_dir, output_dir): + benchmark_run_params_file = os.path.join(input_dir, DEFAULT_BENCHMARKS_PARAMS_FILENAME) + benchmark_run_params = _load_yaml_file(benchmark_run_params_file) + + # Check directory for actual benchmarks. + benchmarks_dir = os.path.join(input_dir, DEFAULT_BENCHMARKS_METRICS_DIRNAME) + if not os.path.isdir(benchmarks_dir): + # NOTE(aznashwan): some Windows benchmarks used `Benchmarks` (capitalized): + benchmarks_dir = os.path.join(input_dir, DEFAULT_BENCHMARKS_METRICS_DIRNAME.capitalize()) + + if not os.path.isdir(benchmarks_dir): + raise Exception( + f"Could not find benchmarks dir '{benchmarks_dir}' (capitalized or otherwise) " + "within the provided input dir '{input_dir}'.") + + # Process each metrics file: + for metrics_file_path in os.listdir(benchmarks_dir): + mfilepath = os.path.join(benchmarks_dir, metrics_file_path) + if os.path.isfile(mfilepath): + print(f"Processing metrics file '{metrics_file_path}'") + _process_metrics_file( + mfilepath, benchmark_run_params, output_dir, generate_markdown=True) + + +def main(): + parser = argparse.ArgumentParser( + description="Process ContainerD benchmarking results.") + _add_args(parser) + args = parser.parse_args() + + _process_benchmarks_directory(args.benchmarks_root_dir, args.output_dir) + + +if __name__ == "__main__": + main()