From 510e4836e934bd0a8e7c56f2acc1a12c9c25018f Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 7 Dec 2022 11:57:20 +0100 Subject: [PATCH 1/4] QoL improvements for benchmarks --- .gitlab/benchmarks.yml | 2 +- .gitlab/scripts/analyze-results.sh | 4 ++++ .gitlab/scripts/post-pr-comment.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 3b52332ddd..ca81288c87 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -15,7 +15,7 @@ benchmark: script: - ./.gitlab/scripts/capture-hardware-software-info.sh - ./.gitlab/scripts/run-benchmarks.sh - # - ./.gitlab/scripts/upload-results-to-s3.sh #commented out until we have merged our permissions changes + - ./.gitlab/scripts/upload-results-to-s3.sh - ./.gitlab/scripts/analyze-results.sh - ./.gitlab/scripts/post-pr-comment.sh artifacts: diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 32522b939b..1a39b8853a 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -4,6 +4,10 @@ ARTIFACTS_DIR="/artifacts/${CI_JOB_ID}" REPORTS_DIR="$(pwd)/reports/" mkdir "${REPORTS_DIR}" || : +# Change threshold for detection of regression +# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference +UNCONFIDENCE_THRESHOLD=2.0 + BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" || :) source /benchmark-analyzer/.venv/bin/activate diff --git a/.gitlab/scripts/post-pr-comment.sh b/.gitlab/scripts/post-pr-comment.sh index 07aeccdaa0..5fe21372e9 100755 --- a/.gitlab/scripts/post-pr-comment.sh +++ b/.gitlab/scripts/post-pr-comment.sh @@ -2,4 +2,4 @@ REPORTS_DIR="$(pwd)/reports/" -cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks" +cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks" --on-duplicate=replace From ba4093e089b2e03f4732ecbd3cf8b4472148af4d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 7 Dec 2022 13:17:49 +0100 Subject: [PATCH 2/4] Update link to image --- .gitlab/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index ca81288c87..19532638be 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,6 +1,6 @@ variables: - # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform - BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum_test + # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform/-/jobs/201492583 + BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum INDEX_FILE: index.txt benchmark: From e7eb580df2898d88a36e325b5aa9dc9c0df1ec43 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 7 Dec 2022 13:29:59 +0100 Subject: [PATCH 3/4] Add candidate and baseline commit info to PR comment --- .gitlab/scripts/analyze-results.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 1a39b8853a..5db01da8ba 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -8,21 +8,34 @@ mkdir "${REPORTS_DIR}" || : # @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference UNCONFIDENCE_THRESHOLD=2.0 -BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" || :) +CANDIDATE_COMMIT_SHA=$CI_COMMIT_SHA +CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME + +cd dd-trace-go/ddtrace/tracer/ +BASELINE_COMMIT_SHA=$(git rev-parse HEAD) +BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :) source /benchmark-analyzer/.venv/bin/activate cd /benchmark-analyzer ./benchmark_analyzer convert \ --framework=GoBench \ + --extra-params="{\ + \"config\":\"candidate\", \ + \"git_commit_sha\":\"$CANDIDATE_COMMIT_SHA\", \ + \"git_branch\":\"$CANDIDATE_BRANCH\"\ + }" \ --outpath="pr.json" \ - --extra-params="{\"dd_trace_go\":\"${CI_COMMIT_REF_NAME}\"}" \ "${ARTIFACTS_DIR}/pr_bench.txt" ./benchmark_analyzer convert \ --framework=GoBench \ + --extra-params="{\ + \"config\":\"baseline\", \ + \"git_commit_sha\":\"$BASELINE_COMMIT_SHA\", \ + \"git_branch\":\"$BASELINE_BRANCH\"\ + }" \ --outpath="main.json" \ - --extra-params="{\"dd_trace_go\":\"main\"}" \ "${ARTIFACTS_DIR}/main_bench.txt" ./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json From 20b755355b6e3182de4713f775381e0324113ec3 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 7 Dec 2022 17:37:34 +0100 Subject: [PATCH 4/4] Ensure that baseline and candidate commits are compared in correct order --- .gitlab/scripts/analyze-results.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 5db01da8ba..767bd82036 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -38,6 +38,6 @@ cd /benchmark-analyzer --outpath="main.json" \ "${ARTIFACTS_DIR}/main_bench.txt" -./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json -./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json +./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json +./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json