ci: QoL improvements for benchmarks (#1609)

Lots of small improvements: fixed comparison ordering, fix distortion from long labels, upload results of benchmark to s3, edit result PR comment instead of making a new comment each run, increase significance threshold to reduce false positives

.gitlab/benchmarks.yml

@@ -1,6 +1,6 @@
 variables:
-  # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform
-  BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum_test
+  # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform/-/jobs/201492583
+  BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum
   INDEX_FILE: index.txt
 
 benchmark:
@@ -15,7 +15,7 @@ benchmark:
   script:
     - ./.gitlab/scripts/capture-hardware-software-info.sh
     - ./.gitlab/scripts/run-benchmarks.sh
-    # - ./.gitlab/scripts/upload-results-to-s3.sh #commented out until we have merged our permissions changes
+    - ./.gitlab/scripts/upload-results-to-s3.sh
     - ./.gitlab/scripts/analyze-results.sh
     - ./.gitlab/scripts/post-pr-comment.sh
   artifacts:

.gitlab/scripts/analyze-results.sh

@@ -4,23 +4,40 @@ ARTIFACTS_DIR="/artifacts/${CI_JOB_ID}"
 REPORTS_DIR="$(pwd)/reports/"
 mkdir "${REPORTS_DIR}" || :
 
-BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" || :)
+# Change threshold for detection of regression
+# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference
+UNCONFIDENCE_THRESHOLD=2.0
+
+CANDIDATE_COMMIT_SHA=$CI_COMMIT_SHA
+CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME
+
+cd dd-trace-go/ddtrace/tracer/
+BASELINE_COMMIT_SHA=$(git rev-parse HEAD)
+BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :)
 
 source /benchmark-analyzer/.venv/bin/activate
 cd /benchmark-analyzer
 
 ./benchmark_analyzer convert \
   --framework=GoBench \
+  --extra-params="{\
+    \"config\":\"candidate\", \
+    \"git_commit_sha\":\"$CANDIDATE_COMMIT_SHA\", \
+    \"git_branch\":\"$CANDIDATE_BRANCH\"\
+    }" \
   --outpath="pr.json" \
-  --extra-params="{\"dd_trace_go\":\"${CI_COMMIT_REF_NAME}\"}" \
   "${ARTIFACTS_DIR}/pr_bench.txt"
 
 ./benchmark_analyzer convert \
   --framework=GoBench \
+  --extra-params="{\
+    \"config\":\"baseline\", \
+    \"git_commit_sha\":\"$BASELINE_COMMIT_SHA\", \
+    \"git_branch\":\"$BASELINE_BRANCH\"\
+    }" \
   --outpath="main.json" \
-  --extra-params="{\"dd_trace_go\":\"main\"}" \
   "${ARTIFACTS_DIR}/main_bench.txt"
 
-./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json
-./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json
+./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json
+./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json
 

.gitlab/scripts/post-pr-comment.sh

@@ -2,4 +2,4 @@
 
 REPORTS_DIR="$(pwd)/reports/"
 
-cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks"
+cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks" --on-duplicate=replace