diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 3b52332ddd..19532638be 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,6 +1,6 @@ variables: - # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform - BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum_test + # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/relenv-microbenchmarking-platform/-/jobs/201492583 + BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-go_knusbaum INDEX_FILE: index.txt benchmark: @@ -15,7 +15,7 @@ benchmark: script: - ./.gitlab/scripts/capture-hardware-software-info.sh - ./.gitlab/scripts/run-benchmarks.sh - # - ./.gitlab/scripts/upload-results-to-s3.sh #commented out until we have merged our permissions changes + - ./.gitlab/scripts/upload-results-to-s3.sh - ./.gitlab/scripts/analyze-results.sh - ./.gitlab/scripts/post-pr-comment.sh artifacts: diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 32522b939b..767bd82036 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -4,23 +4,40 @@ ARTIFACTS_DIR="/artifacts/${CI_JOB_ID}" REPORTS_DIR="$(pwd)/reports/" mkdir "${REPORTS_DIR}" || : -BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" || :) +# Change threshold for detection of regression +# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference +UNCONFIDENCE_THRESHOLD=2.0 + +CANDIDATE_COMMIT_SHA=$CI_COMMIT_SHA +CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME + +cd dd-trace-go/ddtrace/tracer/ +BASELINE_COMMIT_SHA=$(git rev-parse HEAD) +BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :) source /benchmark-analyzer/.venv/bin/activate cd /benchmark-analyzer ./benchmark_analyzer convert \ --framework=GoBench \ + --extra-params="{\ + \"config\":\"candidate\", \ + \"git_commit_sha\":\"$CANDIDATE_COMMIT_SHA\", \ + \"git_branch\":\"$CANDIDATE_BRANCH\"\ + }" \ --outpath="pr.json" \ - --extra-params="{\"dd_trace_go\":\"${CI_COMMIT_REF_NAME}\"}" \ "${ARTIFACTS_DIR}/pr_bench.txt" ./benchmark_analyzer convert \ --framework=GoBench \ + --extra-params="{\ + \"config\":\"baseline\", \ + \"git_commit_sha\":\"$BASELINE_COMMIT_SHA\", \ + \"git_branch\":\"$BASELINE_BRANCH\"\ + }" \ --outpath="main.json" \ - --extra-params="{\"dd_trace_go\":\"main\"}" \ "${ARTIFACTS_DIR}/main_bench.txt" -./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json -./benchmark_analyzer compare pairwise --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json +./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report.md --format md-nodejs main.json pr.json +./benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath ${REPORTS_DIR}/report_full.html --format html main.json pr.json diff --git a/.gitlab/scripts/post-pr-comment.sh b/.gitlab/scripts/post-pr-comment.sh index 07aeccdaa0..5fe21372e9 100755 --- a/.gitlab/scripts/post-pr-comment.sh +++ b/.gitlab/scripts/post-pr-comment.sh @@ -2,4 +2,4 @@ REPORTS_DIR="$(pwd)/reports/" -cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks" +cat ${REPORTS_DIR}/report.md | /usr/local/bin/pr-commenter --for-repo="$CI_PROJECT_NAME" --for-pr="$CI_COMMIT_REF_NAME" --header="Benchmarks" --on-duplicate=replace