diff --git a/.buildkite/package-lock.json b/.buildkite/package-lock.json index e8509e8126adba4..95f21ca691d02d3 100644 --- a/.buildkite/package-lock.json +++ b/.buildkite/package-lock.json @@ -8,7 +8,7 @@ "name": "kibana-buildkite", "version": "1.0.0", "dependencies": { - "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a0037514b7650296a23dbad99b165601d4eab1be" + "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a5010e2b33291290dc45d2c0013b5d5646fbfa09" } }, "node_modules/@nodelib/fs.scandir": { @@ -184,24 +184,11 @@ "follow-redirects": "^1.14.0" } }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "node_modules/before-after-hook": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", @@ -368,15 +355,14 @@ }, "node_modules/kibana-buildkite-library": { "version": "1.0.0", - "resolved": "git+https://git@github.com/elastic/kibana-buildkite-library.git#a0037514b7650296a23dbad99b165601d4eab1be", - "integrity": "sha512-W9oH2c0q21IbO3sKJR2BkebhDlXVuWfqKO1r6T/E8/RRxCXJg/Wf073k8aDdpl1Enk8Pq47F+lG7/IVT+kAcFA==", + "resolved": "git+https://git@github.com/elastic/kibana-buildkite-library.git#a5010e2b33291290dc45d2c0013b5d5646fbfa09", + "integrity": "sha512-JbHzLX3eEAlZ1uVFD7lU4pSV9j4onQvnyveUmn83PQn0NhBgUjP7WvqZ+rFQWy+X2oACML6tapLRQlkZW0yrzw==", "license": "MIT", "dependencies": { "@octokit/rest": "^18.10.0", "axios": "^0.21.4", "globby": "^11.1.0", - "js-yaml": "^4.1.0", - "minimatch": "^5.0.1" + "js-yaml": "^4.1.0" } }, "node_modules/merge2": { @@ -399,17 +385,6 @@ "node": ">=8.6" } }, - "node_modules/minimatch": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", - "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -709,24 +684,11 @@ "follow-redirects": "^1.14.0" } }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "before-after-hook": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "requires": { - "balanced-match": "^1.0.0" - } - }, "braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", @@ -839,15 +801,14 @@ } }, "kibana-buildkite-library": { - "version": "git+https://git@github.com/elastic/kibana-buildkite-library.git#a0037514b7650296a23dbad99b165601d4eab1be", - "integrity": "sha512-W9oH2c0q21IbO3sKJR2BkebhDlXVuWfqKO1r6T/E8/RRxCXJg/Wf073k8aDdpl1Enk8Pq47F+lG7/IVT+kAcFA==", - "from": "kibana-buildkite-library@git+https://git@github.com/elastic/kibana-buildkite-library#a0037514b7650296a23dbad99b165601d4eab1be", + "version": "git+https://git@github.com/elastic/kibana-buildkite-library.git#a5010e2b33291290dc45d2c0013b5d5646fbfa09", + "integrity": "sha512-JbHzLX3eEAlZ1uVFD7lU4pSV9j4onQvnyveUmn83PQn0NhBgUjP7WvqZ+rFQWy+X2oACML6tapLRQlkZW0yrzw==", + "from": "kibana-buildkite-library@git+https://git@github.com/elastic/kibana-buildkite-library#a5010e2b33291290dc45d2c0013b5d5646fbfa09", "requires": { "@octokit/rest": "^18.10.0", "axios": "^0.21.4", "globby": "^11.1.0", - "js-yaml": "^4.1.0", - "minimatch": "^5.0.1" + "js-yaml": "^4.1.0" } }, "merge2": { @@ -864,14 +825,6 @@ "picomatch": "^2.3.1" } }, - "minimatch": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", - "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", diff --git a/.buildkite/package.json b/.buildkite/package.json index 7f15a2fdf75bc27..2350a54a76af5ab 100644 --- a/.buildkite/package.json +++ b/.buildkite/package.json @@ -3,6 +3,6 @@ "version": "1.0.0", "private": true, "dependencies": { - "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a0037514b7650296a23dbad99b165601d4eab1be" + "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a5010e2b33291290dc45d2c0013b5d5646fbfa09" } } diff --git a/.buildkite/scripts/steps/code_coverage/ftr_configs.sh b/.buildkite/scripts/steps/code_coverage/ftr_configs.sh new file mode 100755 index 000000000000000..08e3db5065e96ed --- /dev/null +++ b/.buildkite/scripts/steps/code_coverage/ftr_configs.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh +.buildkite/scripts/build_kibana_plugins.sh + +is_test_execution_step + +export JOB_NUM=$BUILDKITE_PARALLEL_JOB +export JOB=ftr-configs-${JOB_NUM} +export CODE_COVERAGE=1 + +FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}" + +# a FTR failure will result in the script returning an exit code of 10 +exitCode=0 + +configs="${FTR_CONFIG:-}" + +if [[ "$configs" == "" ]]; then + echo "--- downloading ftr test run order" + buildkite-agent artifact download ftr_run_order.json . + configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json) +fi + +failedConfigs="" +results=() + +while read -r config; do + if [[ ! "$config" ]]; then + continue + fi + + echo "--- $ node scripts/functional_tests --config $config --exclude-tag ''skipCoverage''" + start=$(date +%s) + + # prevent non-zero exit code from breaking the loop + set +e + NODE_OPTIONS=--max_old_space_size=14336 \ + ./node_modules/.bin/nyc \ + --nycrc-path ./src/dev/code_coverage/nyc_config/nyc.server.config.js \ + node scripts/functional_tests \ + --config="$config" \ + --exclude-tag "skipCoverage" + lastCode=$? + set -e + + dasherize() { + withoutExtension=${1%.*} + dasherized=$(echo "$withoutExtension" | tr '\/' '\-') + } + dasherize $config + + serverAndClientSummary="target/kibana-coverage/functional/xpack-${dasherized}-server-coverage.json" + functionalSummary="target/kibana-coverage/functional/xpack-${dasherized}-coverage.json" + + # Server side and client side (server and public dirs) + if [[ -d "$KIBANA_DIR/target/kibana-coverage/server" ]]; then + echo "--- Server and Client side code coverage collected" + mkdir -p target/kibana-coverage/functional + mv target/kibana-coverage/server/coverage-final.json "$serverAndClientSummary" + fi + + # Each browser unload event, creates a new coverage file. + # So, we merge them here. + if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then + echo "--- Merging code coverage for FTR Config: $config" + yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json + rm -rf target/kibana-coverage/functional/* + mv target/kibana-coverage/functional-combined/coverage-final.json "$functionalSummary" + else + echo "--- Code coverage not found" + fi + + # Check for empty summary files. + empties=() + emptyCheck() { + echo "### Checking $1" + echo $(head -5 $1) | grep -E -i "pct.+Unknown" >/dev/null + lastCode=$? + if [ $lastCode -eq 0 ]; then + echo " --- Empty Summary File: $1" + empties+=($1) + fi + } + emptyCheck $serverAndClientSummary + emptyCheck $functionalSummary + + if [[ ${#empties[@]} -ge 2 ]]; then + echo "### Empty count = ${#empties[@]}, fail the build" + else + echo "### Empty count < 2, dont fail the build" + fi + + timeSec=$(($(date +%s) - start)) + if [[ $timeSec -gt 60 ]]; then + min=$((timeSec / 60)) + sec=$((timeSec - (min * 60))) + duration="${min}m ${sec}s" + else + duration="${timeSec}s" + fi + + results+=("- $config + duration: ${duration} + result: ${lastCode}") + + if [ $lastCode -ne 0 ]; then + exitCode=10 + echo "FTR exited with code $lastCode" + echo "^^^ +++" + + if [[ "$failedConfigs" ]]; then + failedConfigs="${failedConfigs}"$'\n'"$config" + else + failedConfigs="$config" + fi + fi +done <<<"$configs" + +if [[ "$failedConfigs" ]]; then + buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs" +fi + +echo "--- FTR configs complete" +printf "%s\n" "${results[@]}" +echo "" + +# Force exit 0 to ensure the next build step starts. +exit 0 diff --git a/.buildkite/scripts/steps/code_coverage/jest_integration.sh b/.buildkite/scripts/steps/code_coverage/jest_integration.sh index e35a4aa61cc0cfa..44a127e81c7195d 100755 --- a/.buildkite/scripts/steps/code_coverage/jest_integration.sh +++ b/.buildkite/scripts/steps/code_coverage/jest_integration.sh @@ -9,5 +9,7 @@ is_test_execution_step .buildkite/scripts/bootstrap.sh echo '--- Jest Integration code coverage' -node --max-old-space-size=14336 scripts/jest_integration --ci --coverage --coverageReporters json || true -mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json" \ No newline at end of file +checks-reporter-with-killswitch "Jest Integration Tests $((BUILDKITE_PARALLEL_JOB+1))" \ + .buildkite/scripts/steps/test/jest_parallel.sh jest.integration.config.js + + diff --git a/.buildkite/scripts/steps/code_coverage/jest_parallel.sh b/.buildkite/scripts/steps/code_coverage/jest_parallel.sh index dc8a67320c5ede5..6a6e4adc933fa24 100755 --- a/.buildkite/scripts/steps/code_coverage/jest_parallel.sh +++ b/.buildkite/scripts/steps/code_coverage/jest_parallel.sh @@ -1,38 +1,64 @@ #!/bin/bash -set -uo pipefail +set -euo pipefail -JOB=$BUILDKITE_PARALLEL_JOB -JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT +export JOB=$BUILDKITE_PARALLEL_JOB # a jest failure will result in the script returning an exit code of 10 - -i=0 exitCode=0 +results=() + +if [[ "$1" == 'jest.config.js' ]]; then + TEST_TYPE="unit" +else + TEST_TYPE="integration" +fi + +export TEST_TYPE +echo "--- downloading jest test run order" +buildkite-agent artifact download jest_run_order.json . +configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json) while read -r config; do - if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then - echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest" - node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" \ - --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest \ - --passWithNoTests || true - if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then - echo "Rename coverage-final.json to avoid overwrite" - mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json" - else - echo "Cannot find coverage-final.json" - fi - lastCode=$? - - if [ $lastCode -ne 0 ]; then - exitCode=10 - echo "Jest exited with code $lastCode" - echo "^^^ +++" - fi + echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest" + + start=$(date +%s) + + # prevent non-zero exit code from breaking the loop + set +e + NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest \ + --config="$config" --runInBand --ci --coverage \ + --coverageReporters json --passWithNoTests \ + --coverageDirectory target/kibana-coverage/jest + lastCode=$? + set -e + + if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then + echo "Rename coverage-final.json to avoid overwrite" + mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json" + else + echo "Cannot find coverage-final.json" + fi + + timeSec=$(($(date +%s) - start)) + if [[ $timeSec -gt 60 ]]; then + min=$((timeSec / 60)) + sec=$((timeSec - (min * 60))) + duration="${min}m ${sec}s" + else + duration="${timeSec}s" fi - ((i=i+1)) -# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode -done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)" + results+=("- $config + duration: ${duration} + result: ${lastCode}") + + if [ $lastCode -ne 0 ]; then + echo "Jest exited with code $lastCode" + echo "^^^ +++" + fi +done <<<"$configs" -exit $exitCode \ No newline at end of file +echo "--- Jest configs complete" +printf "%s\n" "${results[@]}" +echo ""