[QA][Code Coverage] fixup for auto config handling

Add support for automatic ftr config groups.
Add dasherize fn for unique names.
Bump pipeline lib version.
Drop retry.
Force exit 0 to continue pipeline, so ingestion always happens.
Add dasherize fn for unique names
Add empty coverage summary file check.

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a0037514b7650296a23dbad99b165601d4eab1be"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#a5010e2b33291290dc45d2c0013b5d5646fbfa09"
   }
 }

.buildkite/scripts/steps/code_coverage/ftr_configs.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana_plugins.sh
+
+is_test_execution_step
+
+export JOB_NUM=$BUILDKITE_PARALLEL_JOB
+export JOB=ftr-configs-${JOB_NUM}
+export CODE_COVERAGE=1
+
+FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}"
+
+# a FTR failure will result in the script returning an exit code of 10
+exitCode=0
+
+configs="${FTR_CONFIG:-}"
+
+if [[ "$configs" == "" ]]; then
+  echo "--- downloading ftr test run order"
+  buildkite-agent artifact download ftr_run_order.json .
+  configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
+fi
+
+failedConfigs=""
+results=()
+
+while read -r config; do
+  if [[ ! "$config" ]]; then
+    continue
+  fi
+
+  echo "--- $ node scripts/functional_tests --config $config --exclude-tag ''skipCoverage''"
+  start=$(date +%s)
+
+  # prevent non-zero exit code from breaking the loop
+  set +e
+  NODE_OPTIONS=--max_old_space_size=14336 \
+    ./node_modules/.bin/nyc \
+    --nycrc-path ./src/dev/code_coverage/nyc_config/nyc.server.config.js \
+    node scripts/functional_tests \
+    --config="$config" \
+    --exclude-tag "skipCoverage"
+  lastCode=$?
+  set -e
+
+  dasherize() {
+    withoutExtension=${1%.*}
+    dasherized=$(echo "$withoutExtension" | tr '\/' '\-')
+  }
+  dasherize $config
+
+  serverAndClientSummary="target/kibana-coverage/functional/xpack-${dasherized}-server-coverage.json"
+  functionalSummary="target/kibana-coverage/functional/xpack-${dasherized}-coverage.json"
+
+  # Server side and client side (server and public dirs)
+  if [[ -d "$KIBANA_DIR/target/kibana-coverage/server" ]]; then
+    echo "--- Server and Client side code coverage collected"
+    mkdir -p target/kibana-coverage/functional
+    mv target/kibana-coverage/server/coverage-final.json "$serverAndClientSummary"
+  fi
+
+  # Each browser unload event, creates a new coverage file.
+  # So, we merge them here.
+  if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
+    echo "--- Merging code coverage for FTR Config: $config"
+    yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
+    rm -rf target/kibana-coverage/functional/*
+    mv target/kibana-coverage/functional-combined/coverage-final.json "$functionalSummary"
+  else
+    echo "--- Code coverage not found"
+  fi
+
+  # Check for empty summary files.
+  empties=()
+  emptyCheck() {
+    echo "### Checking $1"
+    echo $(head -5 $1) | grep -E -i "pct.+Unknown" >/dev/null
+    lastCode=$?
+    if [ $lastCode -eq 0 ]; then
+      echo "  --- Empty Summary File: $1"
+      empties+=($1)
+    fi
+  }
+  emptyCheck $serverAndClientSummary
+  emptyCheck $functionalSummary
+
+  if [[ ${#empties[@]} -ge 2 ]]; then
+    echo "### Empty count = ${#empties[@]}, fail the build"
+  else
+    echo "### Empty count < 2, dont fail the build"
+  fi
+
+  timeSec=$(($(date +%s) - start))
+  if [[ $timeSec -gt 60 ]]; then
+    min=$((timeSec / 60))
+    sec=$((timeSec - (min * 60)))
+    duration="${min}m ${sec}s"
+  else
+    duration="${timeSec}s"
+  fi
+
+  results+=("- $config
+    duration: ${duration}
+    result: ${lastCode}")
+
+  if [ $lastCode -ne 0 ]; then
+    exitCode=10
+    echo "FTR exited with code $lastCode"
+    echo "^^^ +++"
+
+    if [[ "$failedConfigs" ]]; then
+      failedConfigs="${failedConfigs}"$'\n'"$config"
+    else
+      failedConfigs="$config"
+    fi
+  fi
+done <<<"$configs"
+
+if [[ "$failedConfigs" ]]; then
+  buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
+fi
+
+echo "--- FTR configs complete"
+printf "%s\n" "${results[@]}"
+echo ""
+
+# Force exit 0 to ensure the next build step starts.
+exit 0

.buildkite/scripts/steps/code_coverage/jest_integration.sh

@@ -9,5 +9,7 @@ is_test_execution_step
 .buildkite/scripts/bootstrap.sh
 
 echo '--- Jest Integration code coverage'
-node --max-old-space-size=14336 scripts/jest_integration  --ci --coverage --coverageReporters json || true
-mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json"
+checks-reporter-with-killswitch "Jest Integration Tests $((BUILDKITE_PARALLEL_JOB+1))" \
+  .buildkite/scripts/steps/test/jest_parallel.sh jest.integration.config.js
+
+

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -1,38 +1,64 @@
 #!/bin/bash
 
-set -uo pipefail
+set -euo pipefail
 
-JOB=$BUILDKITE_PARALLEL_JOB
-JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+export JOB=$BUILDKITE_PARALLEL_JOB
 
 # a jest failure will result in the script returning an exit code of 10
-
-i=0
 exitCode=0
+results=()
+
+if [[ "$1" == 'jest.config.js' ]]; then
+  TEST_TYPE="unit"
+else
+  TEST_TYPE="integration"
+fi
+
+export TEST_TYPE
+echo "--- downloading jest test run order"
+buildkite-agent artifact download jest_run_order.json .
+configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
 
 while read -r config; do
-  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
-    echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
-    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" \
-      --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest \
-      --passWithNoTests || true
-    if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then
-      echo "Rename coverage-final.json to avoid overwrite"
-      mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
-    else
-      echo "Cannot find coverage-final.json"
-    fi
-    lastCode=$?
-
-    if [ $lastCode -ne 0 ]; then
-      exitCode=10
-      echo "Jest exited with code $lastCode"
-      echo "^^^ +++"
-    fi
+  echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
+
+  start=$(date +%s)
+
+  # prevent non-zero exit code from breaking the loop
+  set +e
+  NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest \
+    --config="$config" --runInBand --ci --coverage \
+    --coverageReporters json --passWithNoTests \
+    --coverageDirectory target/kibana-coverage/jest
+  lastCode=$?
+  set -e
+
+  if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then
+    echo "Rename coverage-final.json to avoid overwrite"
+    mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+  else
+    echo "Cannot find coverage-final.json"
+  fi
+
+  timeSec=$(($(date +%s) - start))
+  if [[ $timeSec -gt 60 ]]; then
+    min=$((timeSec / 60))
+    sec=$((timeSec - (min * 60)))
+    duration="${min}m ${sec}s"
+  else
+    duration="${timeSec}s"
   fi
 
-  ((i=i+1))
-# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
-done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
+  results+=("- $config
+    duration: ${duration}
+    result: ${lastCode}")
+
+  if [ $lastCode -ne 0 ]; then
+    echo "Jest exited with code $lastCode"
+    echo "^^^ +++"
+  fi
+done <<<"$configs"
 
-exit $exitCode
+echo "--- Jest configs complete"
+printf "%s\n" "${results[@]}"
+echo ""