Bunch of changes

Performance tests were added (marckhouzam#15)
Add dirs that caused tests to fail (marckhouzam#18)
Fixed all other tests for zulu

Co-authored-by: Marc Khouzam <marc.khouzam@gmail.com>
Co-authored-by: =?UTF-8?q?Ville=20Skytt=C3=A4?= <ville.skytta@upcloud.com>

Makefile

@@ -1,20 +1,25 @@
 TESTPROG_DIR := $(CURDIR)/testprog
 
+TESTS := $(patsubst tests/Dockerfile.%,%,$(wildcard tests/Dockerfile.*))
+SHELLS := bash fish
+
+containing = $(foreach v,$2,$(if $(findstring $1,$v),$v))
+not-containing = $(foreach v,$2,$(if $(findstring $1,$v),,$v))
+
 .PHONY: all
-all:
-	@src/test-all.sh bash fish
+all: $(TESTS)
 
 .PHONY: build-linux
 build-linux:
 	@cd $(TESTPROG_DIR) && make build-linux
 
-.PHONY: bash
-bash:
-	@src/test-all.sh bash
+.PHONE: $(TESTS)
+$(TESTS): build-linux
+	@src/test-all.sh $@
 
-.PHONY: fish
-fish:
-	@src/test-all.sh fish
+.PHONY: $(SHELLS)
+$(SHELLS):
+	$(MAKE) $(call containing,$@,$(TESTS))
 
 .PHONY: test
 test: clean

src/comp-test-lib.bash

@@ -137,6 +137,22 @@ _completionTests_sort() {
   fi
 }
 
+# $1 - The completion to measure
+# $2 - The maximum time to consider it an error
+# $3 - An output prefix
+_completionTests_timing() {
+  TIMEFORMAT=%R
+  timing=$({ time { _completionTests_complete "$1" > /dev/null; } } 2>&1)
+  if (( $(echo "$timing > ${2}" | bc -l) )); then
+    _completionTests_TEST_FAILED=1
+    echo -e "${RED}<= TIMING => ${3}: 1000 completions took ${timing} seconds > ${2-0.1} seconds limit$NC"
+    return 1
+  else
+    echo -e "${GREEN}<= TIMING => ${3}: 1000 completions took ${timing} seconds < ${2-0.1} seconds limit$NC"
+    return 0
+  fi
+}
+
 # Find the completion function associated with the binary.
 # $1 is the first argument of the line to complete which allows
 # us to find the existing completion function name.
@@ -157,8 +173,7 @@ _completionTests_findCompletionFunction() {
 _completionTests_complete() {
   local cmdLine=$1
 
-  # Set the bash completion variables which are
-  # used for both bash and zsh completion
+  # Set the bash completion variables
   COMP_LINE=${cmdLine}
   COMP_POINT=${#COMP_LINE}
   COMP_TYPE=${COMP_TYPE-9} # 9 is TAB, but we allow to override for some tests
@@ -175,10 +190,16 @@ _completionTests_complete() {
   # to stderr.
   eval "$(_completionTests_findCompletionFunction "${COMP_WORDS[0]}")" 2>&1
 
-  # Return the result of the completion.
+  # Return the result of the call to the completion function.
+  # We separate each completion with a space and not a newline; using newlines
+  # was preventing us from detecting empty completions as newlines are stripped
+  # automatically by the sub-shell call to this function.
+  result="$(printf "%s " "${COMPREPLY[@]}")"
+  # remove the last space we inserted ourselves
+  result="${result% }"
   # We use printf instead of echo as the first completion could be -n which
   # would be interpreted as an argument to echo
-  printf "%s\n" "${COMPREPLY[@]}"
+  printf "%s" "${result}"
 }
 
 _completionTests_exit() {

src/comp-test-lib.fish

@@ -45,7 +45,7 @@ function _completionTests_verifyDebug
    set debugfile /tmp/comptests.fish.debug
    rm -f $debugfile
    set -g BASH_COMP_DEBUG_FILE $debugfile
-   _completionTests_verifyCompletion "testprog comp" "completion"
+   _completionTests_verifyCompletion "testprog dasharg --arg" ""
    if not test -s $debugfile
       # File should not be empty
       echo -e $RED"ERROR: No debug logs were printed to $debugfile$NC"

src/test-all.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -euo pipefail
+set -euxo pipefail
 
 # This script runs completion tests in different environments and different shells.
 
@@ -24,29 +24,35 @@ set +e
 GOT_FAILURE=0
 trap "GOT_FAILURE=1" ERR
 
+mapfile -d '' shellTypes < <(find "${BASE_DIR}" -name "comp-tests.*" -printf '%f\0' | cut -z -c 12-)
+getTestShellType() {
+  for shell in "${shellTypes[@]}"; do
+    if [[ $1 == *"-$shell-"* ]]; then
+      printf "%s" "$shell"
+      break
+    fi
+  done
+}
+
 declare -A test_cases=()
 
-for SHELL_TYPE in "$@"; do
-  mapfile -d '' tests < <(find "${BASE_DIR}/tests" -name "Dockerfile.*-${SHELL_TYPE}-*" -print0)
-  for testFile in "${tests[@]}"; do
-    filename="$(basename "$testFile")"
-    testName="${filename:11}" # strip Dockerfile.
+for testName in "$@"; do
+  testFile="${BASE_DIR}/tests/Dockerfile.${testName}"
 
-    imageName="comp-test:$SHELL_TYPE-$testName"
-    test_cases[$imageName]="$SHELL_TYPE"
+  imageName="comp-test:$testName"
+  test_cases[$imageName]="$(getTestShellType "$testName")"
 
-    (
-      exec > >(trap "" INT TERM; sed 's/^/'"$testName"': /')
-      exec 2> >(trap "" INT TERM; sed 's/^/'"$testName"': /' >&2)
-      $CONTAINER_ENGINE build -t "${imageName}" "${BASE_DIR}" -f "$testFile"
-    ) &
-  done
+  (
+    exec > >(trap "" INT TERM; sed 's/^/'"$testName"': /')
+    exec 2> >(trap "" INT TERM; sed 's/^/'"$testName"': /' >&2)
+    $CONTAINER_ENGINE build -t "${imageName}" "${BASE_DIR}" -f "$testFile"
+  ) &
 done
 
 wait
 
 for imageName in "${!test_cases[@]}"; do
-  shellType=${test_cases[$imageName]}
+  shellType="${test_cases[$imageName]}"
   "$CONTAINER_ENGINE" run --rm "${imageName}" "tests/comp-tests.$shellType"
 done
 # Indicate if anything failed during the run

testprog/go.mod

@@ -1,7 +1,14 @@
-module github.com/marckhouzam/cobra-completion-testing
+module github.com/gowarden/cobra-completion-testing
 
-go 1.14
+go 1.18
+
+require github.com/gowarden/zulu v1.1.3
+
+require (
+	github.com/gowarden/zflag v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+)
 
 replace github.com/gowarden/zulu => ../../zulu
 
-require github.com/gowarden/zulu v1.1.3
+replace github.com/gowarden/zflag => ../../zflag