Merge pull request #16260 from carlosmiei/optimize-tests

Travis :: Optimize tests

.travis.yml

@@ -38,11 +38,8 @@ script:
 - npm run test-base-ws
 - pip list
 - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
-  node run-tests --js --python-async --php-async 5 2>&1 &
-  rest_pid=$!;
-  node run-tests-ws --js --python-async --php-async 5 2>&1 &
-  ws_pid=$!;
-  wait $rest_pid && wait $ws_pid; fi
+  ./tests-manager.sh 2>&1;
+  fi
 - git checkout master composer.json
 - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then
   env COMMIT_MESSAGE=${NPM_VERSION:1} GITHUB_TOKEN=${GITHUB_TOKEN} ./build/push.sh;

build/push.sh

@@ -39,7 +39,7 @@ echo "Pushing generated files back to GitHub..."
 LAST_COMMIT_MESSAGE="$(git log --no-merges -1 --pretty=%B)"
 git config --global user.email "travis@travis-ci.org"
 git config --global user.name "Travis CI"
-git add php/*.php php/async/*.php php/pro/*.php php/test/*.php python/ccxt/test/*.py python/ccxt/async_support/*.py python/ccxt/*.py python/ccxt/pro/*.py python/ccxt/pro/test/*.py python/ccxt/pro/test/exchange/*.py wiki/*
+git add last-run.txt php/*.php php/async/*.php php/pro/*.php php/test/*.php python/ccxt/test/*.py python/ccxt/async_support/*.py python/ccxt/*.py python/ccxt/pro/*.py python/ccxt/pro/test/*.py python/ccxt/pro/test/exchange/*.py wiki/*
 git add -f python/LICENSE.txt python/package.json python/README.md
 git commit -a -m "${COMMIT_MESSAGE}" -m '[ci skip]'
 git tag -a "${COMMIT_MESSAGE}" -m "${LAST_COMMIT_MESSAGE}" -m "" -m "[ci skip]"

js/bybit.js

@@ -1215,7 +1215,7 @@ module.exports = class bybit extends Exchange {
         //             "nextPageCursor": ""
         //         },
         //         "retExtInfo": {},
-        //         "time": 1667533491917
+        //         "time": 1667533491916
         //     }
         //
         // option response

last-run.txt

@@ -0,0 +1 @@
+1672336002

python/fastflake.sh

@@ -2,12 +2,5 @@
 
 # saves time over the other running flake8 on all the files
 
-files=$(git diff --name-only --relative origin/master | sed -n '/py$/p' | xargs)
-command="flake8 --ignore=F722,F841,F821,W504,E402,E501,E275,E902 --exclude static_dependencies,node_modules,.tox"
-execute="${command} ${files}"
-printf "\n${execute}\n\n"
-
-if [[ -n ${files} ]]; then
-  # only one variable expansion happens here so it is safe
-  ${execute}
-fi
+echo -e '\n checking syntax using flake8 over the following files:\n'
+git diff --name-only --relative origin/master | sed -n '/py$/p' | tee /dev/stderr | xargs -n 1000 flake8 --ignore=F722,F841,F821,W504,E402,E501,E275,E902 --exclude static_dependencies,node_modules,.tox

tests-manager.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+if [ $# -gt 0 ]; then
+  echo "E2BIG: too many arguments"
+  exit 7
+fi
+
+timestamp_file="./last-run.txt"
+if ! [ -f "$timestamp_file" ]; then
+  echo "ENOENT: file $timestamp_file not found"
+  exit 2
+fi
+
+now=$(date -u +%s)
+last_run=$(< $timestamp_file)
+delta=$((now - last_run))
+six_hours=$((60 * 60 * 6))
+diff=$(git diff master --name-only)
+
+function run_tests {
+  local rest_args=
+  local ws_args=
+  if [ $# -eq 2 ]; then
+    rest_args="$1"
+    ws_args="$2"
+    if [ -z "$rest_args" ]; then
+      : &
+      local rest_pid=$!
+    fi
+    if [ -z "$ws_args" ]; then
+      : &
+      local ws_pid=$!
+    fi
+  fi
+  if [ -z "$rest_pid" ]; then
+    # shellcheck disable=SC2086
+    node run-tests --js --python-async --php-async $rest_args &
+    local rest_pid=$!
+  fi
+  if [ -z "$ws_pid" ]; then
+    # shellcheck disable=SC2086
+    node run-tests-ws --js --python-async --php-async $ws_args &
+    local ws_pid=$!
+  fi
+  wait $rest_pid && wait $ws_pid
+}
+
+if [ "$delta" -gt $six_hours ] || grep -q -E 'Exchange.php|/test|/base|^build|static_dependencies|^run-tests' <<< "$diff"; then
+  run_tests && date -u +%s > "$timestamp_file"
+else
+  run_tests "$(sed -E -n 's:^js/([^/]+)\.js$:\1:p' <<< "$diff" | xargs)" "$(sed -E -n 's:^js/pro/([^/]+)\.js$:\1:p' <<< "$diff" | xargs)"
+fi