Add parallelism to functional testing (#4096)

wandb · Aug 13, 2022 · fc2f267 · fc2f267
1 parent 5853e4e
commit fc2f267
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 29 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -220,8 +220,12 @@ jobs:
       notify_on_failure:
         type: boolean
         default: false
+      parallelism:
+        type: integer
+        default: 1
     docker:
       - image: "python:<<parameters.python_version_major>>.<<parameters.python_version_minor>>"
+    parallelism: << parameters.parallelism >>
     resource_class: xlarge
     working_directory: /mnt/ramdisk
     steps:
@@ -240,7 +244,7 @@ jobs:
           name: Run tests
           no_output_timeout: 10m
           command: |
-            tox -v -e << parameters.toxenv >>
+            CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" tox -v -e << parameters.toxenv >>
       - save-test-results
       # conditionally post a notification to slack if the job failed
       - when:
@@ -1027,7 +1031,7 @@ workflows:
           name: "unit-s_base_mock_server-lin-py<<matrix.python_version_major>><<matrix.python_version_minor>>"
           toxenv: "py<<matrix.python_version_major>><<matrix.python_version_minor>>,covercircle"
       #
-      # Functional tests with yea on Linux
+      # Functional tests with yea on Linux (base only)
       #
       - tox-base:
           matrix:
@@ -1036,6 +1040,18 @@ workflows:
               python_version_minor: [7]
               shard:
                 - "base"
+          name: "func-s_<<matrix.shard>>-lin-py<<matrix.python_version_major>><<matrix.python_version_minor>>"
+          toxenv: "func-s_<<matrix.shard>>-py<<matrix.python_version_major>><<matrix.python_version_minor>>,func-covercircle"
+          parallelism: 4
+      #
+      # Functional tests with yea on Linux
+      #
+      - tox-base:
+          matrix:
+            parameters:
+              python_version_major: [3]
+              python_version_minor: [7]
+              shard:
                 - "sklearn"
                 - "metaflow"
                 - "tf115"
@@ -1082,7 +1098,7 @@ workflows:
               python_version_minor: [9]
           name: "func-s_base-win-py<<matrix.python_version_major>><<matrix.python_version_minor>>"
           toxenv: "func-s_base-py<<matrix.python_version_major>><<matrix.python_version_minor>>"
-          parallelism: 1
+          parallelism: 6
           xdist: 1
       #
       # Unit tests with pytest on Windows and MacOS, using real wandb server

diff --git a/.codecov.yml b/.codecov.yml
@@ -4,14 +4,14 @@ codecov:
     # To calculate after_n_builds use
     # ./tools/coverage-tool.py jobs | wc -l
     # also change comment block after_n_builds just below
-    after_n_builds: 33
+    after_n_builds: 36
     wait_for_ci: no
 
 comment:
   layout: "reach, diff, flags, files"
   behavior: default
   require_changes: no
-  after_n_builds: 33
+  after_n_builds: 36
 
 ignore:
   - "wandb/vendor"

diff --git a/.yea_durations b/.yea_durations
@@ -0,0 +1 @@
+{"0.artifacts.link-artifact": 6.310358285903931, "0.artifacts.link-model-outside-run": 1.749068021774292, "0.artifacts.link-model": 6.807616710662842, "0.artifacts.log-artifact": 6.233159303665161, "0.artifacts.log-image-artifact-path": 9.815216541290283, "0.artifacts.log-model": 6.839049816131592, "0.artifacts.log-unicode-artifact": 6.296761512756348, "0.artifacts.public-link-model": 6.786419630050659, "0.artifacts.use-and-link-model": 6.771046161651611, "0.artifacts.use-model-error": 6.478207588195801, "0.artifacts.use-model-outside-run-error": 6.30046534538269, "0.artifacts.use-model": 6.773815155029297, "0.catboost.1": 8.337672710418701, "0.code_save.code_save_disabled.0-None": 6.283750772476196, "0.code_save.code_save_enabled": 6.306480884552002, "0.code_save.log_code.log_code": 8.311027526855469, "0.console.1": 17.30700922012329, "0.console.2": 15.320633888244629, "0.console.3": 9.287165403366089, "0.core.1": 5.32770848274231, "0.core.2": 6.323151111602783, "0.core.3": 7.28487491607666, "0.core.4": 5.282705545425415, "0.core.5": 5.323633193969727, "0.core.6": 6.263854265213013, "0.debug.1": 5.27385139465332, "0.debug.2": 8.303571701049805, "0.debug.3": 4.733357667922974, "0.debug.4": 5.304134130477905, "0.debug.5": 5.4722607135772705, "0.debug.6": 5.454894304275513, "0.fastai.1": 14.65076470375061, "0.jax.01-log-bfloat16": 8.949407815933228, "0.jobs.job_artifact_creation": 6.28191614151001, "0.lightgbm.1": 6.843756437301636, "0.login.1": 6.912102222442627, "0.login.2": 6.367185831069946, "0.magic.1": 12.075379610061646, "0.mp.06-5-share-child-non-service.0-spawn": 7.984593868255615, "0.mp.06-5-share-child-non-service.1-forkserver": 7.991549015045166, "0.mp.06-5-share-child-non-service.2-fork": 7.008553981781006, "0.offline.1": 5.094182729721069, "0.prodigy.prodigy_integration": 149.22574615478516, "0.profiler.profiler": 7.020074367523193, "0.sacred.1": 8.48173475265503, "0.save.save-dir": 6.528398036956787, "0.save.save-files": 9.792961597442627, "0.save.save-glob": 13.845168352127075, "0.sb3.1": 19.3912136554718, "0.settings.python-m": 6.280124664306641, "tests.standalone_tests.basic": 6.3352744579315186, "tests.standalone_tests.table_mem_test": 54.88446283340454, "tests.standalone_tests.tweets": 7.539431571960449, "0.stats.1": 10.30295705795288, "0.tensorflow.1": 5.352811336517334, "0.tensorflow.2": 8.014575004577637, "0.torch.1": 32.52781319618225, "0.xgboost.xgboost_clf": 8.04253339767456, "0.xgboost.xgboost_clf_old": 8.002850532531738, "0.xgboost.xgboost_reg": 10.378774166107178, "0.jobs.job_container_creation": 6.297541379928589}
diff --git a/tools/coverage-tool.py b/tools/coverage-tool.py
@@ -170,6 +170,7 @@ def coverage_coveragerc_check(toxenv_list, args):
     paths = cf.get("paths", "canonicalsrc")
     paths = paths.split()
 
+    toxenv_list = list(set(toxenv_list))
     toxenv_list.sort()
 
     # lets generate what paths should look like

diff --git a/tox.ini b/tox.ini
@@ -391,40 +391,43 @@ setenv =
     s_kfp: WB_PROBE_PACKAGE=true
 passenv =
     USERNAME
+    CI_PYTEST_SPLIT_ARGS
 deps =
     -r{toxinidir}/requirements.txt
     func-s_{base,metaflow}-py{36,37,38,39,310}: -r{toxinidir}/requirements_dev.txt
     pytest-mock
-    yea-wandb=={env:YEA_WANDB_VERSION}
+    https://github.com/wandb/yea/archive/add-split.zip
+    https://github.com/wandb/yea-wandb/archive/add-split.zip
+    ; yea-wandb=={env:YEA_WANDB_VERSION}
 extras =
     func-s_service-py{36,37,38,39,310}: service
     func-s_grpc-py{36,37,38,39,310}: grpc
 whitelist_externals =
     mkdir
 commands =
     mkdir -p test-results
-    func-s_base-py{36,37,38,39,310}: yea --strict --shard default run {posargs:--all}
-    func-s_sklearn-py{36,37,38,39,310}: yea --strict --shard sklearn run {posargs:--all}
-    func-s_metaflow-py{36,37,38,39,310}: yea --strict --shard metaflow run {posargs:--all}
-    func-s_tf115-py{36,37,38,39,310}: yea --strict --shard tf115 run {posargs:--all}
-    func-s_tf21-py{36,37,38,39,310}: yea --strict --shard tf21 run {posargs:--all}
-    func-s_tf25-py{36,37,38,39,310}: yea --strict --shard tf25 run {posargs:--all}
-    func-s_tf26-py{36,37,38,39,310}: yea --strict --shard tf26 run {posargs:--all}
-    func-s_ray112-py{36,37,38,39,310}: yea --strict --shard ray112 run {posargs:--all}
-    func-s_ray2-py{36,37,38,39,310}: yea --strict --shard ray2 run {posargs:--all}
-    func-s_service-py{36,37,38,39,310}: yea --strict --shard service run {posargs:--all}
-    func-s_grpc-py{36,37,38,39,310}: yea --strict --shard grpc run {posargs:--all}
-    func-s_py310-py{36,37,38,39,310}: yea --strict --shard py310 run {posargs:--all}
-    func-s_docs-py{36,37,38,39,310}: yea --strict --yeadoc --shard docs run {posargs:--all}
-    func-s_imports1-py{36,37,38,39,310}: yea --strict --shard imports1 run {posargs:--all}
-    func-s_imports2-py{36,37,38,39,310}: yea --strict --shard imports2 run {posargs:--all}
-    func-s_imports3-py{36,37,38,39,310}: yea --strict --shard imports3 run {posargs:--all}
-    func-s_imports4-py{36,37,38,39,310}: yea --strict --shard imports4 run {posargs:--all}
-    func-s_imports5-py{36,37,38,39,310}: yea --strict --shard imports5 run {posargs:--all}
-    func-s_imports6-py{36,37,38,39,310}: yea --strict --shard imports6 run {posargs:--all}
-    func-s_imports7-py{36,37,38,39,310}: yea --strict --shard imports7 run {posargs:--all}
-    func-s_noml-py{36,37,38,39,310}: yea --strict --shard noml run {posargs:--all}
-    func-s_kfp-py{37}: yea --strict -p wandb:mockserver-bind=0.0.0.0 -p wandb:mockserver-host=__auto__ --shard kfp run {posargs:--all}
+    func-s_base-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard default run {posargs:--all}
+    func-s_sklearn-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard sklearn run {posargs:--all}
+    func-s_metaflow-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard metaflow run {posargs:--all}
+    func-s_tf115-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf115 run {posargs:--all}
+    func-s_tf21-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf21 run {posargs:--all}
+    func-s_tf25-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf25 run {posargs:--all}
+    func-s_tf26-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf26 run {posargs:--all}
+    func-s_ray112-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard ray112 run {posargs:--all}
+    func-s_ray2-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard ray2 run {posargs:--all}
+    func-s_service-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard service run {posargs:--all}
+    func-s_grpc-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard grpc run {posargs:--all}
+    func-s_py310-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard py310 run {posargs:--all}
+    func-s_docs-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --yeadoc --shard docs run {posargs:--all}
+    func-s_imports1-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports1 run {posargs:--all}
+    func-s_imports2-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports2 run {posargs:--all}
+    func-s_imports3-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports3 run {posargs:--all}
+    func-s_imports4-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports4 run {posargs:--all}
+    func-s_imports5-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports5 run {posargs:--all}
+    func-s_imports6-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports6 run {posargs:--all}
+    func-s_imports7-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports7 run {posargs:--all}
+    func-s_noml-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard noml run {posargs:--all}
+    func-s_kfp-py{37}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict -p wandb:mockserver-bind=0.0.0.0 -p wandb:mockserver-host=__auto__ --shard kfp run {posargs:--all}
 
 [testenv:func-cover]
 skip_install = true
@@ -477,7 +480,9 @@ deps =
     -r{toxinidir}/requirements.txt
     standalone-{cpu,gpu,tpu,local}-py{36,37,38,39,310}: -r{toxinidir}/requirements_dev.txt
     pytest-mock
-    yea-wandb=={env:YEA_WANDB_VERSION}
+    https://github.com/wandb/yea/archive/add-split.zip
+    https://github.com/wandb/yea-wandb/archive/add-split.zip
+    ; yea-wandb=={env:YEA_WANDB_VERSION}
 whitelist_externals =
     date
     echo