From fc2f2678783103320e2a91e9ed57a12aafaef927 Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Sat, 13 Aug 2022 15:16:35 -0700 Subject: [PATCH] Add parallelism to functional testing (#4096) --- .circleci/config.yml | 22 +++++++++++++++--- .codecov.yml | 4 ++-- .yea_durations | 1 + tools/coverage-tool.py | 1 + tox.ini | 53 +++++++++++++++++++++++------------------- 5 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 .yea_durations diff --git a/.circleci/config.yml b/.circleci/config.yml index b15c7aa8611..479d4f40de2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -220,8 +220,12 @@ jobs: notify_on_failure: type: boolean default: false + parallelism: + type: integer + default: 1 docker: - image: "python:<>.<>" + parallelism: << parameters.parallelism >> resource_class: xlarge working_directory: /mnt/ramdisk steps: @@ -240,7 +244,7 @@ jobs: name: Run tests no_output_timeout: 10m command: | - tox -v -e << parameters.toxenv >> + CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" tox -v -e << parameters.toxenv >> - save-test-results # conditionally post a notification to slack if the job failed - when: @@ -1027,7 +1031,7 @@ workflows: name: "unit-s_base_mock_server-lin-py<><>" toxenv: "py<><>,covercircle" # - # Functional tests with yea on Linux + # Functional tests with yea on Linux (base only) # - tox-base: matrix: @@ -1036,6 +1040,18 @@ workflows: python_version_minor: [7] shard: - "base" + name: "func-s_<>-lin-py<><>" + toxenv: "func-s_<>-py<><>,func-covercircle" + parallelism: 4 + # + # Functional tests with yea on Linux + # + - tox-base: + matrix: + parameters: + python_version_major: [3] + python_version_minor: [7] + shard: - "sklearn" - "metaflow" - "tf115" @@ -1082,7 +1098,7 @@ workflows: python_version_minor: [9] name: "func-s_base-win-py<><>" toxenv: "func-s_base-py<><>" - parallelism: 1 + parallelism: 6 xdist: 1 # # Unit tests with pytest on Windows and MacOS, using real wandb server diff --git a/.codecov.yml b/.codecov.yml index 2ae8dc80855..c22e6bde78b 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -4,14 +4,14 @@ codecov: # To calculate after_n_builds use # ./tools/coverage-tool.py jobs | wc -l # also change comment block after_n_builds just below - after_n_builds: 33 + after_n_builds: 36 wait_for_ci: no comment: layout: "reach, diff, flags, files" behavior: default require_changes: no - after_n_builds: 33 + after_n_builds: 36 ignore: - "wandb/vendor" diff --git a/.yea_durations b/.yea_durations new file mode 100644 index 00000000000..02e6855a2bf --- /dev/null +++ b/.yea_durations @@ -0,0 +1 @@ +{"0.artifacts.link-artifact": 6.310358285903931, "0.artifacts.link-model-outside-run": 1.749068021774292, "0.artifacts.link-model": 6.807616710662842, "0.artifacts.log-artifact": 6.233159303665161, "0.artifacts.log-image-artifact-path": 9.815216541290283, "0.artifacts.log-model": 6.839049816131592, "0.artifacts.log-unicode-artifact": 6.296761512756348, "0.artifacts.public-link-model": 6.786419630050659, "0.artifacts.use-and-link-model": 6.771046161651611, "0.artifacts.use-model-error": 6.478207588195801, "0.artifacts.use-model-outside-run-error": 6.30046534538269, "0.artifacts.use-model": 6.773815155029297, "0.catboost.1": 8.337672710418701, "0.code_save.code_save_disabled.0-None": 6.283750772476196, "0.code_save.code_save_enabled": 6.306480884552002, "0.code_save.log_code.log_code": 8.311027526855469, "0.console.1": 17.30700922012329, "0.console.2": 15.320633888244629, "0.console.3": 9.287165403366089, "0.core.1": 5.32770848274231, "0.core.2": 6.323151111602783, "0.core.3": 7.28487491607666, "0.core.4": 5.282705545425415, "0.core.5": 5.323633193969727, "0.core.6": 6.263854265213013, "0.debug.1": 5.27385139465332, "0.debug.2": 8.303571701049805, "0.debug.3": 4.733357667922974, "0.debug.4": 5.304134130477905, "0.debug.5": 5.4722607135772705, "0.debug.6": 5.454894304275513, "0.fastai.1": 14.65076470375061, "0.jax.01-log-bfloat16": 8.949407815933228, "0.jobs.job_artifact_creation": 6.28191614151001, "0.lightgbm.1": 6.843756437301636, "0.login.1": 6.912102222442627, "0.login.2": 6.367185831069946, "0.magic.1": 12.075379610061646, "0.mp.06-5-share-child-non-service.0-spawn": 7.984593868255615, "0.mp.06-5-share-child-non-service.1-forkserver": 7.991549015045166, "0.mp.06-5-share-child-non-service.2-fork": 7.008553981781006, "0.offline.1": 5.094182729721069, "0.prodigy.prodigy_integration": 149.22574615478516, "0.profiler.profiler": 7.020074367523193, "0.sacred.1": 8.48173475265503, "0.save.save-dir": 6.528398036956787, "0.save.save-files": 9.792961597442627, "0.save.save-glob": 13.845168352127075, "0.sb3.1": 19.3912136554718, "0.settings.python-m": 6.280124664306641, "tests.standalone_tests.basic": 6.3352744579315186, "tests.standalone_tests.table_mem_test": 54.88446283340454, "tests.standalone_tests.tweets": 7.539431571960449, "0.stats.1": 10.30295705795288, "0.tensorflow.1": 5.352811336517334, "0.tensorflow.2": 8.014575004577637, "0.torch.1": 32.52781319618225, "0.xgboost.xgboost_clf": 8.04253339767456, "0.xgboost.xgboost_clf_old": 8.002850532531738, "0.xgboost.xgboost_reg": 10.378774166107178, "0.jobs.job_container_creation": 6.297541379928589} \ No newline at end of file diff --git a/tools/coverage-tool.py b/tools/coverage-tool.py index 74393bffaec..19a2c87c28e 100755 --- a/tools/coverage-tool.py +++ b/tools/coverage-tool.py @@ -170,6 +170,7 @@ def coverage_coveragerc_check(toxenv_list, args): paths = cf.get("paths", "canonicalsrc") paths = paths.split() + toxenv_list = list(set(toxenv_list)) toxenv_list.sort() # lets generate what paths should look like diff --git a/tox.ini b/tox.ini index 9babcc0943b..5e0bd4300e1 100644 --- a/tox.ini +++ b/tox.ini @@ -391,11 +391,14 @@ setenv = s_kfp: WB_PROBE_PACKAGE=true passenv = USERNAME + CI_PYTEST_SPLIT_ARGS deps = -r{toxinidir}/requirements.txt func-s_{base,metaflow}-py{36,37,38,39,310}: -r{toxinidir}/requirements_dev.txt pytest-mock - yea-wandb=={env:YEA_WANDB_VERSION} + https://github.com/wandb/yea/archive/add-split.zip + https://github.com/wandb/yea-wandb/archive/add-split.zip + ; yea-wandb=={env:YEA_WANDB_VERSION} extras = func-s_service-py{36,37,38,39,310}: service func-s_grpc-py{36,37,38,39,310}: grpc @@ -403,28 +406,28 @@ whitelist_externals = mkdir commands = mkdir -p test-results - func-s_base-py{36,37,38,39,310}: yea --strict --shard default run {posargs:--all} - func-s_sklearn-py{36,37,38,39,310}: yea --strict --shard sklearn run {posargs:--all} - func-s_metaflow-py{36,37,38,39,310}: yea --strict --shard metaflow run {posargs:--all} - func-s_tf115-py{36,37,38,39,310}: yea --strict --shard tf115 run {posargs:--all} - func-s_tf21-py{36,37,38,39,310}: yea --strict --shard tf21 run {posargs:--all} - func-s_tf25-py{36,37,38,39,310}: yea --strict --shard tf25 run {posargs:--all} - func-s_tf26-py{36,37,38,39,310}: yea --strict --shard tf26 run {posargs:--all} - func-s_ray112-py{36,37,38,39,310}: yea --strict --shard ray112 run {posargs:--all} - func-s_ray2-py{36,37,38,39,310}: yea --strict --shard ray2 run {posargs:--all} - func-s_service-py{36,37,38,39,310}: yea --strict --shard service run {posargs:--all} - func-s_grpc-py{36,37,38,39,310}: yea --strict --shard grpc run {posargs:--all} - func-s_py310-py{36,37,38,39,310}: yea --strict --shard py310 run {posargs:--all} - func-s_docs-py{36,37,38,39,310}: yea --strict --yeadoc --shard docs run {posargs:--all} - func-s_imports1-py{36,37,38,39,310}: yea --strict --shard imports1 run {posargs:--all} - func-s_imports2-py{36,37,38,39,310}: yea --strict --shard imports2 run {posargs:--all} - func-s_imports3-py{36,37,38,39,310}: yea --strict --shard imports3 run {posargs:--all} - func-s_imports4-py{36,37,38,39,310}: yea --strict --shard imports4 run {posargs:--all} - func-s_imports5-py{36,37,38,39,310}: yea --strict --shard imports5 run {posargs:--all} - func-s_imports6-py{36,37,38,39,310}: yea --strict --shard imports6 run {posargs:--all} - func-s_imports7-py{36,37,38,39,310}: yea --strict --shard imports7 run {posargs:--all} - func-s_noml-py{36,37,38,39,310}: yea --strict --shard noml run {posargs:--all} - func-s_kfp-py{37}: yea --strict -p wandb:mockserver-bind=0.0.0.0 -p wandb:mockserver-host=__auto__ --shard kfp run {posargs:--all} + func-s_base-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard default run {posargs:--all} + func-s_sklearn-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard sklearn run {posargs:--all} + func-s_metaflow-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard metaflow run {posargs:--all} + func-s_tf115-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf115 run {posargs:--all} + func-s_tf21-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf21 run {posargs:--all} + func-s_tf25-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf25 run {posargs:--all} + func-s_tf26-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard tf26 run {posargs:--all} + func-s_ray112-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard ray112 run {posargs:--all} + func-s_ray2-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard ray2 run {posargs:--all} + func-s_service-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard service run {posargs:--all} + func-s_grpc-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard grpc run {posargs:--all} + func-s_py310-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard py310 run {posargs:--all} + func-s_docs-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --yeadoc --shard docs run {posargs:--all} + func-s_imports1-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports1 run {posargs:--all} + func-s_imports2-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports2 run {posargs:--all} + func-s_imports3-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports3 run {posargs:--all} + func-s_imports4-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports4 run {posargs:--all} + func-s_imports5-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports5 run {posargs:--all} + func-s_imports6-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports6 run {posargs:--all} + func-s_imports7-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard imports7 run {posargs:--all} + func-s_noml-py{36,37,38,39,310}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict --shard noml run {posargs:--all} + func-s_kfp-py{37}: yea {env:CI_PYTEST_SPLIT_ARGS:} --strict -p wandb:mockserver-bind=0.0.0.0 -p wandb:mockserver-host=__auto__ --shard kfp run {posargs:--all} [testenv:func-cover] skip_install = true @@ -477,7 +480,9 @@ deps = -r{toxinidir}/requirements.txt standalone-{cpu,gpu,tpu,local}-py{36,37,38,39,310}: -r{toxinidir}/requirements_dev.txt pytest-mock - yea-wandb=={env:YEA_WANDB_VERSION} + https://github.com/wandb/yea/archive/add-split.zip + https://github.com/wandb/yea-wandb/archive/add-split.zip + ; yea-wandb=={env:YEA_WANDB_VERSION} whitelist_externals = date echo