From 0b7943cf1ad2c8676a649433c8ebcfd0a832f90a Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 30 Sep 2019 12:33:40 -0400 Subject: [PATCH 1/4] Add support for ruamel.yaml --- myia/utils/serialize.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/myia/utils/serialize.py b/myia/utils/serialize.py index 32a3bbef0..076ad1343 100644 --- a/myia/utils/serialize.py +++ b/myia/utils/serialize.py @@ -1,9 +1,34 @@ """Serialization utilities for graphs and their properties.""" -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: # pragma: no cover - from yaml import SafeLoader, SafeDumper +# This is a mess of imports + +# 1) We prefer ruamel.yaml since it has merged the large files fix. +# 2) Conda has a different name for it since it doesn't like namespaces. +# 3) We attempt to use pyyaml as a fallback +try: # pragma: no cover + try: + from ruamel.yaml import ( + CSafeLoader as SafeLoader, + CSafeDumper as SafeDumper + ) + except ImportError: + try: + from ruamel_yaml import ( + CSafeLoader as SafeLoader, + CSafeDumper as SafeDumper + ) + except ImportError: + from yaml import ( + CSafeLoader as SafeLoader, + CSafeDumper as SafeDumper + ) +except ImportError as e: # pragma: no cover + raise RuntimeError(""" +Couldn't find a C-backed version of yaml. + +Please install either ruamel.yaml or PyYAML with the C extension. The python + versions are just too slow to work properly. +""") from e import codecs import os From 2d848aa2ef7d2c70c3d26aa5735d8220f1edc1d8 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 30 Sep 2019 13:25:26 -0400 Subject: [PATCH 2/4] Change requirements for the yaml backend --- requirements-cpu.conda | 2 +- requirements-gpu.conda | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-cpu.conda b/requirements-cpu.conda index f92c871b0..9fc0daa11 100644 --- a/requirements-cpu.conda +++ b/requirements-cpu.conda @@ -4,7 +4,7 @@ numpy abergeron::tvm==0.6dev+3.* pytest pytest-cov -pyyaml +ruamel_yaml flake8==3.7.7 pytorch::pytorch==1.1.0 pydocstyle==2.1.1 diff --git a/requirements-gpu.conda b/requirements-gpu.conda index e93ffb2a2..fc330f868 100644 --- a/requirements-gpu.conda +++ b/requirements-gpu.conda @@ -5,7 +5,7 @@ abergeron/label/cuda::tvm-libs abergeron::tvm==0.6dev+3.* pytest pytest-cov -pyyaml +ruamel_yaml flake8==3.7.7 pytorch::pytorch==1.1.0 pydocstyle==2.1.1 From a198e4554f031fa039329f88604325e55268db64 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 30 Sep 2019 13:50:33 -0400 Subject: [PATCH 3/4] Use a conda env to install ruamel.yaml from pip --- .travis.yml | 2 +- Jenkinsfile | 17 +++++++++-------- ci-install.sh | 2 ++ requirements-cpu.conda | 2 +- requirements-gpu.conda | 2 +- requirements.txt | 2 ++ setup.py | 2 +- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93e9fab10..23ea43fc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ env: - TEST_SUITE=unit script: - if [[ $TEST_SUITE == "static" ]]; then flake8 && pydocstyle myia && isort -c -df; fi - - if [[ $TEST_SUITE == "unit" ]]; then pytest tests --cov=./ --cov-report term-missing; fi + - if [[ $TEST_SUITE == "unit" ]]; then source activate test && pytest tests --cov=./ --cov-report term-missing; fi after_success: - if [[ $TEST_SUITE == "unit" ]]; then codecov; fi diff --git a/Jenkinsfile b/Jenkinsfile index cc040055d..b396e5434 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,19 +7,20 @@ node ('gpu') { } try { stage ('Test') { - sh script: '$HOME/miniconda/bin/pytest --cov=./ --cov-report= --gpu --junit-xml test-report.xml' + sh script: 'source activate test && pytest --cov=./ --cov-report= --gpu --junit-xml test-report.xml' } } finally { junit 'test-report.xml' } stage ('Coverage') { - withEnv(['PATH+CONDA=/home/jenkins/miniconda/bin']) { - sh script: './cov.sh' - sh script: 'coverage xml' - sh script: 'pip install codecov' - withCredentials([string(credentialsId: 'myia_codecov', variable: 'CODECOV_TOKEN')]) { - sh script: 'codecov' - } + sh script: """ +source activate test && +./cov.sh && +coverage xml +""" + sh script: '$HOME/miniconda/bin/pip install codecov' + withCredentials([string(credentialsId: 'myia_codecov', variable: 'CODECOV_TOKEN')]) { + sh script: '$HOME/miniconda/bin/codecov' } } } diff --git a/ci-install.sh b/ci-install.sh index 737ff04fa..70fc12103 100755 --- a/ci-install.sh +++ b/ci-install.sh @@ -23,6 +23,8 @@ export PATH="$HOME/miniconda/bin:$PATH" hash -r conda config --set always_yes yes --set changeps1 no conda update -q conda +conda create -y -n test python=3.7 +source activate test conda install --file=requirements-$DEV.conda pip install -r requirements.txt pip install -e . --no-deps diff --git a/requirements-cpu.conda b/requirements-cpu.conda index 9fc0daa11..95ea8171e 100644 --- a/requirements-cpu.conda +++ b/requirements-cpu.conda @@ -4,7 +4,7 @@ numpy abergeron::tvm==0.6dev+3.* pytest pytest-cov -ruamel_yaml +yaml flake8==3.7.7 pytorch::pytorch==1.1.0 pydocstyle==2.1.1 diff --git a/requirements-gpu.conda b/requirements-gpu.conda index fc330f868..209fc50aa 100644 --- a/requirements-gpu.conda +++ b/requirements-gpu.conda @@ -5,7 +5,7 @@ abergeron/label/cuda::tvm-libs abergeron::tvm==0.6dev+3.* pytest pytest-cov -ruamel_yaml +yaml flake8==3.7.7 pytorch::pytorch==1.1.0 pydocstyle==2.1.1 diff --git a/requirements.txt b/requirements.txt index 5467c8d86..acbdb5351 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ asttokens codecov prettyprinter +ruamel.yaml +ruamel.yaml.clib>=0.2 diff --git a/setup.py b/setup.py index 7feedee90..f38ff008d 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ ], packages=find_packages(exclude=['docs', 'tests']), install_requires=['asttokens', 'colorama', 'prettyprinter', - 'numpy', 'pyyaml'], + 'numpy', 'ruamel.yaml'], extras_require={ 'test': ['flake8', 'pytest', 'codecov', 'isort', 'pytest-cov', 'pydocstyle', 'docopt'], From 32a621ccbb9d202180eacb3c5c63c446a76016d2 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 30 Sep 2019 13:54:08 -0400 Subject: [PATCH 4/4] Fix env actiavtion --- .travis.yml | 2 +- Jenkinsfile | 4 ++-- ci-install.sh | 1 + requirements-cpu.conda | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23ea43fc3..e3aafa685 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ env: - TEST_SUITE=static - TEST_SUITE=unit script: - - if [[ $TEST_SUITE == "static" ]]; then flake8 && pydocstyle myia && isort -c -df; fi + - if [[ $TEST_SUITE == "static" ]]; then source activate test && flake8 && pydocstyle myia && isort -c -df; fi - if [[ $TEST_SUITE == "unit" ]]; then source activate test && pytest tests --cov=./ --cov-report term-missing; fi after_success: - if [[ $TEST_SUITE == "unit" ]]; then codecov; fi diff --git a/Jenkinsfile b/Jenkinsfile index b396e5434..077350b60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,14 +7,14 @@ node ('gpu') { } try { stage ('Test') { - sh script: 'source activate test && pytest --cov=./ --cov-report= --gpu --junit-xml test-report.xml' + sh script: '. $HOME/miniconda/bin/activate test && pytest --cov=./ --cov-report= --gpu --junit-xml test-report.xml' } } finally { junit 'test-report.xml' } stage ('Coverage') { sh script: """ -source activate test && +. $HOME/miniconda/bin/activate test && ./cov.sh && coverage xml """ diff --git a/ci-install.sh b/ci-install.sh index 70fc12103..c9da0b736 100755 --- a/ci-install.sh +++ b/ci-install.sh @@ -24,6 +24,7 @@ hash -r conda config --set always_yes yes --set changeps1 no conda update -q conda conda create -y -n test python=3.7 +conda init source activate test conda install --file=requirements-$DEV.conda pip install -r requirements.txt diff --git a/requirements-cpu.conda b/requirements-cpu.conda index 95ea8171e..af468236b 100644 --- a/requirements-cpu.conda +++ b/requirements-cpu.conda @@ -6,7 +6,7 @@ pytest pytest-cov yaml flake8==3.7.7 -pytorch::pytorch==1.1.0 +pytorch::pytorch-cpu==1.1.0 pydocstyle==2.1.1 isort==4.3.21 docopt==0.6.2