From 4234fe765e996dbb3a1738567299ebb4d8c28af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Tue, 20 Sep 2022 14:00:51 +0200 Subject: [PATCH] fix: consistently use text output in conda shell commands and various little fixes for failing test cases due to conda package changes (#1864) * fix: consistently use text output in conda shell commands * add cwl-utils * fix: do not use nbformat 5.5 as it seems to have a bug during notebook validation * special test env for windows, as cwl-utils is not available there. * fix win env --- .github/workflows/main.yml | 2 +- snakemake/deployment/conda.py | 35 ++++++++++++++-------------- test-environment.yml | 1 + tests/test_jupyter_notebook/env.yaml | 1 + 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2bc1c19ea..b9fa4dfbb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -216,7 +216,7 @@ jobs: shell: python run: | import fileinput - excluded_on_win = ["environment-modules", "cwltool"] + excluded_on_win = ["environment-modules", "cwltool", "cwl-utils"] for line in fileinput.input("test-environment.yml", inplace=True): if all(pkg not in line for pkg in excluded_on_win): print(line) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index a80a88a19..cf5ebc451 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -130,7 +130,7 @@ def _get_content(self): content = shell.check_output( "conda env export {}".format(self.address_argument), stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, ) return content.encode() else: @@ -295,7 +295,7 @@ def create_archive(self): out = shell.check_output( "conda list --explicit {}".format(self.address_argument), stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, ) logger.debug(out) except subprocess.CalledProcessError as e: @@ -361,6 +361,7 @@ def execute_deployment_script(self, env_file, deploy_file): shell.check_output( self.conda.shellcmd(self.address, f"{interpreter} {deploy_file}"), stderr=subprocess.STDOUT, + text=True, ) def create(self, dryrun=False): @@ -417,13 +418,14 @@ def create(self, dryrun=False): quiet=True, ), stderr=subprocess.PIPE, + text=True, ) except subprocess.CalledProcessError as e: raise WorkflowError( "Unable to find environment in container image. " "Maybe a conda environment was modified without containerizing again " "(see snakemake --containerize)?\nDetails:\n{}\n{}".format( - e, e.stderr.decode() + e, e.stderr ) ) return env_path @@ -503,9 +505,7 @@ def create(self, dryrun=False): args=self._singularity_args, envvars=self.get_singularity_envvars(), ) - out = shell.check_output( - cmd, stderr=subprocess.STDOUT, universal_newlines=True - ) + out = shell.check_output(cmd, stderr=subprocess.STDOUT, text=True) else: def create_env(env_file, filetype="yaml"): @@ -550,18 +550,20 @@ def create_env(env_file, filetype="yaml"): envvars=self.get_singularity_envvars(), ) out = shell.check_output( - cmd, stderr=subprocess.STDOUT, universal_newlines=True + cmd, stderr=subprocess.STDOUT, text=True ) # cleanup if requested if self._cleanup is CondaCleanupMode.tarballs: logger.info("Cleaning up conda package tarballs.") - shell.check_output("conda clean -y --tarballs") + shell.check_output("conda clean -y --tarballs", text=True) elif self._cleanup is CondaCleanupMode.cache: logger.info( "Cleaning up conda package tarballs and package cache." ) - shell.check_output("conda clean -y --tarballs --packages") + shell.check_output( + "conda clean -y --tarballs --packages", text=True + ) return out if pin_file is not None: @@ -662,10 +664,7 @@ def __init__( self.frontend = frontend self.info = json.loads( - shell.check_output( - self._get_cmd(f"conda info --json"), - universal_newlines=True, - ) + shell.check_output(self._get_cmd(f"conda info --json"), text=True) ) if prefix_path is None or container_img is not None: @@ -706,7 +705,9 @@ def _check(self): locate_cmd = f"where {frontend}" try: - shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT) + shell.check_output( + self._get_cmd(locate_cmd), stderr=subprocess.STDOUT, text=True + ) except subprocess.CalledProcessError as e: if self.container_img: msg = ( @@ -757,9 +758,7 @@ def _check_version(self): from snakemake.shell import shell version = shell.check_output( - self._get_cmd("conda --version"), - stderr=subprocess.PIPE, - universal_newlines=True, + self._get_cmd("conda --version"), stderr=subprocess.PIPE, text=True ) version_matches = re.findall("\d+.\d+.\d+", version) if len(version_matches) != 1: @@ -783,7 +782,7 @@ def _check_condarc(self): res = json.loads( shell.check_output( self._get_cmd("conda config --get channel_priority --json"), - universal_newlines=True, + text=True, stderr=subprocess.PIPE, ) ) diff --git a/test-environment.yml b/test-environment.yml index 24f2a668f..5c2cd9e59 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -41,6 +41,7 @@ dependencies: - appdirs - python-irodsclient <1.1.2 # bug in 1.1.2 leads to KeyError - cwltool + - cwl-utils - jsonschema - pandas - python-kubernetes diff --git a/tests/test_jupyter_notebook/env.yaml b/tests/test_jupyter_notebook/env.yaml index 7c96e3b1d..5657edc42 100644 --- a/tests/test_jupyter_notebook/env.yaml +++ b/tests/test_jupyter_notebook/env.yaml @@ -4,3 +4,4 @@ channels: dependencies: - python >=3.5 - jupyter + - nbformat <5.5 # there is a bug in 5.5 that leads to a crash during notebook validation