fix: consistently use text output in conda shell commands and various…

… little fixes for failing test cases due to conda package changes (#1864)

* fix: consistently use text output in conda shell commands

* add cwl-utils

* fix: do not use nbformat 5.5 as it seems to have a bug during notebook validation

* special test env for windows, as cwl-utils is not available there.

* fix win env

.github/workflows/main.yml

@@ -216,7 +216,7 @@ jobs:
         shell: python
         run: |
           import fileinput
-          excluded_on_win = ["environment-modules", "cwltool"]
+          excluded_on_win = ["environment-modules", "cwltool", "cwl-utils"]
           for line in fileinput.input("test-environment.yml", inplace=True):
               if all(pkg not in line for pkg in excluded_on_win):
                   print(line)

snakemake/deployment/conda.py

@@ -130,7 +130,7 @@ def _get_content(self):
             content = shell.check_output(
                 "conda env export {}".format(self.address_argument),
                 stderr=subprocess.STDOUT,
-                universal_newlines=True,
+                text=True,
             )
             return content.encode()
         else:
@@ -295,7 +295,7 @@ def create_archive(self):
                 out = shell.check_output(
                     "conda list --explicit {}".format(self.address_argument),
                     stderr=subprocess.STDOUT,
-                    universal_newlines=True,
+                    text=True,
                 )
                 logger.debug(out)
             except subprocess.CalledProcessError as e:
@@ -361,6 +361,7 @@ def execute_deployment_script(self, env_file, deploy_file):
         shell.check_output(
             self.conda.shellcmd(self.address, f"{interpreter} {deploy_file}"),
             stderr=subprocess.STDOUT,
+            text=True,
         )
 
     def create(self, dryrun=False):
@@ -417,13 +418,14 @@ def create(self, dryrun=False):
                             quiet=True,
                         ),
                         stderr=subprocess.PIPE,
+                        text=True,
                     )
                 except subprocess.CalledProcessError as e:
                     raise WorkflowError(
                         "Unable to find environment in container image. "
                         "Maybe a conda environment was modified without containerizing again "
                         "(see snakemake --containerize)?\nDetails:\n{}\n{}".format(
-                            e, e.stderr.decode()
+                            e, e.stderr
                         )
                     )
                 return env_path
@@ -503,9 +505,7 @@ def create(self, dryrun=False):
                             args=self._singularity_args,
                             envvars=self.get_singularity_envvars(),
                         )
-                    out = shell.check_output(
-                        cmd, stderr=subprocess.STDOUT, universal_newlines=True
-                    )
+                    out = shell.check_output(cmd, stderr=subprocess.STDOUT, text=True)
                 else:
 
                     def create_env(env_file, filetype="yaml"):
@@ -550,18 +550,20 @@ def create_env(env_file, filetype="yaml"):
                                 envvars=self.get_singularity_envvars(),
                             )
                         out = shell.check_output(
-                            cmd, stderr=subprocess.STDOUT, universal_newlines=True
+                            cmd, stderr=subprocess.STDOUT, text=True
                         )
 
                         # cleanup if requested
                         if self._cleanup is CondaCleanupMode.tarballs:
                             logger.info("Cleaning up conda package tarballs.")
-                            shell.check_output("conda clean -y --tarballs")
+                            shell.check_output("conda clean -y --tarballs", text=True)
                         elif self._cleanup is CondaCleanupMode.cache:
                             logger.info(
                                 "Cleaning up conda package tarballs and package cache."
                             )
-                            shell.check_output("conda clean -y --tarballs --packages")
+                            shell.check_output(
+                                "conda clean -y --tarballs --packages", text=True
+                            )
                         return out
 
                     if pin_file is not None:
@@ -662,10 +664,7 @@ def __init__(
             self.frontend = frontend
 
             self.info = json.loads(
-                shell.check_output(
-                    self._get_cmd(f"conda info --json"),
-                    universal_newlines=True,
-                )
+                shell.check_output(self._get_cmd(f"conda info --json"), text=True)
             )
 
             if prefix_path is None or container_img is not None:
@@ -706,7 +705,9 @@ def _check(self):
                 locate_cmd = f"where {frontend}"
 
             try:
-                shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT)
+                shell.check_output(
+                    self._get_cmd(locate_cmd), stderr=subprocess.STDOUT, text=True
+                )
             except subprocess.CalledProcessError as e:
                 if self.container_img:
                     msg = (
@@ -757,9 +758,7 @@ def _check_version(self):
         from snakemake.shell import shell
 
         version = shell.check_output(
-            self._get_cmd("conda --version"),
-            stderr=subprocess.PIPE,
-            universal_newlines=True,
+            self._get_cmd("conda --version"), stderr=subprocess.PIPE, text=True
         )
         version_matches = re.findall("\d+.\d+.\d+", version)
         if len(version_matches) != 1:
@@ -783,7 +782,7 @@ def _check_condarc(self):
         res = json.loads(
             shell.check_output(
                 self._get_cmd("conda config --get channel_priority --json"),
-                universal_newlines=True,
+                text=True,
                 stderr=subprocess.PIPE,
             )
         )

test-environment.yml

@@ -41,6 +41,7 @@ dependencies:
   - appdirs
   - python-irodsclient <1.1.2 # bug in 1.1.2 leads to KeyError
   - cwltool
+  - cwl-utils
   - jsonschema
   - pandas
   - python-kubernetes

tests/test_jupyter_notebook/env.yaml

@@ -4,3 +4,4 @@ channels:
 dependencies:
   - python >=3.5
   - jupyter
+  - nbformat <5.5 # there is a bug in 5.5 that leads to a crash during notebook validation