Stop importing pip, use subprocesses instead. Fixes #710 (#797)

libtbx/auto_build/conda_envs/cctbxlite.devenv.yml

@@ -37,6 +37,7 @@ dependencies:
   # extra
   - libsvm-official
   - pytest
+  - packaging
 
   # docs
   - docutils

libtbx/auto_build/install_base_packages.py

@@ -15,6 +15,7 @@
 import sys
 import time
 import zipfile
+import subprocess
 from optparse import OptionParser
 
 if __name__ == '__main__' and __package__ is None:
@@ -644,9 +645,9 @@ def build_python_module_pip(self, package_name, package_version=None, download_o
     if extra_options:
       assert isinstance(extra_options, list), 'extra pip options must be passed as a list'
     if download_only:
-      try:
-        import pip
-      except ImportError:
+      pip_version_cmd = ['python', '-c', 'import pip; print(pip.__version__)']
+      pip_version_result = subprocess.run(pip_version_cmd, stdout=subprocess.PIPE)
+      if pip_version_result.returncode != 0:
         print("Skipping download of python package %s %s" % \
               (pkg_info['name'], pkg_info['version']))
         print("Your current python environment does not include 'pip',")
@@ -659,20 +660,15 @@ def build_python_module_pip(self, package_name, package_version=None, download_o
              pkg_info=pkg_info['summary'],
              pkg_qualifier=' ' + (package_version or ''))
     if download_only:
-      pip_cmd = filter(None, ['download',
+      pip_cmd = filter(None, ['pip', 'download',
                               pkg_info['package'] + pkg_info['version'],
                               '-d', pkg_info['cachedir'], pkg_info['debug']])
       if extra_options:
         pip_cmd.extend(extra_options)
-      if int(pip.__version__.split('.')[0]) > 9:
-        import pip._internal
-        pip_call = pip._internal.main
-        pip_cmd.append('--no-cache-dir')
-      else:
-        pip_call = pip.main
       os.environ['PIP_REQ_TRACKER'] = pkg_info['cachedir']
       print("  Running with pip:", pip_cmd)
-      assert pip_call(pip_cmd) == 0, 'pip download failed'
+
+      assert subprocess.run(pip_cmd).returncode == 0, 'pip download failed'
       return
     if extra_options:
       extra_options = ' '.join(extra_options)

libtbx/pkg_utils.py

@@ -8,47 +8,36 @@
 import itertools
 import os
 import sys
+import subprocess
 
 import libtbx.load_env
 
 try:
-  import pip
+  pip_version_cmd = ['python', '-c', 'import pip; print(pip.__version__)']
+  pip_version_str = subprocess.check_output(pip_version_cmd).decode().strip()
   import pkg_resources
 
   # Don't run if pip version < 9.0.0.
   # There is no technical reason for this, the code should work still.
   # But in the interest of not upsetting legacy build systems let's be cautious.
   if not all(symbol in dir(pkg_resources) for symbol in
              ('parse_version', 'require', 'DistributionNotFound', 'VersionConflict')) \
-     or pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('9.0.0'):
-    pip = None
+     or pkg_resources.parse_version(pip_version_str) < pkg_resources.parse_version('9.0.0'):
+    use_pip = False
     pkg_resources = None
-  if pip:
-    if pkg_resources.parse_version(pip.__version__) >= pkg_resources.parse_version('19.3.0'):
-      import pip._internal.main
-      pip_main = pip._internal.main.main
-    elif pkg_resources.parse_version(pip.__version__) >= pkg_resources.parse_version('10.0.0'):
-      import pip._internal
-      pip_main = pip._internal.main
-    else:
-      pip_main = pip.main
 except ImportError:
-  pip = None
+  use_pip = False
   pkg_resources = None
 
-# Try to find packaging. This is normally(?) installed by setuptools but
-# otherwise pip keeps a copy.
+# Try to find packaging. As of Oct 2022 this should be provided explicitly when
+# the conda env is created. Not safe to import the vendored version from pip;
+# see https://github.com/pypa/setuptools/issues/3297
 try:
   import packaging
   from packaging.requirements import Requirement
 except ImportError:
-  try:
-    import pip._vendor.packaging as packaging
-    from pip._vendor.packaging.requirements import Requirement
-  except ImportError:
-    # If all else fails then we need the symbol to check
-    Requirement = None
-    packaging = None
+  Requirement = None
+  packaging = None
 
 try:
   import setuptools
@@ -77,7 +66,7 @@ def require(pkgname, version=None):
                      eg. '<2', or both, eg. '>=4.5,<4.6'.
      :return: True when the requirement is met, False otherwise.'''
 
-  if not pip:
+  if not use_pip:
     _notice("  WARNING: Can not verify python package requirements - pip/setuptools out of date",
             "  Please update pip and setuptools by running:", "",
             "    libtbx.python -m pip install pip setuptools --upgrade", "",
@@ -157,7 +146,9 @@ def require(pkgname, version=None):
 
   print("attempting {action} of {package}...".format(action=action, package=pkgname))
   has_req_tracker = os.environ.get('PIP_REQ_TRACKER')
-  exit_code = pip_main(['install', requirestring])
+  pip_install_cmd = ['pip', 'install', requirestring]
+  pip_install_result = subprocess.run(pip_install_cmd)
+  exit_code = pip_install_result.returncode
   if not has_req_tracker:
     # clean up environment after pip call for next invocation
     os.environ.pop('PIP_REQ_TRACKER', None)