diff --git a/docs/changelog/2054.feature.rst b/docs/changelog/2054.feature.rst new file mode 100644 index 000000000..79bc2a355 --- /dev/null +++ b/docs/changelog/2054.feature.rst @@ -0,0 +1,2 @@ +On the programmatic API allow passing in the environment variable dictionary to use, defaults to ``os.environ`` if not +specified - by :user:`gaborbernat`. diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index 0995e4c18..3b06fd747 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -1,11 +1,13 @@ from __future__ import absolute_import, print_function, unicode_literals import logging +import os import sys from datetime import datetime -def run(args=None, options=None): +def run(args=None, options=None, env=None): + env = os.environ if env is None else env start = datetime.now() from virtualenv.run import cli_run from virtualenv.util.error import ProcessCallFailed @@ -13,7 +15,7 @@ def run(args=None, options=None): if args is None: args = sys.argv[1:] try: - session = cli_run(args, options) + session = cli_run(args, options, env) logging.warning(LogSession(session, start)) except ProcessCallFailed as exception: print("subprocess call failed for {} with code {}".format(exception.cmd, exception.code)) @@ -54,12 +56,13 @@ def __str__(self): return "\n".join(lines) -def run_with_catch(args=None): +def run_with_catch(args=None, env=None): from virtualenv.config.cli.parser import VirtualEnvOptions + env = os.environ if env is None else env options = VirtualEnvOptions() try: - run(args, options) + run(args, options, env) except (KeyboardInterrupt, SystemExit, Exception) as exception: try: if getattr(options, "with_traceback", False): diff --git a/src/virtualenv/app_data/__init__.py b/src/virtualenv/app_data/__init__.py index 2df0cae5d..1d4b1ead2 100644 --- a/src/virtualenv/app_data/__init__.py +++ b/src/virtualenv/app_data/__init__.py @@ -14,21 +14,22 @@ from .via_tempdir import TempAppData -def _default_app_data_dir(): # type: () -> str +def _default_app_data_dir(env): key = str("VIRTUALENV_OVERRIDE_APP_DATA") - if key in os.environ: - return os.environ[key] + if key in env: + return env[key] else: return user_data_dir(appname="virtualenv", appauthor="pypa") def make_app_data(folder, **kwargs): read_only = kwargs.pop("read_only") + env = kwargs.pop("env") if kwargs: # py3+ kwonly raise TypeError("unexpected keywords: {}") if folder is None: - folder = _default_app_data_dir() + folder = _default_app_data_dir(env) folder = os.path.abspath(folder) if read_only: diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index eb4db30a7..c8e2f551f 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +import os from argparse import SUPPRESS, ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace from collections import OrderedDict @@ -47,9 +48,11 @@ class VirtualEnvConfigParser(ArgumentParser): Custom option parser which updates its defaults by checking the configuration files and environmental variables """ - def __init__(self, options=None, *args, **kwargs): - self.file_config = IniConfig() + def __init__(self, options=None, env=None, *args, **kwargs): + env = os.environ if env is None else env + self.file_config = IniConfig(env) self.epilog_list = [] + self.env = env kwargs["epilog"] = self.file_config.epilog kwargs["add_help"] = False kwargs["formatter_class"] = HelpFormatter @@ -75,7 +78,7 @@ def _fix_default(self, action): names = OrderedDict((i.lstrip("-").replace("-", "_"), None) for i in action.option_strings) outcome = None for name in names: - outcome = get_env_var(name, as_type) + outcome = get_env_var(name, as_type, self.env) if outcome is not None: break if outcome is None and self.file_config: @@ -101,6 +104,7 @@ def parse_known_args(self, args=None, namespace=None): self._fix_defaults() self.options._src = "cli" try: + namespace.env = self.env return super(VirtualEnvConfigParser, self).parse_known_args(args, namespace=namespace) finally: self.options._src = None diff --git a/src/virtualenv/config/env_var.py b/src/virtualenv/config/env_var.py index 259399a70..8f6211cae 100644 --- a/src/virtualenv/config/env_var.py +++ b/src/virtualenv/config/env_var.py @@ -1,22 +1,21 @@ from __future__ import absolute_import, unicode_literals -import os - from virtualenv.util.six import ensure_str, ensure_text from .convert import convert -def get_env_var(key, as_type): +def get_env_var(key, as_type, env): """Get the environment variable option. :param key: the config key requested :param as_type: the type we would like to convert it to + :param env: environment variables to use :return: """ environ_key = ensure_str("VIRTUALENV_{}".format(key.upper())) - if os.environ.get(environ_key): - value = os.environ[environ_key] + if env.get(environ_key): + value = env[environ_key] # noinspection PyBroadException try: source = "env var {}".format(ensure_text(environ_key)) diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index 4dec629a9..fb3fb0651 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -19,8 +19,9 @@ class IniConfig(object): section = "virtualenv" - def __init__(self): - config_file = os.environ.get(self.VIRTUALENV_CONFIG_FILE_ENV_VAR, None) + def __init__(self, env=None): + env = os.environ if env is None else env + config_file = env.get(self.VIRTUALENV_CONFIG_FILE_ENV_VAR, None) self.is_env_var = config_file is not None config_file = ( Path(config_file) diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 1b4ea69f6..6363f8b7e 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -47,6 +47,7 @@ def __init__(self, options, interpreter): self.no_vcs_ignore = options.no_vcs_ignore self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) self.app_data = options.app_data + self.env = options.env def __repr__(self): return ensure_str(self.__unicode__()) @@ -204,7 +205,7 @@ def debug(self): :return: debug information about the virtual environment (only valid after :meth:`create` has run) """ if self._debug is None and self.exe is not None: - self._debug = get_env_debug_info(self.exe, self.debug_script(), self.app_data) + self._debug = get_env_debug_info(self.exe, self.debug_script(), self.app_data, self.env) return self._debug # noinspection PyMethodMayBeStatic @@ -212,8 +213,8 @@ def debug_script(self): return DEBUG_SCRIPT -def get_env_debug_info(env_exe, debug_script, app_data): - env = os.environ.copy() +def get_env_debug_info(env_exe, debug_script, app_data, env): + env = env.copy() env.pop(str("PYTHONPATH"), None) with app_data.ensure_extracted(debug_script) as debug_script: diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 70ffbced0..41b43902b 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -44,7 +44,7 @@ def add_parser_arguments(cls, parser): def run(self): for python_spec in self.python_spec: - result = get_interpreter(python_spec, self.try_first_with, self.app_data) + result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env) if result is not None: return result return None @@ -57,11 +57,12 @@ def __unicode__(self): return "{} discover of python_spec={!r}".format(self.__class__.__name__, spec) -def get_interpreter(key, try_first_with, app_data=None): +def get_interpreter(key, try_first_with, app_data=None, env=None): spec = PythonSpec.from_string_spec(key) logging.info("find interpreter for spec %r", spec) proposed_paths = set() - for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data): + env = os.environ if env is None else env + for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue @@ -72,8 +73,9 @@ def get_interpreter(key, try_first_with, app_data=None): proposed_paths.add(key) -def propose_interpreters(spec, try_first_with, app_data): +def propose_interpreters(spec, try_first_with, app_data, env=None): # 0. try with first + env = os.environ if env is None else env for py_exe in try_first_with: path = os.path.abspath(py_exe) try: @@ -81,7 +83,7 @@ def propose_interpreters(spec, try_first_with, app_data): except OSError: pass else: - yield PythonInfo.from_exe(os.path.abspath(path), app_data), True + yield PythonInfo.from_exe(os.path.abspath(path), app_data, env=env), True # 1. if it's a path and exists if spec.path is not None: @@ -91,7 +93,7 @@ def propose_interpreters(spec, try_first_with, app_data): if spec.is_abs: raise else: - yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data), True + yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data, env=env), True if spec.is_abs: return else: @@ -102,27 +104,27 @@ def propose_interpreters(spec, try_first_with, app_data): if IS_WIN: from .windows import propose_interpreters - for interpreter in propose_interpreters(spec, app_data): + for interpreter in propose_interpreters(spec, app_data, env): yield interpreter, True # finally just find on path, the path order matters (as the candidates are less easy to control by end user) - paths = get_paths() + paths = get_paths(env) tested_exes = set() for pos, path in enumerate(paths): path = ensure_text(path) - logging.debug(LazyPathDump(pos, path)) + logging.debug(LazyPathDump(pos, path, env)) for candidate, match in possible_specs(spec): found = check_path(candidate, path) if found is not None: exe = os.path.abspath(found) if exe not in tested_exes: tested_exes.add(exe) - interpreter = PathPythonInfo.from_exe(exe, app_data, raise_on_error=False) + interpreter = PathPythonInfo.from_exe(exe, app_data, raise_on_error=False, env=env) if interpreter is not None: yield interpreter, match -def get_paths(): - path = os.environ.get(str("PATH"), None) +def get_paths(env): + path = env.get(str("PATH"), None) if path is None: try: path = os.confstr("CS_PATH") @@ -136,16 +138,17 @@ def get_paths(): class LazyPathDump(object): - def __init__(self, pos, path): + def __init__(self, pos, path, env): self.pos = pos self.path = path + self.env = env def __repr__(self): return ensure_str(self.__unicode__()) def __unicode__(self): content = "discover PATH[{}]={}".format(self.pos, self.path) - if os.environ.get(str("_VIRTUALENV_DEBUG")): # this is the over the board debug + if self.env.get(str("_VIRTUALENV_DEBUG")): # this is the over the board debug content += " with =>" for file_name in os.listdir(self.path): try: diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index 7baef386a..d16a8e298 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -23,9 +23,9 @@ _CACHE[Path(sys.executable)] = PythonInfo() -def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False): - """""" - result = _get_from_cache(cls, app_data, exe, ignore_cache=ignore_cache) +def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False): + env = os.environ if env is None else env + result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: raise result @@ -35,14 +35,14 @@ def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False): return result -def _get_from_cache(cls, app_data, exe, ignore_cache=True): +def _get_from_cache(cls, app_data, exe, env, ignore_cache=True): # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a # pyenv.cfg somewhere alongside on python3.4+ exe_path = Path(exe) if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache result = _CACHE[exe_path] else: # otherwise go through the app data cache - py_info = _get_via_file_cache(cls, app_data, exe_path, exe) + py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env) result = _CACHE[exe_path] = py_info # independent if it was from the file or in-memory cache fix the original executable location if isinstance(result, PythonInfo): @@ -50,7 +50,7 @@ def _get_from_cache(cls, app_data, exe, ignore_cache=True): return result -def _get_via_file_cache(cls, app_data, path, exe): +def _get_via_file_cache(cls, app_data, path, exe, env): path_text = ensure_text(str(path)) try: path_modified = path.stat().st_mtime @@ -72,7 +72,7 @@ def _get_via_file_cache(cls, app_data, path, exe): else: py_info_store.remove() if py_info is None: # if not loaded run and save - failure, py_info = _run_subprocess(cls, exe, app_data) + failure, py_info = _run_subprocess(cls, exe, app_data, env) if failure is None: data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()} py_info_store.write(data) @@ -81,12 +81,12 @@ def _get_via_file_cache(cls, app_data, path, exe): return py_info -def _run_subprocess(cls, exe, app_data): +def _run_subprocess(cls, exe, app_data, env): py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" with app_data.ensure_extracted(py_info_script) as py_info_script: cmd = [exe, str(py_info_script)] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 - env = os.environ.copy() + env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: diff --git a/src/virtualenv/discovery/discover.py b/src/virtualenv/discovery/discover.py index 93c3ea7ad..72748c3fa 100644 --- a/src/virtualenv/discovery/discover.py +++ b/src/virtualenv/discovery/discover.py @@ -25,6 +25,7 @@ def __init__(self, options): """ self._has_run = False self._interpreter = None + self._env = options.env @abstractmethod def run(self): diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 46b51df1b..30e13215e 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -308,12 +308,13 @@ def _to_dict(self): return data @classmethod - def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True): + def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None): """Given a path to an executable get the python information""" # this method is not used by itself, so here and called functions can import stuff locally from virtualenv.discovery.cached_py_info import from_exe - proposed = from_exe(cls, app_data, exe, raise_on_error=raise_on_error, ignore_cache=ignore_cache) + env = os.environ if env is None else env + proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache) # noinspection PyProtectedMember if isinstance(proposed, PythonInfo) and resolve_to_host: try: @@ -363,7 +364,7 @@ def _resolve_to_system(cls, app_data, target): _cache_exe_discovery = {} - def discover_exe(self, app_data, prefix, exact=True): + def discover_exe(self, app_data, prefix, exact=True, env=None): key = prefix, exact if key in self._cache_exe_discovery and prefix: logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) @@ -373,9 +374,10 @@ def discover_exe(self, app_data, prefix, exact=True): possible_names = self._find_possible_exe_names() possible_folders = self._find_possible_folders(prefix) discovered = [] + env = os.environ if env is None else env for folder in possible_folders: for name in possible_names: - info = self._check_exe(app_data, folder, name, exact, discovered) + info = self._check_exe(app_data, folder, name, exact, discovered, env) if info is not None: self._cache_exe_discovery[key] = info return info @@ -388,11 +390,11 @@ def discover_exe(self, app_data, prefix, exact=True): msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) raise RuntimeError(msg) - def _check_exe(self, app_data, folder, name, exact, discovered): + def _check_exe(self, app_data, folder, name, exact, discovered, env): exe_path = os.path.join(folder, name) if not os.path.exists(exe_path): return None - info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False) + info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env) if info is None: # ignore if for some reason we can't query return None for item in ["implementation", "architecture", "version_info"]: diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index 556ecf2e4..a9d06dabb 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -9,7 +9,7 @@ class Pep514PythonInfo(PythonInfo): """""" -def propose_interpreters(spec, cache_dir): +def propose_interpreters(spec, cache_dir, env): # see if PEP-514 entries are good # start with higher python versions in an effort to use the latest version available @@ -25,7 +25,7 @@ def propose_interpreters(spec, cache_dir): name = "CPython" registry_spec = PythonSpec(None, name, major, minor, None, arch, exe) if registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, raise_on_error=False) + interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) if interpreter is not None: if interpreter.satisfies(spec, impl_must_match=True): yield interpreter diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 66083df82..e8e7ab138 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, unicode_literals import logging +import os from functools import partial from ..app_data import make_app_data @@ -15,22 +16,24 @@ from .plugin.seeders import SeederSelector -def cli_run(args, options=None, setup_logging=True): +def cli_run(args, options=None, setup_logging=True, env=None): """ Create a virtual environment given some command line interface arguments. :param args: the command line arguments :param options: passing in a ``VirtualEnvOptions`` object allows return of the parsed options :param setup_logging: ``True`` if setup logging handlers, ``False`` to use handlers already registered + :param env: environment variables to use :return: the session object of the creation (its structure for now is experimental and might change on short notice) """ - of_session = session_via_cli(args, options, setup_logging) + env = os.environ if env is None else env + of_session = session_via_cli(args, options, setup_logging, env) with of_session: of_session.run() return of_session -def session_via_cli(args, options=None, setup_logging=True): +def session_via_cli(args, options=None, setup_logging=True, env=None): """ Create a virtualenv session (same as cli_run, but this does not perform the creation). Use this if you just want to query what the virtual environment would look like, but not actually create it. @@ -38,17 +41,19 @@ def session_via_cli(args, options=None, setup_logging=True): :param args: the command line arguments :param options: passing in a ``VirtualEnvOptions`` object allows return of the parsed options :param setup_logging: ``True`` if setup logging handlers, ``False`` to use handlers already registered + :param env: environment variables to use :return: the session object of the creation (its structure for now is experimental and might change on short notice) """ - parser, elements = build_parser(args, options, setup_logging) + env = os.environ if env is None else env + parser, elements = build_parser(args, options, setup_logging, env) options = parser.parse_args(args) creator, seeder, activators = tuple(e.create(options) for e in elements) # create types of_session = Session(options.verbosity, options.app_data, parser._interpreter, creator, seeder, activators) # noqa return of_session -def build_parser(args=None, options=None, setup_logging=True): - parser = VirtualEnvConfigParser(options) +def build_parser(args=None, options=None, setup_logging=True, env=None): + parser = VirtualEnvConfigParser(options, os.environ if env is None else env) add_version_flag(parser) parser.add_argument( "--with-traceback", @@ -84,7 +89,7 @@ def build_parser_only(args=None): def handle_extra_commands(options): if options.upgrade_embed_wheels: - result = manual_upgrade(options.app_data) + result = manual_upgrade(options.app_data, options.env) raise SystemExit(result) @@ -100,8 +105,8 @@ def load_app_data(args, parser, options): parser.add_argument( "--app-data", help="a data folder used as cache by the virtualenv", - type=partial(make_app_data, read_only=options.read_only_app_data), - default=make_app_data(None, read_only=options.read_only_app_data), + type=partial(make_app_data, read_only=options.read_only_app_data, env=options.env), + default=make_app_data(None, read_only=options.read_only_app_data, env=options.env), ) parser.add_argument( "--reset-app-data", diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 372e140dc..c935c0216 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -19,7 +19,7 @@ def run(self, creator): return for_py_version = creator.interpreter.version_release_str with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd: - env = pip_wheel_env_run(self.extra_search_dir, self.app_data) + env = pip_wheel_env_run(self.extra_search_dir, self.app_data, self.env) self._execute(cmd, env) @staticmethod @@ -46,6 +46,7 @@ def get_pip_install_cmd(self, exe, for_py_version): download=False, app_data=self.app_data, do_periodic_update=self.periodic_update, + env=self.env, ) if wheel is None: raise RuntimeError("could not get wheel for distribution {}".format(dist)) diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py index 1afa7978c..9a98a709f 100644 --- a/src/virtualenv/seed/embed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py @@ -89,6 +89,7 @@ def _get(distribution, version): download=download, app_data=self.app_data, do_periodic_update=self.periodic_update, + env=self.env, ) if result is not None: break diff --git a/src/virtualenv/seed/seeder.py b/src/virtualenv/seed/seeder.py index 2bcccfc72..852e85254 100644 --- a/src/virtualenv/seed/seeder.py +++ b/src/virtualenv/seed/seeder.py @@ -17,6 +17,7 @@ def __init__(self, options, enabled): :param enabled: a flag weather the seeder is enabled or not """ self.enabled = enabled + self.env = options.env @classmethod def add_parser_arguments(cls, parser, interpreter, app_data): diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py index e63ecb67c..0ee03ec98 100644 --- a/src/virtualenv/seed/wheels/acquire.py +++ b/src/virtualenv/seed/wheels/acquire.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, unicode_literals import logging -import os import sys from operator import eq, lt @@ -14,13 +13,13 @@ from .util import Version, Wheel, discover_wheels -def get_wheel(distribution, version, for_py_version, search_dirs, download, app_data, do_periodic_update): +def get_wheel(distribution, version, for_py_version, search_dirs, download, app_data, do_periodic_update, env): """ Get a wheel with the given distribution-version-for_py_version trio, by using the extra search dir + download """ # not all wheels are compatible with all python versions, so we need to py version qualify it # 1. acquire from bundle - wheel = from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update) + wheel = from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update, env) # 2. download from the internet if version not in Version.non_version and download: @@ -31,11 +30,12 @@ def get_wheel(distribution, version, for_py_version, search_dirs, download, app_ search_dirs=search_dirs, app_data=app_data, to_folder=app_data.house, + env=env, ) return wheel -def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): +def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): to_download = "{}{}".format(distribution, version_spec or "") logging.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) cmd = [ @@ -55,7 +55,7 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_ to_download, ] # pip has no interface in python - must be a new sub-process - env = pip_wheel_env_run(search_dirs, app_data) + env = pip_wheel_env_run(search_dirs, app_data, env) process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode != 0: @@ -96,9 +96,9 @@ def find_compatible_in_house(distribution, version_spec, for_py_version, in_fold return None if start == end else wheels[start] -def pip_wheel_env_run(search_dirs, app_data): +def pip_wheel_env_run(search_dirs, app_data, env): for_py_version = "{}.{}".format(*sys.version_info[0:2]) - env = os.environ.copy() + env = env.copy() env.update( { ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode) @@ -113,6 +113,7 @@ def pip_wheel_env_run(search_dirs, app_data): download=False, app_data=app_data, do_periodic_update=False, + env=env, ) if wheel is None: raise RuntimeError("could not find the embedded pip") diff --git a/src/virtualenv/seed/wheels/bundle.py b/src/virtualenv/seed/wheels/bundle.py index 7c664bd38..ab2fe5fa5 100644 --- a/src/virtualenv/seed/wheels/bundle.py +++ b/src/virtualenv/seed/wheels/bundle.py @@ -5,7 +5,7 @@ from .util import Version, Wheel, discover_wheels -def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update): +def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update, env): """ Load the bundled wheel to a cache directory. """ @@ -15,7 +15,7 @@ def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do if version != Version.embed: # 2. check if we have upgraded embed if app_data.can_update: - wheel = periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update) + wheel = periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env) # 3. acquire from extra search dir found_wheel = from_dir(distribution, of_version, for_py_version, search_dirs) diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py index fd0ff4c26..18fcfa330 100644 --- a/src/virtualenv/seed/wheels/periodic_update.py +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -6,7 +6,6 @@ import json import logging -import os import ssl import subprocess import sys @@ -36,9 +35,9 @@ pass # pragma: no cov -def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update): +def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env): if do_periodic_update: - handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data) + handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env) now = datetime.now() @@ -57,14 +56,14 @@ def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, return wheel -def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data): +def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env): embed_update_log = app_data.embed_update_log(distribution, for_py_version) u_log = UpdateLog.from_dict(embed_update_log.read()) if u_log.needs_update: u_log.periodic = True u_log.started = datetime.now() embed_update_log.write(u_log.to_dict()) - trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True) + trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True, env=env) DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" @@ -169,7 +168,7 @@ def _check_start(self, now): return self.started is None or now - self.started > timedelta(hours=1) -def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic): +def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic): wheel_path = None if wheel is None else str(wheel.path) cmd = [ sys.executable, @@ -185,7 +184,7 @@ def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, p .strip() .format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic), ] - debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") + debug = env.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") pipe = None if debug else subprocess.PIPE kwargs = {"stdout": pipe, "stderr": pipe} if not debug and sys.platform == "win32": @@ -301,13 +300,13 @@ def _pypi_get_distribution_info(distribution): return content -def manual_upgrade(app_data): +def manual_upgrade(app_data, env): threads = [] for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): # load extra search dir for the given for_py for distribution in distribution_to_package.keys(): - thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version)) + thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version, env)) thread.start() threads.append(thread) @@ -315,7 +314,7 @@ def manual_upgrade(app_data): thread.join() -def _run_manual_upgrade(app_data, distribution, for_py_version): +def _run_manual_upgrade(app_data, distribution, for_py_version, env): start = datetime.now() from .bundle import from_bundle @@ -326,6 +325,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version): search_dirs=[], app_data=app_data, do_periodic_update=False, + env=env, ) logging.warning( "upgrade %s for python %s with current %s", diff --git a/src/virtualenv/util/subprocess/_win_subprocess.py b/src/virtualenv/util/subprocess/_win_subprocess.py index 4c4c5d029..be4e7d2dc 100644 --- a/src/virtualenv/util/subprocess/_win_subprocess.py +++ b/src/virtualenv/util/subprocess/_win_subprocess.py @@ -155,7 +155,7 @@ def _execute_child( args = args.decode('utf-8') startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) + comspec = env.get("COMSPEC", unicode("cmd.exe")) if ( _subprocess.GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com" diff --git a/tests/conftest.py b/tests/conftest.py index d98e9268c..9beaa12b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,8 +224,8 @@ def coverage_env(monkeypatch, link, request): # we inject right after creation, we cannot collect coverage on site.py - used for helper scripts, such as debug from virtualenv import run - def _session_via_cli(args, options, setup_logging): - session = prev_run(args, options, setup_logging) + def _session_via_cli(args, options, setup_logging, env=None): + session = prev_run(args, options, setup_logging, env) old_run = session.creator.run def create_run(): diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index 49a3ee63b..5b9356362 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -25,8 +25,8 @@ def _func(exception): prev_session = session_via_cli - def _session_via_cli(args, options=None, setup_logging=True): - prev_session(args, options, setup_logging) + def _session_via_cli(args, options=None, setup_logging=True, env=None): + prev_session(args, options, setup_logging, env) raise exception mocker.patch("virtualenv.run.session_via_cli", side_effect=_session_via_cli) diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index 7184517f4..35f3f2493 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -91,7 +91,7 @@ def cleanup_sys_path(paths): @pytest.fixture(scope="session") def system(session_app_data): - return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data) + return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) CURRENT_CREATORS = list(i for i in CURRENT.creators().key_to_class.keys() if i != "builtin") @@ -268,8 +268,8 @@ def test_venv_fails_not_inline(tmp_path, capsys, mocker): if os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") - def _session_via_cli(args, options=None, setup_logging=True): - session = session_via_cli(args, options, setup_logging) + def _session_via_cli(args, options=None, setup_logging=True, env=None): + session = session_via_cli(args, options, setup_logging, env) assert session.creator.can_be_inline is False return session diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 393a86b86..7810cfad1 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -226,7 +226,7 @@ def _make_py_info(of): mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) # noinspection PyUnusedLocal - def func(k, app_data, resolve_to_host, raise_on_error): + def func(k, app_data, resolve_to_host, raise_on_error, env): return discovered_with_path[k] mocker.patch.object(target_py_info, "from_exe", side_effect=func) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index 59356a2f4..7d33b22a8 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -58,7 +58,9 @@ def test_relative_path(tmp_path, session_app_data, monkeypatch): def test_discovery_fallback_fail(session_app_data, caplog): caplog.set_level(logging.DEBUG) - builtin = Builtin(Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"])) + builtin = Builtin( + Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ) + ) result = builtin.run() assert result is None @@ -68,7 +70,9 @@ def test_discovery_fallback_fail(session_app_data, caplog): def test_discovery_fallback_ok(session_app_data, caplog): caplog.set_level(logging.DEBUG) - builtin = Builtin(Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable])) + builtin = Builtin( + Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ) + ) result = builtin.run() assert result is not None, caplog.text diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py index 5e2e9b616..f9793e448 100644 --- a/tests/unit/seed/wheels/test_acquire.py +++ b/tests/unit/seed/wheels/test_acquire.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +import os import sys from subprocess import CalledProcessError @@ -13,7 +14,7 @@ def test_pip_wheel_env_run_could_not_find(session_app_data, mocker): mocker.patch("virtualenv.seed.wheels.acquire.from_bundle", return_value=None) with pytest.raises(RuntimeError, match="could not find the embedded pip"): - pip_wheel_env_run([], session_app_data) + pip_wheel_env_run([], session_app_data, os.environ) def test_download_wheel_bad_output(mocker, for_py_version, session_app_data): @@ -29,7 +30,9 @@ def test_download_wheel_bad_output(mocker, for_py_version, session_app_data): available = discover_wheels(BUNDLE_FOLDER, "setuptools", None, for_py_version) as_path.iterdir.return_value = [i.path for i in available] - result = download_wheel(distribution, "=={}".format(embed.version), for_py_version, [], session_app_data, as_path) + result = download_wheel( + distribution, "=={}".format(embed.version), for_py_version, [], session_app_data, as_path, os.environ + ) assert result.path == embed.path @@ -41,7 +44,7 @@ def test_download_fails(mocker, for_py_version, session_app_data): as_path = mocker.MagicMock() with pytest.raises(CalledProcessError) as context: - download_wheel("pip", "==1", for_py_version, [], session_app_data, as_path), + download_wheel("pip", "==1", for_py_version, [], session_app_data, as_path, os.environ), exc = context.value if sys.version_info < (3, 5): assert exc.output == "outerr" diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py index 11729819d..6d0b66d85 100644 --- a/tests/unit/seed/wheels/test_periodic_update.py +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, unicode_literals import json +import os import subprocess import sys from collections import defaultdict @@ -49,7 +50,7 @@ def _do_update(distribution, for_py_version, embed_filename, app_data, search_di return [] do_update_mock = mocker.patch("virtualenv.seed.wheels.periodic_update.do_update", side_effect=_do_update) - manual_upgrade(session_app_data) + manual_upgrade(session_app_data, os.environ) assert "upgrade pip" in caplog.text assert "upgraded pip" in caplog.text @@ -100,7 +101,7 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi ) mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) - result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False) + result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False, os.environ) assert result.path == current.path @@ -119,7 +120,7 @@ def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_versi ) mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) - result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False) + result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False, os.environ) assert result.path == current.path @@ -165,7 +166,7 @@ def test_periodic_update_skip(u_log, mocker, for_py_version, session_app_data, f mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update", side_effect=RuntimeError) - result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True) + result = periodic_update("setuptools", for_py_version, None, [], session_app_data, os.environ, True) assert result is None @@ -193,7 +194,7 @@ def test_periodic_update_trigger(u_log, mocker, for_py_version, session_app_data write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") trigger_update_ = mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update") - result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True) + result = periodic_update("setuptools", for_py_version, None, [], session_app_data, os.environ, True) assert result is None assert trigger_update_.call_count @@ -210,7 +211,9 @@ def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, moc process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) - trigger_update("setuptools", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, True) + trigger_update( + "setuptools", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, os.environ, True + ) assert Popen.call_count == 1 args, kwargs = Popen.call_args @@ -250,7 +253,9 @@ def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) - trigger_update("pip", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, False) + trigger_update( + "pip", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, os.environ, False + ) assert Popen.call_count == 1 args, kwargs = Popen.call_args