From cf21303f1ef9b143e76d613b253730c61edd97c2 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 6 Jan 2022 23:30:19 +0100 Subject: [PATCH 1/7] Speedup startup of Python by importing less ``_distutils_hack`` is imported by a ``.pth`` file at every start of a Python interpreter. The import of costly modules like ``re`` and ``contextlib`` almost doubles the initial startup time of an interpreter. - replace ``contextlib`` with simple context manager and try/except - replace ``re`` with simple string match - move import of ``importlib`` into function body - remove ``warnings.filterwarnings()``, which imports ``re``, too. Fixes: #3006 Signed-off-by: Christian Heimes --- _distutils_hack/__init__.py | 50 ++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 75bc4463bb..5d2a6b20d0 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -1,17 +1,14 @@ +# don't import any costly modules import sys import os -import re -import importlib -import warnings -import contextlib is_pypy = '__pypy__' in sys.builtin_module_names -warnings.filterwarnings('ignore', - r'.+ distutils\b.+ deprecated', - DeprecationWarning) +# warnings.filterwarnings('ignore', +# r'.+ distutils\b.+ deprecated', +# DeprecationWarning) def warn_distutils_present(): @@ -21,6 +18,7 @@ def warn_distutils_present(): # PyPy for 3.6 unconditionally imports distutils, so bypass the warning # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 return + import warnings warnings.warn( "Distutils was imported before Setuptools, but importing Setuptools " "also replaces the `distutils` module in `sys.modules`. This may lead " @@ -33,8 +31,12 @@ def warn_distutils_present(): def clear_distutils(): if 'distutils' not in sys.modules: return + import warnings warnings.warn("Setuptools is replacing distutils.") - mods = [name for name in sys.modules if re.match(r'distutils\b', name)] + mods = [ + name for name in sys.modules + if name == "distutils" or name.startswith("distutils.") + ] for name in mods: del sys.modules[name] @@ -48,6 +50,7 @@ def enabled(): def ensure_local_distutils(): + import importlib clear_distutils() # With the DistutilsMetaFinder in place, @@ -73,17 +76,6 @@ def do_override(): ensure_local_distutils() -class suppress(contextlib.suppress, contextlib.ContextDecorator): - """ - A version of contextlib.suppress with decorator support. - - >>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ - - class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): if path is not None: @@ -94,6 +86,7 @@ def find_spec(self, fullname, path, target=None): return method() def spec_for_distutils(self): + import importlib import importlib.abc import importlib.util @@ -144,13 +137,15 @@ def pip_imported_during_build(cls): ) @classmethod - @suppress(AttributeError) def is_get_pip(cls): """ Detect if get-pip is being invoked. Ref #2993. """ - import __main__ - return os.path.basename(__main__.__file__) == 'get-pip.py' + try: + import __main__ + return os.path.basename(__main__.__file__) == 'get-pip.py' + except AttributeError: + pass @staticmethod def frame_file_is_setup(frame): @@ -168,12 +163,11 @@ def add_shim(): DISTUTILS_FINDER in sys.meta_path or insert_shim() -@contextlib.contextmanager -def shim(): - insert_shim() - try: - yield - finally: +class shim: + def __enter__(self): + insert_shim() + + def __exit__(self, exc, value, tb): remove_shim() From 6587a0be16831dad180dedb9ca86dba81cad78af Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 6 Jan 2022 23:37:59 +0100 Subject: [PATCH 2/7] Add changelog entry --- changelog.d/3006.change.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/3006.change.rst diff --git a/changelog.d/3006.change.rst b/changelog.d/3006.change.rst new file mode 100644 index 0000000000..d5f5d99d02 --- /dev/null +++ b/changelog.d/3006.change.rst @@ -0,0 +1,2 @@ +Fixed startup performance issue of Python interpreter due to imports of +costly modules in ``_distutils_hack`` -- by :user:`tiran` From f39c0631e5269c33b3544004439f8480a41fcd02 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 6 Jan 2022 23:48:41 +0100 Subject: [PATCH 3/7] Temporarily add back filter, tests fail without it --- _distutils_hack/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 5d2a6b20d0..47ea70d49a 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -5,10 +5,10 @@ is_pypy = '__pypy__' in sys.builtin_module_names - -# warnings.filterwarnings('ignore', -# r'.+ distutils\b.+ deprecated', -# DeprecationWarning) +import warnings +warnings.filterwarnings('ignore', + r'.+ distutils\b.+ deprecated', + DeprecationWarning) def warn_distutils_present(): From 804b106a8c97efa2944b92918f120c636ecb6818 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 00:17:15 +0100 Subject: [PATCH 4/7] Use internal warnings API and _TrivialRe hack to install filter --- _distutils_hack/__init__.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 47ea70d49a..d324763d2c 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -1,14 +1,29 @@ # don't import any costly modules import sys import os +import warnings is_pypy = '__pypy__' in sys.builtin_module_names -import warnings -warnings.filterwarnings('ignore', - r'.+ distutils\b.+ deprecated', - DeprecationWarning) + +class _TrivialRe: + def __init__(self, *patterns): + self._patterns = patterns + + def match(self, string): + return all(pat in string for pat in self._patterns) + + +# warnings.filterwarnings() imports the re module +warnings._add_filter( + 'ignore', + _TrivialRe("distutils", "deprecated"), + DeprecationWarning, + None, + 0, + append=False +) def warn_distutils_present(): @@ -18,7 +33,6 @@ def warn_distutils_present(): # PyPy for 3.6 unconditionally imports distutils, so bypass the warning # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 return - import warnings warnings.warn( "Distutils was imported before Setuptools, but importing Setuptools " "also replaces the `distutils` module in `sys.modules`. This may lead " @@ -31,7 +45,6 @@ def warn_distutils_present(): def clear_distutils(): if 'distutils' not in sys.modules: return - import warnings warnings.warn("Setuptools is replacing distutils.") mods = [ name for name in sys.modules From 06d81cb32da88ec0b69d3bc01ad138a9c0d41520 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 00:30:26 +0100 Subject: [PATCH 5/7] Move filter into meta finder --- _distutils_hack/__init__.py | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index d324763d2c..0307734d88 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -1,31 +1,11 @@ # don't import any costly modules import sys import os -import warnings is_pypy = '__pypy__' in sys.builtin_module_names -class _TrivialRe: - def __init__(self, *patterns): - self._patterns = patterns - - def match(self, string): - return all(pat in string for pat in self._patterns) - - -# warnings.filterwarnings() imports the re module -warnings._add_filter( - 'ignore', - _TrivialRe("distutils", "deprecated"), - DeprecationWarning, - None, - 0, - append=False -) - - def warn_distutils_present(): if 'distutils' not in sys.modules: return @@ -33,6 +13,7 @@ def warn_distutils_present(): # PyPy for 3.6 unconditionally imports distutils, so bypass the warning # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 return + import warnings warnings.warn( "Distutils was imported before Setuptools, but importing Setuptools " "also replaces the `distutils` module in `sys.modules`. This may lead " @@ -45,6 +26,7 @@ def warn_distutils_present(): def clear_distutils(): if 'distutils' not in sys.modules: return + import warnings warnings.warn("Setuptools is replacing distutils.") mods = [ name for name in sys.modules @@ -89,6 +71,14 @@ def do_override(): ensure_local_distutils() +class _TrivialRe: + def __init__(self, *patterns): + self._patterns = patterns + + def match(self, string): + return all(pat in string for pat in self._patterns) + + class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): if path is not None: @@ -102,6 +92,17 @@ def spec_for_distutils(self): import importlib import importlib.abc import importlib.util + import warnings + + # warnings.filterwarnings() imports the re module + warnings._add_filter( + 'ignore', + _TrivialRe("distutils", "deprecated"), + DeprecationWarning, + None, + 0, + append=True + ) try: mod = importlib.import_module('setuptools._distutils') From 9c0d81e786978c5cc4cbfc50045d271e5b3ae0b9 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 08:22:43 +0100 Subject: [PATCH 6/7] Suppress distutils deprecation warning --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 5c0ad039a4..2ffa4473f0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,6 +39,8 @@ filterwarnings= # https://github.com/pytest-dev/pytest/discussions/9296 ignore:The distutils.sysconfig module is deprecated, use sysconfig instead + ignore:The distutils package is deprecated.* + # Workaround for pypa/setuptools#2868 # ideally would apply to PyPy only but for # https://github.com/pytest-dev/pytest/discussions/9296 From 7619852f4bfdf3065747c44ecc09c024d675c819 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 7 Jan 2022 10:18:05 +0100 Subject: [PATCH 7/7] pytest.ini uses tabs --- pytest.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 2ffa4473f0..f522a45e22 100644 --- a/pytest.ini +++ b/pytest.ini @@ -38,8 +38,7 @@ filterwarnings= # SETUPTOOLS_USE_DISTUTILS=stdlib but for # https://github.com/pytest-dev/pytest/discussions/9296 ignore:The distutils.sysconfig module is deprecated, use sysconfig instead - - ignore:The distutils package is deprecated.* + ignore:The distutils package is deprecated.* # Workaround for pypa/setuptools#2868 # ideally would apply to PyPy only but for