From ebe13331e54c023eb65c8ed201100fca5b345832 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 26 May 2021 20:15:41 -0400 Subject: [PATCH 1/3] Use normalized names to distinguish unique distributions for performance. Fixes #283. --- importlib_metadata/__init__.py | 15 ++++++++++++++- importlib_metadata/_compat.py | 6 ++++++ setup.cfg | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 0da20fc8..47ca8f6b 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -20,6 +20,7 @@ NullFinder, PyPy_repr, install, + singledispatch, ) from ._functools import method_cache from ._itertools import unique_everseen @@ -842,6 +843,18 @@ def version(distribution_name): return distribution(distribution_name).version +@singledispatch +def normalized_name(dist: Distribution): + return Prepared.normalize(dist.name) + + +@normalized_name.register +def _(dist: PathDistribution): + stem = os.path.basename(str(dist._path)) + name, sep, rest = stem.partition('-') + return name + + def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. @@ -859,7 +872,7 @@ def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: :return: EntryPoints or SelectableGroups for all installed packages. """ - unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) + unique = functools.partial(unique_everseen, key=normalized_name) eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) diff --git a/importlib_metadata/_compat.py b/importlib_metadata/_compat.py index 043ece02..804ea9f9 100644 --- a/importlib_metadata/_compat.py +++ b/importlib_metadata/_compat.py @@ -15,6 +15,12 @@ from typing_extensions import Protocol # type: ignore +if sys.version_info < (3, 7): + from singledispatch import singledispatch +else: + from functools import singledispatch # noqa: F401 + + def install(cls): """ Class decorator for installation on sys.meta_path. diff --git a/setup.cfg b/setup.cfg index eb9b6541..1a0b2396 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,7 @@ python_requires = >=3.6 install_requires = zipp>=0.5 typing-extensions>=3.6.4; python_version < "3.8" + singledispatch; python_version < "3.7" [options.packages.find] exclude = From fddbf5d68b70ed0f26535fc768876a072bf739cd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 27 May 2021 09:01:06 -0400 Subject: [PATCH 2/3] Update changelog. --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9e0e7c44..74517bad 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v4.3.0 +======= + +* #317: De-duplication of distributions no longer requires + loading the full metadata for ``PathDistribution`` objects, + entry point loading performance by ~10x. + v4.2.0 ======= From abc13a27780051d9f743bf1b992537561e998dd6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 27 May 2021 09:07:41 -0400 Subject: [PATCH 3/3] Prefer an OO approach for _normalized_name. Allows other providers to supply _normalized_name. --- importlib_metadata/__init__.py | 27 +++++++++++++-------------- importlib_metadata/_compat.py | 6 ------ setup.cfg | 1 - 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 47ca8f6b..3693a01d 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -20,7 +20,6 @@ NullFinder, PyPy_repr, install, - singledispatch, ) from ._functools import method_cache from ._itertools import unique_everseen @@ -499,6 +498,11 @@ def name(self): """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] + @property + def _normalized_name(self): + """Return a normalized version of the name.""" + return Prepared.normalize(self.name) + @property def version(self): """Return the 'Version' metadata for the distribution package.""" @@ -806,6 +810,12 @@ def read_text(self, filename): def locate_file(self, path): return self._path.parent / path + @property + def _normalized_name(self): + stem = os.path.basename(str(self._path)) + name, sep, rest = stem.partition('-') + return name + def distribution(distribution_name): """Get the ``Distribution`` instance for the named package. @@ -843,18 +853,6 @@ def version(distribution_name): return distribution(distribution_name).version -@singledispatch -def normalized_name(dist: Distribution): - return Prepared.normalize(dist.name) - - -@normalized_name.register -def _(dist: PathDistribution): - stem = os.path.basename(str(dist._path)) - name, sep, rest = stem.partition('-') - return name - - def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. @@ -872,7 +870,8 @@ def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: :return: EntryPoints or SelectableGroups for all installed packages. """ - unique = functools.partial(unique_everseen, key=normalized_name) + norm_name = operator.attrgetter('_normalized_name') + unique = functools.partial(unique_everseen, key=norm_name) eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) diff --git a/importlib_metadata/_compat.py b/importlib_metadata/_compat.py index 804ea9f9..043ece02 100644 --- a/importlib_metadata/_compat.py +++ b/importlib_metadata/_compat.py @@ -15,12 +15,6 @@ from typing_extensions import Protocol # type: ignore -if sys.version_info < (3, 7): - from singledispatch import singledispatch -else: - from functools import singledispatch # noqa: F401 - - def install(cls): """ Class decorator for installation on sys.meta_path. diff --git a/setup.cfg b/setup.cfg index 1a0b2396..eb9b6541 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,6 @@ python_requires = >=3.6 install_requires = zipp>=0.5 typing-extensions>=3.6.4; python_version < "3.8" - singledispatch; python_version < "3.7" [options.packages.find] exclude =