Skip to content

Commit

Permalink
fix: match requests pool_maxsize to num workers (#6805)
Browse files Browse the repository at this point in the history
This avoids trashing connections just to immediately re-create them when
`num-worker` > 10. This should provide a pretty solid speedup on beefy
machines.

I'm not attaching any tests because this would be hard to test and if it
doesn't crash with an unknown keyword argument or something it means
it's probably going to be doing what we expect.
  • Loading branch information
adriangb committed Nov 3, 2022
1 parent d998dcd commit 4896c4b
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 7 deletions.
11 changes: 6 additions & 5 deletions src/poetry/installation/executor.py
Expand Up @@ -60,11 +60,6 @@ def __init__(
self._dry_run = False
self._enabled = True
self._verbose = False
self._authenticator = Authenticator(
config, self._io, disable_cache=disable_cache
)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env, config)

if parallel is None:
parallel = config.get("installer.parallel", True)
Expand All @@ -76,6 +71,12 @@ def __init__(
else:
self._max_workers = 1

self._authenticator = Authenticator(
config, self._io, disable_cache=disable_cache, pool_size=self._max_workers
)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env, config)

self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
self._total_operations = 0
self._executed_operations = 0
Expand Down
12 changes: 10 additions & 2 deletions src/poetry/utils/authenticator.py
Expand Up @@ -17,7 +17,7 @@
import requests.auth
import requests.exceptions

from cachecontrol import CacheControl
from cachecontrol import CacheControlAdapter
from cachecontrol.caches import FileCache
from filelock import FileLock

Expand Down Expand Up @@ -128,6 +128,7 @@ def __init__(
io: IO | None = None,
cache_id: str | None = None,
disable_cache: bool = False,
pool_size: int = 10,
) -> None:
self._config = config or Config.create()
self._io = io
Expand All @@ -153,14 +154,21 @@ def __init__(
self.get_repository_config_for_url = functools.lru_cache(maxsize=None)(
self._get_repository_config_for_url
)
self._pool_size = pool_size

def create_session(self) -> requests.Session:
session = requests.Session()

if self._cache_control is None:
return session

session = CacheControl(sess=session, cache=self._cache_control)
adapter = CacheControlAdapter(
cache=self._cache_control,
pool_maxsize=self._pool_size,
)
session.mount("http://", adapter)
session.mount("https://", adapter)

return session

def get_session(self, url: str | None = None) -> requests.Session:
Expand Down

0 comments on commit 4896c4b

Please sign in to comment.