Commit 4896c4bd by Adrian Garcia Badaracco Committed by GitHub

fix: match requests pool_maxsize to num workers (#6805)

This avoids trashing connections just to immediately re-create them when
`num-worker` > 10. This should provide a pretty solid speedup on beefy
machines.

I'm not attaching any tests because this would be hard to test and if it
doesn't crash with an unknown keyword argument or something it means
it's probably going to be doing what we expect.
parent d998dcd0
......@@ -60,11 +60,6 @@ class Executor:
self._dry_run = False
self._enabled = True
self._verbose = False
self._authenticator = Authenticator(
config, self._io, disable_cache=disable_cache
)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env, config)
if parallel is None:
parallel = config.get("installer.parallel", True)
......@@ -76,6 +71,12 @@ class Executor:
else:
self._max_workers = 1
self._authenticator = Authenticator(
config, self._io, disable_cache=disable_cache, pool_size=self._max_workers
)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env, config)
self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
self._total_operations = 0
self._executed_operations = 0
......
......@@ -17,7 +17,7 @@ import requests
import requests.auth
import requests.exceptions
from cachecontrol import CacheControl
from cachecontrol import CacheControlAdapter
from cachecontrol.caches import FileCache
from filelock import FileLock
......@@ -128,6 +128,7 @@ class Authenticator:
io: IO | None = None,
cache_id: str | None = None,
disable_cache: bool = False,
pool_size: int = 10,
) -> None:
self._config = config or Config.create()
self._io = io
......@@ -153,6 +154,7 @@ class Authenticator:
self.get_repository_config_for_url = functools.lru_cache(maxsize=None)(
self._get_repository_config_for_url
)
self._pool_size = pool_size
def create_session(self) -> requests.Session:
session = requests.Session()
......@@ -160,7 +162,13 @@ class Authenticator:
if self._cache_control is None:
return session
session = CacheControl(sess=session, cache=self._cache_control)
adapter = CacheControlAdapter(
cache=self._cache_control,
pool_maxsize=self._pool_size,
)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def get_session(self, url: str | None = None) -> requests.Session:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment