Add Downloader.get_slot_key() without a spider parameter (#6352)

docs/news.rst

@@ -3,6 +3,20 @@
 Release notes
 =============
 
+
+.. _release-VERSION:
+
+Scrapy VERSION (YYYY-MM-DD)
+---------------------------
+
+Deprecations
+~~~~~~~~~~~~
+
+-   :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated.
+    Consider using its corresponding public method get_slot_key() instead.
+    (:issue:`6340`)
+
+
 .. _release-2.11.1:
 
 Scrapy 2.11.1 (2024-02-14)

scrapy/core/downloader/__init__.py

@@ -1,4 +1,5 @@
 import random
+import warnings
 from collections import deque
 from datetime import datetime
 from time import time
@@ -10,6 +11,7 @@
 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response
 from scrapy.resolver import dnscache
 from scrapy.settings import BaseSettings
@@ -125,7 +127,7 @@ def needs_backout(self) -> bool:
         return len(self.active) >= self.total_concurrency
 
     def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
-        key = self._get_slot_key(request, spider)
+        key = self.get_slot_key(request)
         if key not in self.slots:
             slot_settings = self.per_slot_settings.get(key, {})
             conc = (
@@ -143,7 +145,7 @@ def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
 
         return key, self.slots[key]
 
-    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+    def get_slot_key(self, request: Request) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
             return cast(str, request.meta[self.DOWNLOAD_SLOT])
 
@@ -153,6 +155,14 @@ def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
 
         return key
 
+    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+        warnings.warn(
+            "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.get_slot_key(request)
+
     def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
         key, slot = self._get_slot(request, spider)
         request.meta[self.DOWNLOAD_SLOT] = key

scrapy/pqueues.py

@@ -180,7 +180,7 @@ def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:
         return [(self._active_downloads(slot), slot) for slot in possible_slots]
 
     def get_slot_key(self, request: Request) -> str:
-        return self.downloader._get_slot_key(request, None)
+        return self.downloader.get_slot_key(request)
 
     def _active_downloads(self, slot: str) -> int:
         """Return a number of requests in a Downloader for a given slot"""

tests/test_scheduler.py

@@ -25,7 +25,7 @@ class MockDownloader:
     def __init__(self):
         self.slots = {}
 
-    def _get_slot_key(self, request, spider):
+    def get_slot_key(self, request):
         if Downloader.DOWNLOAD_SLOT in request.meta:
             return request.meta[Downloader.DOWNLOAD_SLOT]
 
@@ -273,14 +273,14 @@ def test_logic(self):
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
 
         for request in requests:
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             downloader.decrement(slot)
 
         self.assertTrue(