new allow_offsite parameter in OffsiteMiddleware

docs/topics/request-response.rst

@@ -144,9 +144,9 @@ Request objects
     :type priority: int
 
     :param dont_filter: indicates that this request should not be filtered by
-       the scheduler. This is used when you want to perform an identical
-       request multiple times, to ignore the duplicates filter. Use it with
-       care, or you will get into crawling loops. Default to ``False``.
+       the scheduler or some middlewares. This is used when you want to perform
+       an identical request multiple times, to ignore the duplicates filter.
+       Use it with care, or you will get into crawling loops. Default to ``False``.
     :type dont_filter: bool
 
     :param errback: a function that will be called if any exception was

docs/topics/spider-middleware.rst

@@ -345,9 +345,10 @@ OffsiteMiddleware
    :attr:`~scrapy.Spider.allowed_domains` attribute, or the
    attribute is empty, the offsite middleware will allow all requests.
 
-   If the request has the :attr:`~scrapy.Request.dont_filter` attribute
-   set, the offsite middleware will allow the request even if its domain is not
-   listed in allowed domains.
+   If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
+   ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
+   the OffsiteMiddleware will allow the request even if its domain is not listed
+   in allowed domains.
 
 
 RefererMiddleware

scrapy/spidermiddlewares/offsite.py

@@ -50,7 +50,11 @@ async def process_spider_output_async(
     def _filter(self, request: Any, spider: Spider) -> bool:
         if not isinstance(request, Request):
             return True
-        if request.dont_filter or self.should_follow(request, spider):
+        if (
+            request.dont_filter
+            or request.meta.get("allow_offsite")
+            or self.should_follow(request, spider)
+        ):
             return True
         domain = urlparse_cached(request).hostname
         if domain and domain not in self.domains_seen:

tests/test_spidermiddleware_offsite.py

@@ -29,6 +29,7 @@ def test_process_spider_output(self):
             Request("http://scrapy.org/1"),
             Request("http://sub.scrapy.org/1"),
             Request("http://offsite.tld/letmepass", dont_filter=True),
+            Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}),
             Request("http://scrapy.test.org/"),
             Request("http://scrapy.test.org:8000/"),
         ]