Refactor cookie handling in CookiesMiddleware

scrapy/downloadermiddlewares/cookies.py

@@ -66,6 +66,17 @@ def process_response(self, request, response, spider):
         if request.meta.get("dont_merge_cookies", False):
             return response
 
+        for cookie in response.headers.getlist("Set-Cookie"):
+            cookie_domain = cookie.split(b";")[1].split(b"=")[1]
+            cookie_domain = cookie_domain.decode("utf-8")
+            request_domain = urlparse_cached(request).hostname.lower()
+            if cookie_domain == '/':
+                continue
+            if cookie_domain not in request_domain:
+                # set cookie domain to request domain
+                cookie = cookie.replace(cookie_domain.encode(), request_domain.encode())
+                response.headers.setlist("Set-Cookie", [cookie])
+
         # extract cookies from Set-Cookie and drop invalid/expired cookies
         cookiejarkey = request.meta.get("cookiejar")
         jar = self.jars[cookiejarkey]

scrapy/http/cookies.py

@@ -11,9 +11,15 @@
 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
 
 
+class CustomDefaultCookiePolicy(DefaultCookiePolicy):
+    def set_ok(self, cookie, request):
+        # always set cookies
+        return True
+
+
 class CookieJar:
     def __init__(self, policy=None, check_expired_frequency=10000):
-        self.policy = policy or DefaultCookiePolicy()
+        self.policy = policy or CustomDefaultCookiePolicy()
         self.jar = _CookieJar(self.policy)
         self.jar._cookies_lock = _DummyLock()
         self.check_expired_frequency = check_expired_frequency

tests/test_downloadermiddleware_cookies.py

@@ -732,3 +732,29 @@ def test_server_set_cookie_domain_public_period(self):
             "co.uk",
             cookies=True,
         )
+
+    def test_off_domain_jar_storage(self):
+        request1 = Request(
+            "https://a.example",
+            cookies=[
+                {
+                    "name": "foo",
+                    "value": "bar",
+                    "domain": "b.example",
+                },
+            ],
+        )
+        assert self.mw.process_request(request1, self.spider) is None
+        self.assertNotIn(b"Cookie", request1.headers)
+
+        request2 = Request("https://b.example/")
+        assert self.mw.process_request(request2, self.spider) is None
+        self.assertEqual(request2.headers.get(b"Cookie"), b"foo=bar")
+
+    def test_process_response_unrelated_domain(self):
+        request = Request("http://a.example")
+        response = Response("http://a.example")
+        response.headers["Set-Cookie"] = "foo=bar; domain=b.example"
+        self.mw.process_response(request, response, spider=None)
+        jar = self.mw.jars[None]
+        self.assertFalse(jar._cookies)