Edits to media_downloaded in files.py to handle 201 response status (#1615 and #1806)

scrapy/pipelines/files.py

@@ -479,7 +479,7 @@ def media_failed(self, failure, request, info):
     def media_downloaded(self, response, request, info, *, item=None):
         referer = referer_str(request)
 
-        if response.status != 200:
+        if response.status not in [200, 201]:
             logger.warning(
                 "File (code: %(status)s): Error downloading file from "
                 "%(request)s referred in <%(referer)s>",
@@ -489,13 +489,17 @@ def media_downloaded(self, response, request, info, *, item=None):
             raise FileException("download-error")
 
         if not response.body:
-            logger.warning(
-                "File (empty-content): Empty file from %(request)s referred "
-                "in <%(referer)s>: no-content",
-                {"request": request, "referer": referer},
-                extra={"spider": info.spider},
-            )
-            raise FileException("empty-content")
+            if response.status == 201 and 'Location' in response.headers:
+                extra_request = Request(response.headers['Location'], callback=self.file_downloaded, meta={'info': info, 'item': item})
+                return extra_request
+            else:
+                logger.warning(
+                    "File (empty-content): Empty file from %(request)s referred "
+                    "in <%(referer)s>: no-content",
+                    {"request": request, "referer": referer},
+                    extra={"spider": info.spider},
+                )
+                raise FileException("empty-content")
 
         status = "cached" if "cached" in response.flags else "downloaded"
         logger.debug(

tests/keys/mitmproxy-dhparam.pem

@@ -0,0 +1,14 @@
+
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3
+O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv
+j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ
+Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB
+chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC
+ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq
+o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX
+IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv
+A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8
+6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I
+rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
+-----END DH PARAMETERS-----

tests/test_pipeline_media.py

@@ -67,6 +67,11 @@ def test_default_media_downloaded(self):
         response = Response("http://url", body=b"")
         assert self.pipe.media_downloaded(response, request, self.info) is response
 
+    def test_media_downloaded_201(self):
+        request = Request("http://p2.so.qhimgs1.com/t013690341ce9bfc61e.jpg")
+        response = Response("http://p2.so.qhimgs1.com/t013690341ce9bfc61e.jpg", body=b"")
+        assert self.pipe.media_downloaded(response, request, self.info) is response
+
     def test_default_media_failed(self):
         request = Request("http://url")
         fail = Failure(Exception())