Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Edits to media_downloaded in files.py to handle 201 response status (#1615 and #1806) #6309

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Edits to media_downloaded in files.py to handle 201 response status (#1615 and #1806) #6309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
27c0fd6
test 201 status code
zoemyatt Apr 7, 2024
c17d670
Delete tests/keys/mitmproxy-dhparam.pem
zoemyatt Apr 13, 2024
934a70f
Update test_pipeline_media.py
zoemyatt Apr 13, 2024
fd31c49
pre-commit
zoemyatt Apr 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 16 additions & 8 deletions scrapy/pipelines/files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@ def media_failed(self, failure, request, info):
def media_downloaded(self, response, request, info, *, item=None):
referer = referer_str(request)

if response.status != 200:
if response.status not in [200, 201]:
logger.warning(
"File (code: %(status)s): Error downloading file from "
"%(request)s referred in <%(referer)s>",
Expand All @@ -489,13 +489,21 @@ def media_downloaded(self, response, request, info, *, item=None):
raise FileException("download-error")

if not response.body:
logger.warning(
"File (empty-content): Empty file from %(request)s referred "
"in <%(referer)s>: no-content",
{"request": request, "referer": referer},
extra={"spider": info.spider},
)
raise FileException("empty-content")
if response.status == 201 and "Location" in response.headers:
extra_request = Request(
response.headers["Location"],
callback=self.file_downloaded,
meta={"info": info, "item": item},
)
return extra_request
else:
logger.warning(
"File (empty-content): Empty file from %(request)s referred "
"in <%(referer)s>: no-content",
{"request": request, "referer": referer},
extra={"spider": info.spider},
)
raise FileException("empty-content")

status = "cached" if "cached" in response.flags else "downloaded"
logger.debug(
Expand Down
7 changes: 7 additions & 0 deletions tests/test_pipeline_media.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ def test_default_media_downloaded(self):
response = Response("http://url", body=b"")
assert self.pipe.media_downloaded(response, request, self.info) is response

def test_media_downloaded_201(self):
request = Request("http://url")
# Create a response object with status code 201
response = Response("http://url", status=201)
# Call the media_downloaded method and assert that it returns the response object
assert self.pipe.media_downloaded(response, request, self.info) is response

def test_default_media_failed(self):
request = Request("http://url")
fail = Failure(Exception())
Expand Down