Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MRG+1] Fix missing values #3819

Merged
merged 5 commits into from Jun 13, 2019
Merged

[MRG+1] Fix missing values #3819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bd8a103
Add values (if there're any) when initiating items from dicts
sortafreel Jun 6, 2019
754f52b
Preprocess values if item built from dict.
sortafreel Jun 7, 2019
a1bca6a
Add tests.
sortafreel Jun 11, 2019
7dad2f7
Add more tests.
sortafreel Jun 11, 2019
cdeccac
Linting (return previous indentation).
sortafreel Jun 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions scrapy/loader/__init__.py
View file
Open in desktop
Expand Up @@ -35,6 +35,10 @@ def __init__(self, item=None, selector=None, response=None, parent=None, **conte
self.parent = parent
self._local_item = context['item'] = item
self._local_values = defaultdict(list)
# Preprocess values if item built from dict
# Values need to be added to item._values if added them from dict (not with add_values)
for field_name, value in item.items():
self._values[field_name] = self._process_input_value(field_name, value)

@property
def _values(self):
Expand Down
37 changes: 37 additions & 0 deletions tests/test_loader.py
View file
Open in desktop
Expand Up @@ -419,6 +419,43 @@ class TestItemLoader(NameItemLoader):
self.assertEqual(item['url'], u'rabbit.hole')
self.assertEqual(item['summary'], u'rabbithole')

def test_create_item_from_dict(self):
class TestItem(Item):
title = Field()

class TestItemLoader(ItemLoader):
default_item_class = TestItem

input_item = {'title': 'Test item title 1'}
il = TestItemLoader(item=input_item)
# Getting output value mustn't remove value from item
self.assertEqual(il.load_item(), {
'title': 'Test item title 1',
})
self.assertEqual(il.get_output_value('title'), 'Test item title 1')
self.assertEqual(il.load_item(), {
'title': 'Test item title 1',
})

input_item = {'title': 'Test item title 2'}
il = TestItemLoader(item=input_item)
# Values from dict must be added to item _values
self.assertEqual(il._values.get('title'), 'Test item title 2')

input_item = {'title': [u'Test item title 3', u'Test item 4']}
il = TestItemLoader(item=input_item)
# Same rules must work for lists
self.assertEqual(il._values.get('title'),
[u'Test item title 3', u'Test item 4'])
self.assertEqual(il.load_item(), {
'title': [u'Test item title 3', u'Test item 4'],
})
self.assertEqual(il.get_output_value('title'),
[u'Test item title 3', u'Test item 4'])
self.assertEqual(il.load_item(), {
'title': [u'Test item title 3', u'Test item 4'],
})


class ProcessorsTest(unittest.TestCase):

Expand Down