Merge pull request #3819 from sortafreel/fix_missing_values

[MRG+1] Fix missing values
scrapy · Jun 13, 2019 · c05d24c · c05d24c
2 parents 66c40d0 + cdeccac
commit c05d24c
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 0 deletions.
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
@@ -35,6 +35,10 @@ def __init__(self, item=None, selector=None, response=None, parent=None, **conte
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
+        # Preprocess values if item built from dict
+        # Values need to be added to item._values if added them from dict (not with add_values)
+        for field_name, value in item.items():
+            self._values[field_name] = self._process_input_value(field_name, value)
 
     @property
     def _values(self):

diff --git a/tests/test_loader.py b/tests/test_loader.py
@@ -419,6 +419,43 @@ class TestItemLoader(NameItemLoader):
         self.assertEqual(item['url'], u'rabbit.hole')
         self.assertEqual(item['summary'], u'rabbithole')
 
+    def test_create_item_from_dict(self):
+        class TestItem(Item):
+            title = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        input_item = {'title': 'Test item title 1'}
+        il = TestItemLoader(item=input_item)
+        # Getting output value mustn't remove value from item
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+        self.assertEqual(il.get_output_value('title'), 'Test item title 1')
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+
+        input_item = {'title': 'Test item title 2'}
+        il = TestItemLoader(item=input_item)
+        # Values from dict must be added to item _values
+        self.assertEqual(il._values.get('title'), 'Test item title 2')
+
+        input_item = {'title': [u'Test item title 3', u'Test item 4']}
+        il = TestItemLoader(item=input_item)
+        # Same rules must work for lists
+        self.assertEqual(il._values.get('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+        self.assertEqual(il.get_output_value('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+
 
 class ProcessorsTest(unittest.TestCase):