Merge pull request scrapinghub#53 from starrify/master

changed: Constraint `RequiredFields`'s behaviour. Solves scrapinghub#52
starrify · Mar 25, 2015 · 27663d4 · 27663d4
2 parents 84b0e38 + f1f7b56
commit 27663d4
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 5 deletions.
diff --git a/scrapylib/constraints/__init__.py b/scrapylib/constraints/__init__.py
@@ -33,15 +33,29 @@ class Product(Item):
 
 
 class RequiredFields(object):
+    """Assert that the specified fields are populated"""
+
+    def __init__(self, *fields):
+        self.fields = fields
+
+    def __call__(self, item):
+        for f in self.fields:
+            assert f in item.keys(), "missing field: %s" % f
+
+class NonEmptyFields(object):
     """Assert that the specified fields are populated and non-empty"""
 
     def __init__(self, *fields):
         self.fields = fields
 
     def __call__(self, item):
         for f in self.fields:
-            v = item.get(f)
-            assert v, "missing field: %s" % f
+            assert f in item.keys(), "missing field: %s" % f
+            v = item[f]
+            try:
+                assert len(v) > 0, "empty field: %s" % f
+            except TypeError:
+                pass
 
 class IsType(object):
     """Assert that the specified fields are of the given type"""

diff --git a/tests/test_constraints.py b/tests/test_constraints.py
@@ -1,21 +1,36 @@
 import unittest
 
-from scrapylib.constraints import RequiredFields, IsType, IsNumber, IsPrice, MaxLen, MinLen
+from scrapylib.constraints import RequiredFields, NonEmptyFields, IsType, IsNumber, IsPrice, MaxLen, MinLen
 
 
 class RequiredFieldsTest(unittest.TestCase):
 
     def setUp(self):
-        self.item = {'str': 'bar', 'list': ['one']}
+        self.item = {'str': 'bar', 'list': ['one'], 'bool': False, 'none': None}
 
     def test_basic(self):
         RequiredFields('str')(self.item)
-        RequiredFields('str', 'list')(self.item)
+        RequiredFields('str', 'list', 'bool', 'none')(self.item)
 
     def test_fail(self):
         self.assertRaises(AssertionError, RequiredFields('list', 'xxx'), self.item)
 
 
+class NonEmptyFieldsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.item = {'str': 'foo', 'list': [0], 'empty_str': '', 'empty_list': []}
+
+    def test_basic(self):
+        NonEmptyFields('str')(self.item)
+        NonEmptyFields('str', 'list')(self.item)
+
+    def test_fail(self):
+        self.assertRaises(AssertionError, NonEmptyFields('list', 'xxx'), self.item)
+        self.assertRaises(AssertionError, NonEmptyFields('empty_str'), self.item)
+        self.assertRaises(AssertionError, NonEmptyFields('empty_list'), self.item)
+
+
 class IsTypeTest(unittest.TestCase):
 
     def setUp(self):