diff --git a/README.rst b/README.rst index 77af67e..32887c5 100644 --- a/README.rst +++ b/README.rst @@ -293,8 +293,11 @@ Vladiate comes with the following input types: *class* ``S3File`` - Read from a file in S3. Uses the `boto `_ - library. Optionally can specify either a full path, or a bucket/key pair. + Read from a file in S3. Optionally can specify either a full path, or a + bucket/key pair. + + Requires the `boto `_ library, which should be + installed via ``pip install vladiate[s3]``. :``path=None``: A full S3 filepath (e.g., ``s3://foo.bar/path/to/file.csv``) diff --git a/setup.py b/setup.py index b5c8c72..67448c1 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand -__version__ = '0.0.18' +__version__ = '0.0.19' class PyTest(TestCommand): @@ -60,7 +60,8 @@ def readme(): packages=find_packages(exclude=['examples', 'tests']), include_package_data=True, zip_safe=False, - install_requires=['boto'], + install_requires=[], + extras_require={'s3': ['boto']}, tests_require=['pretend', 'pytest', 'flake8'], cmdclass={'test': PyTest}, entry_points={ diff --git a/tox.ini b/tox.ini index 80f6311..fa4df7a 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ basepython = python3.6 commands = flake8 [testenv] -commands = coverage run --source=vladiate setup.py test +commands = coverage run --append --source=vladiate setup.py test deps = pytest coverage diff --git a/vladiate/exceptions.py b/vladiate/exceptions.py index 46b7986..500977e 100644 --- a/vladiate/exceptions.py +++ b/vladiate/exceptions.py @@ -8,3 +8,12 @@ class BadValidatorException(Exception): def __init__(self, extra): message = 'Row does not contain the following unique_with fields: {}' super(BadValidatorException, self).__init__(message.format(extra)) + + +class MissingExtraException(Exception): + ''' Thrown when an extra dependency is missing ''' + def __init__(self): + super(MissingExtraException, self).__init__( + 'The `s3` extra is required to use the `S3File` class. Install' + ' it via `pip install vladiate[s3]`.' + ) diff --git a/vladiate/inputs.py b/vladiate/inputs.py index 4639604..c86b26c 100644 --- a/vladiate/inputs.py +++ b/vladiate/inputs.py @@ -1,5 +1,4 @@ import io -import boto try: from urlparse import urlparse except: @@ -9,6 +8,8 @@ except: from io import StringIO +from vladiate.exceptions import MissingExtraException + class VladInput(object): ''' A generic input class ''' @@ -41,6 +42,15 @@ class S3File(VladInput): ''' Read from a file in S3 ''' def __init__(self, path=None, bucket=None, key=None): + try: + import boto # noqa + self.boto = boto + except: + # 2.7 workaround, should just be `raise Exception() from None` + exc = MissingExtraException() + exc.__context__ = None + raise exc + if path and not any((bucket, key)): self.path = path parse_result = urlparse(path) @@ -57,7 +67,7 @@ def __init__(self, path=None, bucket=None, key=None): ) def open(self): - s3 = boto.connect_s3() + s3 = self.boto.connect_s3() bucket = s3.get_bucket(self.bucket) key = bucket.new_key(self.key) contents = key.get_contents_as_string() diff --git a/vladiate/test/test_inputs.py b/vladiate/test/test_inputs.py index b35843c..b2aa700 100644 --- a/vladiate/test/test_inputs.py +++ b/vladiate/test/test_inputs.py @@ -1,15 +1,32 @@ import pytest from pretend import stub, call, call_recorder +from ..exceptions import MissingExtraException from ..inputs import S3File, StringIO, String, VladInput from ..vlad import Vlad +def mock_boto(result): + try: + import builtins + except: + import __builtin__ as builtins + realimport = builtins.__import__ + + def badimport(name, *args, **kwargs): + if name == 'boto': + return result() + return realimport(name, *args, **kwargs) + + builtins.__import__ = badimport + + @pytest.mark.parametrize('kwargs', [ ({'path': 's3://some.bucket/some/s3/key.csv'}), ({'bucket': 'some.bucket', 'key': '/some/s3/key.csv'}), ]) def test_s3_input_works(kwargs): + mock_boto(lambda: stub()) S3File(**kwargs) @@ -21,6 +38,7 @@ def test_s3_input_works(kwargs): ({'key': '/some/s3/key.csv'}), ]) def test_s3_input_fails(kwargs): + mock_boto(lambda: stub()) with pytest.raises(ValueError): S3File(**kwargs) @@ -35,7 +53,7 @@ def test_string_input_works(kwargs): assert Vlad(source=source, validators=validators).validate() -def test_open_s3file(monkeypatch): +def test_open_s3file(): new_key = call_recorder(lambda *args, **kwargs: stub( get_contents_as_string=lambda: 'contents'.encode() )) @@ -44,9 +62,10 @@ def test_open_s3file(monkeypatch): mock_boto = stub(connect_s3=lambda: stub(get_bucket=get_bucket)) - monkeypatch.setattr('vladiate.inputs.boto', mock_boto) + s3file = S3File('s3://some.bucket/some/s3/key.csv') + s3file.boto = mock_boto - result = S3File('s3://some.bucket/some/s3/key.csv').open() + result = s3file.open() assert get_bucket.calls == [ call('some.bucket') @@ -77,3 +96,13 @@ def __init__(self): with pytest.raises(NotImplementedError): repr(PartiallyImplemented()) + + +def test_s3file_raises_when_no_boto(): + def import_result(): + raise ImportError + + mock_boto(import_result) + + with pytest.raises(MissingExtraException): + S3File()