diff --git a/glance/store/__init__.py b/glance/store/__init__.py index ae0c5ed2b6..2955217d8a 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -53,13 +53,17 @@ def get_backend_class(backend): """ # NOTE(sirp): avoiding circular import from glance.store.http import HTTPBackend + from glance.store.s3 import S3Backend from glance.store.swift import SwiftBackend from glance.store.filesystem import FilesystemBackend - BACKENDS = {"file": FilesystemBackend, - "http": HTTPBackend, - "https": HTTPBackend, - "swift": SwiftBackend} + BACKENDS = { + "file": FilesystemBackend, + "http": HTTPBackend, + "https": HTTPBackend, + "swift": SwiftBackend, + "s3": S3Backend + } try: return BACKENDS[backend] @@ -99,3 +103,41 @@ def get_store_from_location(location): """ loc_pieces = urlparse.urlparse(location) return loc_pieces.scheme + + +def parse_uri_tokens(parsed_uri, example_url): + """ + Given a URI and an example_url, attempt to parse the uri to assemble an + authurl. This method returns the user, key, authurl, referenced container, + and the object we're looking for in that container. + + Parsing the uri is three phases: + 1) urlparse to split the tokens + 2) use RE to split on @ and / + 3) reassemble authurl + + """ + path = parsed_uri.path.lstrip('//') + netloc = parsed_uri.netloc + + try: + try: + creds, netloc = netloc.split('@') + except ValueError: + # Python 2.6.1 compat + # see lp659445 and Python issue7904 + creds, path = path.split('@') + user, key = creds.split(':') + path_parts = path.split('/') + obj = path_parts.pop() + container = path_parts.pop() + except (ValueError, IndexError): + raise BackendException( + "Expected four values to unpack in: %s:%s. " + "Should have received something like: %s." + % (parsed_uri.scheme, parsed_uri.path, example_url)) + + authurl = "https://%s" % '/'.join(path_parts) + + return user, key, authurl, container, obj + diff --git a/glance/store/backends/__init__.py b/glance/store/backends/__init__.py index a90ac4879f..14d83a9112 100644 --- a/glance/store/backends/__init__.py +++ b/glance/store/backends/__init__.py @@ -14,107 +14,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -import os -import urlparse - -from glance.common import exception - - -# TODO(sirp): should this be moved out to common/utils.py ? -def _file_iter(f, size): - """ - Return an iterator for a file-like object - """ - chunk = f.read(size) - while chunk: - yield chunk - chunk = f.read(size) - - -class BackendException(Exception): - pass - - -class UnsupportedBackend(BackendException): - pass - - -class Backend(object): - CHUNKSIZE = 4096 - - -class FilesystemBackend(Backend): - @classmethod - def get(cls, parsed_uri, expected_size, opener=lambda p: open(p, "rb")): - """ Filesystem-based backend - - file:///path/to/file.tar.gz.0 - """ - #FIXME: must prevent attacks using ".." and "." paths - with opener(parsed_uri.path) as f: - return _file_iter(f, cls.CHUNKSIZE) - - @classmethod - def delete(cls, parsed_uri): - """ - Removes a file from the filesystem backend. - - :param parsed_uri: Parsed pieces of URI in form of:: - file:///path/to/filename.ext - - :raises NotFound if file does not exist - :raises NotAuthorized if cannot delete because of permissions - """ - fn = parsed_uri.path - if os.path.exists(fn): - try: - os.unlink(fn) - except OSError: - raise exception.NotAuthorized("You cannot delete file %s" % fn) - else: - raise exception.NotFound("File %s does not exist" % fn) - - -def get_backend_class(backend): - """ - Returns the backend class as designated in the - backend name - - :param backend: Name of backend to create - """ - # NOTE(sirp): avoiding circular import - from glance.store.backends.http import HTTPBackend - from glance.store.backends.swift import SwiftBackend - - BACKENDS = {"file": FilesystemBackend, - "http": HTTPBackend, - "https": HTTPBackend, - "swift": SwiftBackend} - - try: - return BACKENDS[backend] - except KeyError: - raise UnsupportedBackend("No backend found for '%s'" % scheme) - - -def get_from_backend(uri, **kwargs): - """Yields chunks of data from backend specified by uri""" - - parsed_uri = urlparse.urlparse(uri) - scheme = parsed_uri.scheme - - backend_class = get_backend_class(scheme) - - return backend_class.get(parsed_uri, **kwargs) - - -def delete_from_backend(uri, **kwargs): - """Removes chunks of data from backend specified by uri""" - - parsed_uri = urlparse.urlparse(uri) - scheme = parsed_uri.scheme - - backend_class = get_backend_class(scheme) - - return backend_class.delete(parsed_uri, **kwargs) diff --git a/glance/store/s3.py b/glance/store/s3.py new file mode 100644 index 0000000000..f235279bd4 --- /dev/null +++ b/glance/store/s3.py @@ -0,0 +1,109 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The s3 backend adapter""" + +import glance.store + + +class S3Backend(glance.store.Backend): + """An implementation of the s3 adapter.""" + + EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0" + + @classmethod + def get(cls, parsed_uri, expected_size, conn_class=None): + """ + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects + to s3 and downloads the file. Returns the generator resp_body provided + by get_object. + """ + + if conn_class: + pass + else: + import boto.s3.connection + conn_class = boto.s3.connection.S3Connection + + (access_key, secret_key, host, bucket, obj) = \ + cls._parse_s3_tokens(parsed_uri) + + # Close the connection when we're through. + with conn_class(access_key, secret_key, host=host) as s3_conn: + bucket = cls._get_bucket(s3_conn, bucket) + + # Close the key when we're through. + with cls._get_key(bucket, obj) as key: + if not key.size == expected_size: + raise glance.store.BackendException( + "Expected %s bytes, got %s" % + (expected_size, key.size)) + + key.BufferSize = cls.CHUNKSIZE + for chunk in key: + yield chunk + + @classmethod + def delete(cls, parsed_uri, conn_class=None): + """ + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects + to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete() + returns. + """ + + if conn_class: + pass + else: + conn_class = boto.s3.connection.S3Connection + + (access_key, secret_key, host, bucket, obj) = \ + cls._parse_s3_tokens(parsed_uri) + + # Close the connection when we're through. + with conn_class(access_key, secret_key, host=host) as s3_conn: + bucket = cls._get_bucket(s3_conn, bucket) + + # Close the key when we're through. + with cls._get_key(bucket, obj) as key: + return key.delete() + + @classmethod + def _get_bucket(cls, conn, bucket_id): + """Get a bucket from an s3 connection""" + + bucket = conn.get_bucket(bucket_id) + if not bucket: + raise glance.store.BackendException("Could not find bucket: %s" % + bucket_id) + + return bucket + + @classmethod + def _get_key(cls, bucket, obj): + """Get a key from a bucket""" + + key = bucket.get_key(obj) + if not key: + raise glance.store.BackendException("Could not get key: %s" % key) + return key + + @classmethod + def _parse_s3_tokens(cls, parsed_uri): + """Parse tokens from the parsed_uri""" + return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL) diff --git a/glance/store/swift.py b/glance/store/swift.py index 71a259bb51..68655e78bc 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import import glance.store @@ -114,21 +115,7 @@ def _parse_swift_tokens(cls, parsed_uri): def get_connection_class(conn_class): - if conn_class: - pass # Use the provided conn_class - else: - # NOTE(sirp): A standard import statement won't work here because - # this file ('swift.py') is shadowing the swift module, and since - # the import statement searches locally before globally, we'd end - # up importing ourselves. - # - # NOTE(jaypipes): This can be resolved by putting this code in - # /glance/store/swift/__init__.py - # - # see http://docs.python.org/library/functions.html#__import__ - PERFORM_ABSOLUTE_IMPORTS = 0 - swift = __import__('swift.common.client', globals(), locals(), [], - PERFORM_ABSOLUTE_IMPORTS) - + if not conn_class: + import swift.common.client conn_class = swift.common.client.Connection return conn_class diff --git a/tests/stubs.py b/tests/stubs.py index f785e8e4a8..1b8d9b28c3 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -105,6 +105,44 @@ def stub_out_filesystem_backend(): f.close() +def stub_out_s3_backend(stubs): + """ Stubs out the S3 Backend with fake data and calls. + + The stubbed swift backend provides back an iterator over + the data "" + + :param stubs: Set of stubout stubs + + """ + + class FakeSwiftAuth(object): + pass + class FakeS3Connection(object): + pass + + class FakeS3Backend(object): + CHUNK_SIZE = 2 + DATA = 'I am a teapot, short and stout\n' + + @classmethod + def get(cls, parsed_uri, expected_size, conn_class=None): + S3Backend = glance.store.s3.S3Backend + + # raise BackendException if URI is bad. + (user, key, authurl, container, obj) = \ + S3Backend._parse_s3_tokens(parsed_uri) + + def chunk_it(): + for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE): + yield cls.DATA[i:i+cls.CHUNK_SIZE] + + return chunk_it() + + fake_swift_backend = FakeS3Backend() + stubs.Set(glance.store.s3.S3Backend, 'get', + fake_swift_backend.get) + + def stub_out_swift_backend(stubs): """Stubs out the Swift Glance backend with fake data and calls. diff --git a/tests/unit/test_stores.py b/tests/unit/test_stores.py index 212a39c816..24c02d3f30 100644 --- a/tests/unit/test_stores.py +++ b/tests/unit/test_stores.py @@ -21,6 +21,7 @@ import unittest import urlparse +from glance.store.s3 import S3Backend from glance.store.swift import SwiftBackend from glance.store import Backend, BackendException, get_from_backend from tests import stubs @@ -87,6 +88,25 @@ def test_https_get(self): self.assertEqual(chunks, expected_returns) +class TestS3Backend(TestBackend): + def setUp(self): + super(TestS3Backend, self).setUp() + stubs.stub_out_s3_backend(self.stubs) + + def test_get(self): + s3_uri = "s3://user:password@localhost/bucket1/file.tar.gz" + + expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', + 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] + fetcher = get_from_backend(s3_uri, + expected_size=8, + conn_class=S3Backend) + + chunks = [c for c in fetcher] + self.assertEqual(chunks, expected_returns) + + + class TestSwiftBackend(TestBackend): def setUp(self):