From 786f000b39340461b55a793455c2da0fbd95840a Mon Sep 17 00:00:00 2001 From: Christopher MacGown Date: Wed, 5 Jan 2011 10:47:21 +0100 Subject: [PATCH 1/3] Implement S3 to the level of swift --- glance/store/__init__.py | 44 +++++++++++- glance/store/backends/__init__.py | 6 +- glance/store/s3.py | 110 ++++++++++++++++++++++++++++++ glance/store/swift.py | 19 +----- tests/stubs.py | 38 +++++++++++ tests/unit/test_stores.py | 20 ++++++ 6 files changed, 217 insertions(+), 20 deletions(-) create mode 100644 glance/store/s3.py diff --git a/glance/store/__init__.py b/glance/store/__init__.py index ff4463afeb..2955217d8a 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -53,6 +53,7 @@ def get_backend_class(backend): """ # NOTE(sirp): avoiding circular import from glance.store.http import HTTPBackend + from glance.store.s3 import S3Backend from glance.store.swift import SwiftBackend from glance.store.filesystem import FilesystemBackend @@ -60,13 +61,14 @@ def get_backend_class(backend): "file": FilesystemBackend, "http": HTTPBackend, "https": HTTPBackend, - "swift": SwiftBackend + "swift": SwiftBackend, + "s3": S3Backend } try: return BACKENDS[backend] except KeyError: - raise UnsupportedBackend("No backend found for '%s'" % scheme) + raise UnsupportedBackend("No backend found for '%s'" % backend) def get_from_backend(uri, **kwargs): @@ -101,3 +103,41 @@ def get_store_from_location(location): """ loc_pieces = urlparse.urlparse(location) return loc_pieces.scheme + + +def parse_uri_tokens(parsed_uri, example_url): + """ + Given a URI and an example_url, attempt to parse the uri to assemble an + authurl. This method returns the user, key, authurl, referenced container, + and the object we're looking for in that container. + + Parsing the uri is three phases: + 1) urlparse to split the tokens + 2) use RE to split on @ and / + 3) reassemble authurl + + """ + path = parsed_uri.path.lstrip('//') + netloc = parsed_uri.netloc + + try: + try: + creds, netloc = netloc.split('@') + except ValueError: + # Python 2.6.1 compat + # see lp659445 and Python issue7904 + creds, path = path.split('@') + user, key = creds.split(':') + path_parts = path.split('/') + obj = path_parts.pop() + container = path_parts.pop() + except (ValueError, IndexError): + raise BackendException( + "Expected four values to unpack in: %s:%s. " + "Should have received something like: %s." + % (parsed_uri.scheme, parsed_uri.path, example_url)) + + authurl = "https://%s" % '/'.join(path_parts) + + return user, key, authurl, container, obj + diff --git a/glance/store/backends/__init__.py b/glance/store/backends/__init__.py index df2935826c..4344e59c34 100644 --- a/glance/store/backends/__init__.py +++ b/glance/store/backends/__init__.py @@ -86,18 +86,20 @@ def get_backend_class(backend): # NOTE(sirp): avoiding circular import from glance.store.backends.http import HTTPBackend from glance.store.backends.swift import SwiftBackend + from glance.store.backends.s3 import S3Backend BACKENDS = { "file": FilesystemBackend, "http": HTTPBackend, "https": HTTPBackend, - "swift": SwiftBackend + "swift": SwiftBackend, + "s3": S3Backend } try: return BACKENDS[backend] except KeyError: - raise UnsupportedBackend("No backend found for '%s'" % scheme) + raise UnsupportedBackend("No backend found for '%s'" % backend) def get_from_backend(uri, **kwargs): diff --git a/glance/store/s3.py b/glance/store/s3.py new file mode 100644 index 0000000000..f9c134a14c --- /dev/null +++ b/glance/store/s3.py @@ -0,0 +1,110 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" the s3 backend adapter """ + +from __future__ import absolute_import +import glance.store +import boto.s3.connection + + +class S3Backend(glance.store.Backend): + """ An implementation of the s3 adapter. """ + + EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0" + + @classmethod + def get(cls, parsed_uri, expected_size, conn_class=None): + """ + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3 + and downloads the file. Returns the generator resp_body provided by + get_object. + """ + + if conn_class: + pass + else: + conn_class = boto.s3.connection.S3Connection + + (access_key, secret_key, host, bucket, obj) = \ + cls._parse_s3_tokens(parsed_uri) + + # Close the connection when we're through. + with conn_class(access_key, secret_key, host=host) as s3_conn: + bucket = cls._get_bucket(s3_conn, bucket) + + # Close the key when we're through. + with cls._get_key(bucket, obj) as key: + if not key.size == expected_size: + raise glance.store.BackendException("Expected %s bytes, got %s" + % (expected_size, key.size)) + + key.BufferSize = cls.CHUNKSIZE + for chunk in key: + yield chunk + + @classmethod + def delete(cls, parsed_uri, conn_class=None): + """ + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3 + and deletes the file. Returns whatever boto.s3.key.Key.delete() returns. + """ + + if conn_class: + pass + else: + conn_class = boto.s3.connection.S3Connection + + (access_key, secret_key, host, bucket, obj) = \ + cls._parse_s3_tokens(parsed_uri) + + # Close the connection when we're through. + with conn_class(access_key, secret_key, host=host) as s3_conn: + bucket = cls._get_bucket(s3_conn, bucket) + + # Close the key when we're through. + with cls._get_key(bucket, obj) as key: + return key.delete() + + + + @classmethod + def _get_bucket(cls, conn, bucket_id): + """ Get a bucket from an s3 connection """ + + bucket = conn.get_bucket(bucket_id) + if not bucket: + raise glance.store.BackendException("Could not find bucket: %s" % + bucket_id) + + return bucket + + @classmethod + def _get_key(cls, bucket, obj): + """ Get a key from a bucket """ + + key = bucket.get_key(obj) + if not key: + raise glance.store.BackendException("Could not get key: %s" % key) + return key + + @classmethod + def _parse_s3_tokens(cls, parsed_uri): + """ Parse tokens from the parsed_uri """ + return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL) diff --git a/glance/store/swift.py b/glance/store/swift.py index f7ca6fbca4..b0e4b0dd0a 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -15,7 +15,9 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import import glance.store +import swift.common.client class SwiftBackend(glance.store.Backend): @@ -113,21 +115,6 @@ def _parse_swift_tokens(cls, parsed_uri): def get_connection_class(conn_class): - if conn_class: - pass # Use the provided conn_class - else: - # NOTE(sirp): A standard import statement won't work here because - # this file ('swift.py') is shadowing the swift module, and since - # the import statement searches locally before globally, we'd end - # up importing ourselves. - # - # NOTE(jaypipes): This can be resolved by putting this code in - # /glance/store/swift/__init__.py - # - # see http://docs.python.org/library/functions.html#__import__ - PERFORM_ABSOLUTE_IMPORTS = 0 - swift = __import__('swift.common.client', globals(), locals(), [], - PERFORM_ABSOLUTE_IMPORTS) - + if not conn_class: conn_class = swift.common.client.Connection return conn_class diff --git a/tests/stubs.py b/tests/stubs.py index 2f357c92cd..4155ad33ee 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -105,6 +105,44 @@ def stub_out_filesystem_backend(): f.close() +def stub_out_s3_backend(stubs): + """ Stubs out the S3 Backend with fake data and calls. + + The stubbed swift backend provides back an iterator over + the data "" + + :param stubs: Set of stubout stubs + + """ + + class FakeSwiftAuth(object): + pass + class FakeS3Connection(object): + pass + + class FakeS3Backend(object): + CHUNK_SIZE = 2 + DATA = 'I am a teapot, short and stout\n' + + @classmethod + def get(cls, parsed_uri, expected_size, conn_class=None): + S3Backend = glance.store.s3.S3Backend + + # raise BackendException if URI is bad. + (user, key, authurl, container, obj) = \ + S3Backend._parse_s3_tokens(parsed_uri) + + def chunk_it(): + for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE): + yield cls.DATA[i:i+cls.CHUNK_SIZE] + + return chunk_it() + + fake_swift_backend = FakeS3Backend() + stubs.Set(glance.store.s3.S3Backend, 'get', + fake_swift_backend.get) + + def stub_out_swift_backend(stubs): """Stubs out the Swift Glance backend with fake data and calls. diff --git a/tests/unit/test_stores.py b/tests/unit/test_stores.py index adb9e491be..5c6c4db002 100644 --- a/tests/unit/test_stores.py +++ b/tests/unit/test_stores.py @@ -21,6 +21,7 @@ import unittest import urlparse +from glance.store.s3 import S3Backend from glance.store.swift import SwiftBackend from glance.store import Backend, BackendException, get_from_backend from tests import stubs @@ -85,6 +86,25 @@ def test_https_get(self): self.assertEqual(chunks, expected_returns) +class TestS3Backend(TestBackend): + def setUp(self): + super(TestS3Backend, self).setUp() + stubs.stub_out_s3_backend(self.stubs) + + def test_get(self): + s3_uri = "s3://user:password@localhost/bucket1/file.tar.gz" + + expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', + 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] + fetcher = get_from_backend(s3_uri, + expected_size=8, + conn_class=S3Backend) + + chunks = [c for c in fetcher] + self.assertEqual(chunks, expected_returns) + + + class TestSwiftBackend(TestBackend): def setUp(self): From 71650ebbbaaf50713542f8acc4a33076ae730933 Mon Sep 17 00:00:00 2001 From: Christopher MacGown Date: Wed, 5 Jan 2011 10:56:58 +0100 Subject: [PATCH 2/3] Fixed pylint/pep8 for glance.store.s3 --- glance/store/s3.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/glance/store/s3.py b/glance/store/s3.py index f9c134a14c..9ba34d5e52 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -30,13 +30,13 @@ class S3Backend(glance.store.Backend): @classmethod def get(cls, parsed_uri, expected_size, conn_class=None): """ - Takes a parsed_uri in the format of: - s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3 - and downloads the file. Returns the generator resp_body provided by - get_object. + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects + to s3 and downloads the file. Returns the generator resp_body provided + by get_object. """ - if conn_class: + if conn_class: pass else: conn_class = boto.s3.connection.S3Connection @@ -51,8 +51,9 @@ def get(cls, parsed_uri, expected_size, conn_class=None): # Close the key when we're through. with cls._get_key(bucket, obj) as key: if not key.size == expected_size: - raise glance.store.BackendException("Expected %s bytes, got %s" - % (expected_size, key.size)) + raise glance.store.BackendException( + "Expected %s bytes, got %s" % + (expected_size, key.size)) key.BufferSize = cls.CHUNKSIZE for chunk in key: @@ -61,12 +62,13 @@ def get(cls, parsed_uri, expected_size, conn_class=None): @classmethod def delete(cls, parsed_uri, conn_class=None): """ - Takes a parsed_uri in the format of: - s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects to s3 - and deletes the file. Returns whatever boto.s3.key.Key.delete() returns. + Takes a parsed_uri in the format of: + s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects + to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete() + returns. """ - if conn_class: + if conn_class: pass else: conn_class = boto.s3.connection.S3Connection @@ -82,15 +84,13 @@ def delete(cls, parsed_uri, conn_class=None): with cls._get_key(bucket, obj) as key: return key.delete() - - @classmethod def _get_bucket(cls, conn, bucket_id): """ Get a bucket from an s3 connection """ bucket = conn.get_bucket(bucket_id) if not bucket: - raise glance.store.BackendException("Could not find bucket: %s" % + raise glance.store.BackendException("Could not find bucket: %s" % bucket_id) return bucket From 14ca119e295e7f29f2b24d727fdcc09cb4572cb4 Mon Sep 17 00:00:00 2001 From: Christopher MacGown Date: Wed, 12 Jan 2011 22:03:54 +0100 Subject: [PATCH 3/3] Fixes suggested by JayPipes review. Did not modify docstrings in non-related files. --- glance/store/s3.py | 10 +++++----- glance/store/swift.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/glance/store/s3.py b/glance/store/s3.py index 9ba34d5e52..f23bcb1bf6 100644 --- a/glance/store/s3.py +++ b/glance/store/s3.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" the s3 backend adapter """ +"""the s3 backend adapter""" from __future__ import absolute_import import glance.store @@ -23,7 +23,7 @@ class S3Backend(glance.store.Backend): - """ An implementation of the s3 adapter. """ + """An implementation of the s3 adapter.""" EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0" @@ -86,7 +86,7 @@ def delete(cls, parsed_uri, conn_class=None): @classmethod def _get_bucket(cls, conn, bucket_id): - """ Get a bucket from an s3 connection """ + """Get a bucket from an s3 connection""" bucket = conn.get_bucket(bucket_id) if not bucket: @@ -97,7 +97,7 @@ def _get_bucket(cls, conn, bucket_id): @classmethod def _get_key(cls, bucket, obj): - """ Get a key from a bucket """ + """Get a key from a bucket""" key = bucket.get_key(obj) if not key: @@ -106,5 +106,5 @@ def _get_key(cls, bucket, obj): @classmethod def _parse_s3_tokens(cls, parsed_uri): - """ Parse tokens from the parsed_uri """ + """Parse tokens from the parsed_uri""" return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL) diff --git a/glance/store/swift.py b/glance/store/swift.py index f7c342cc15..68655e78bc 100644 --- a/glance/store/swift.py +++ b/glance/store/swift.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import glance.store -import swift.common.client class SwiftBackend(glance.store.Backend): @@ -117,5 +116,6 @@ def _parse_swift_tokens(cls, parsed_uri): def get_connection_class(conn_class): if not conn_class: + import swift.common.client conn_class = swift.common.client.Connection return conn_class