Skip to content

Commit

Permalink
Merge "Fixing bug 794582 - Now able to stream http(s) images"
Browse files Browse the repository at this point in the history
  • Loading branch information
Jenkins authored and openstack-gerrit committed Sep 19, 2011
2 parents e7ec4a1 + ef19685 commit ec4af4b
Show file tree
Hide file tree
Showing 10 changed files with 56 additions and 35 deletions.
8 changes: 5 additions & 3 deletions glance/api/v1/images.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,15 @@ def show(self, req, id):
"""
image = self.get_active_image_meta_or_404(req, id)

def get_from_store(image):
def get_from_store(image_meta):
"""Called if caching disabled"""
try:
image = get_from_backend(image['location'])
location = image_meta['location']
image_data, image_size = get_from_backend(location)
image_meta["size"] = image_size or image_meta["size"]
except exception.NotFound, e:
raise HTTPNotFound(explanation="%s" % e)
return image
return image_data

def get_from_cache(image, cache):
"""Called if cache hit"""
Expand Down
4 changes: 2 additions & 2 deletions glance/store/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ def configure(self):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
Expand Down
6 changes: 3 additions & 3 deletions glance/store/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ def _option_get(self, param):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
Expand All @@ -140,7 +140,7 @@ def get(self, location):
else:
msg = _("Found image at %s. Returning in ChunkedFile.") % filepath
logger.debug(msg)
return ChunkedFile(filepath)
return (ChunkedFile(filepath), None)

def delete(self, location):
"""
Expand Down
15 changes: 9 additions & 6 deletions glance/store/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,14 @@ def parse_uri(self, uri):
self.path = path


def http_response_iterator(conn, size):
def http_response_iterator(conn, response, size):
"""
Return an iterator for a file-like object.
:param conn: HTTP(S) Connection
:param response: httplib.HTTPResponse object
:param size: Chunk size to iterate with
"""
response = conn.getresponse()
chunk = response.read(size)
while chunk:
yield chunk
Expand All @@ -107,20 +107,23 @@ class Store(glance.store.base.Store):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
:raises `glance.exception.NotFound` if image does not exist
"""
loc = location.store_location

conn_class = self._get_conn_class(loc)
conn = conn_class(loc.netloc)
conn.request("GET", loc.path, "", {})
resp = conn.getresponse()

return http_response_iterator(conn, self.CHUNKSIZE)
content_length = resp.getheader('content-length', 0)
iterator = http_response_iterator(conn, resp, self.CHUNKSIZE)

return (iterator, content_length)

def _get_conn_class(self, loc):
"""
Expand Down
6 changes: 3 additions & 3 deletions glance/store/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,8 @@ def _option_get(self, param):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
Expand Down Expand Up @@ -253,7 +253,7 @@ def get(self, location):
# raise glance.store.BackendException(msg)

key.BufferSize = self.CHUNKSIZE
return ChunkedFile(key)
return (ChunkedFile(key), None)

def add(self, image_id, image_file, image_size):
"""
Expand Down
6 changes: 3 additions & 3 deletions glance/store/swift.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,8 @@ def configure(self):
def get(self, location):
"""
Takes a `glance.store.location.Location` object that indicates
where to find the image file, and returns a generator for reading
the image file
where to find the image file, and returns a tuple of generator
(for reading the image file) and image_size
:param location `glance.store.location.Location` object, supplied
from glance.store.location.get_location_from_uri()
Expand Down Expand Up @@ -260,7 +260,7 @@ def get(self, location):
# "Expected %s byte file, Swift has %s bytes" %
# (expected_size, obj_size))

return resp_body
return (resp_body, None)

def _make_swift_connection(self, auth_url, user, key):
"""
Expand Down
4 changes: 2 additions & 2 deletions glance/tests/unit/test_filesystem_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def tearDown(self):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri("file:///tmp/glance-tests/2")
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)

expected_data = "chunk00000remainder"
expected_num_chunks = 2
Expand Down Expand Up @@ -95,7 +95,7 @@ def test_add(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri("file:///tmp/glance-tests/42")
new_image_file = self.store.get(loc)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0

Expand Down
20 changes: 16 additions & 4 deletions glance/tests/unit/test_http_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,25 @@ def stub_out_http_backend(stubs):
:param stubs: Set of stubout stubs
"""

class FakeHTTPConnection(object):
class FakeHTTPResponse(object):

DATA = 'I am a teapot, short and stout\n'
HEADERS = {'content-length': 31}

def __init__(self, *args, **kwargs):
self.data = StringIO.StringIO(self.DATA)
self.read = self.data.read

def getheader(self, name, default=None):
return self.HEADERS.get(name.lower(), default)

class FakeHTTPConnection(object):

def __init__(self, *args, **kwargs):
pass

def getresponse(self):
return StringIO.StringIO(self.DATA)
return FakeHTTPResponse()

def request(self, *_args, **_kwargs):
pass
Expand All @@ -72,7 +82,8 @@ def test_http_get(self):
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)

chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
Expand All @@ -82,7 +93,8 @@ def test_https_get(self):
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
image_file = self.store.get(loc)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)

chunks = [c for c in image_file]
self.assertEqual(chunks, expected_returns)
Expand Down
8 changes: 5 additions & 3 deletions glance/tests/unit/test_s3_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,9 @@ def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri(
"s3://user:key@auth_address/glance/2")
image_s3 = self.store.get(loc)
(image_s3, image_size) = self.store.get(loc)

self.assertEqual(image_size, None)

expected_data = "*" * FIVE_KB
data = ""
Expand Down Expand Up @@ -217,7 +219,7 @@ def test_add(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_s3 = self.store.get(loc)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = StringIO.StringIO()
for chunk in new_image_s3:
new_image_contents.write(chunk)
Expand Down Expand Up @@ -267,7 +269,7 @@ def test_add_host_variations(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_s3 = self.store.get(loc)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = new_image_s3.getvalue()
new_image_s3_size = new_image_s3.len

Expand Down
14 changes: 8 additions & 6 deletions glance/tests/unit/test_swift_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,8 @@ def tearDown(self):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks"""
loc = get_location_from_uri("swift://user:key@auth_address/glance/2")
image_swift = self.store.get(loc)
(image_swift, image_size) = self.store.get(loc)
self.assertEqual(image_size, None)

expected_data = "*" * FIVE_KB
data = ""
Expand All @@ -214,7 +215,8 @@ def test_get_with_http_auth(self):
"""
loc = get_location_from_uri("swift+http://user:key@auth_address/"
"glance/2")
image_swift = self.store.get(loc)
(image_swift, image_size) = self.store.get(loc)
self.assertEqual(image_size, None)

expected_data = "*" * FIVE_KB
data = ""
Expand Down Expand Up @@ -255,7 +257,7 @@ def test_add(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len

Expand Down Expand Up @@ -303,7 +305,7 @@ def test_add_auth_url_variations(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len

Expand Down Expand Up @@ -363,7 +365,7 @@ def test_add_no_container_and_create(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len

Expand Down Expand Up @@ -408,7 +410,7 @@ def test_add_large_object(self):
self.assertEquals(expected_checksum, checksum)

loc = get_location_from_uri(expected_location)
new_image_swift = self.store.get(loc)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = new_image_swift.getvalue()
new_image_swift_size = new_image_swift.len

Expand Down

0 comments on commit ec4af4b

Please sign in to comment.