Skip to content

Commit

Permalink
support versioned open (#39)
Browse files Browse the repository at this point in the history
* support versioned open

* bump version
  • Loading branch information
Chenyang Liu authored Apr 18, 2017
1 parent cfc5273 commit 7a0449f
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 5 deletions.
4 changes: 2 additions & 2 deletions baiji/cached_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ class CachedFile(object):
CachedFile('s3://bucket/path/to/file.ext', 'w') opens a temp file for writing and uploads it on close
CachedFile('s3://bucket/path/to/file.ext', 'x') verifies that the file doesn't exist on s3, then behaves like 'w'
'''
def __init__(self, key, mode='r', connection=None, encrypt=True):
def __init__(self, key, mode='r', connection=None, encrypt=True, version_id=None):
from baiji.connection import S3Connection
self.encrypt = encrypt
self.key = key
Expand Down Expand Up @@ -110,7 +110,7 @@ def __init__(self, key, mode='r', connection=None, encrypt=True):
self.name = self.f.name
self.remotename = key # Used by some serialization code to find files which sit along side the file in question, like textures which sit next to a mesh file
if self.mode.reading:
self.connection.cp(self.key, self.name, force=True)
self.connection.cp(self.key, self.name, force=True, version_id=version_id)
def upload(self):
self.connection.cp(self.name, self.key, encrypt=self.encrypt, force=True)
def __enter__(self):
Expand Down
2 changes: 1 addition & 1 deletion baiji/package_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
#
# See https://www.python.org/dev/peps/pep-0420/#namespace-packages-today

__version__ = '2.7.2'
__version__ = '2.8.0'
6 changes: 4 additions & 2 deletions baiji/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def enable_versioning(*args, **kwargs):
def disable_versioning(*args, **kwargs):
return S3Connection().disable_versioning(*args, **kwargs)

def open(key, mode='rb'): # pylint: disable=redefined-builtin
def open(key, mode='rb', version_id=None): # pylint: disable=redefined-builtin
'''
Acts like open(key, mode), opening a file.
Expand All @@ -121,6 +121,8 @@ def open(key, mode='rb'): # pylint: disable=redefined-builtin
existing file or key, ValueError for an invalid key, and IOError for an
underlying local file-system failure.
version_id: comes handy when opening remote versioned files. no-op otherwise
'''
from baiji.cached_file import CachedFile
return CachedFile(key, mode)
return CachedFile(key, mode, version_id=version_id)
16 changes: 16 additions & 0 deletions baiji/test_s3_cached_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,22 @@ def test_s3_open_read_raises_error_for_nonexistent_remote_file(self):
with self.assertRaises(s3.KeyNotFound):
s3.open(nonexistent_file, 'r')

def test_s3_open_read_versioned_remote_file(self):
remote_file_name = self.existing_versioned_remote_file
version_id = s3.info(remote_file_name)['version_id']

with s3.open(remote_file_name, 'r', version_id=version_id) as f:
tempname = f.name

self.assertFalse(os.path.exists(tempname))

def test_s3_open_read_versioned_remote_file_with_unkown_version_id_raise_key_not_found(self):
remote_file_name = self.existing_versioned_remote_file
unknown_version_id = '5elgojhtA8BGJerqfbciN78eU74SJ9mX'

with self.assertRaises(s3.KeyNotFound):
s3.open(remote_file_name, 'r', version_id=unknown_version_id)

def test_s3_open_write_does_not_raise_error_for_nonexistent_remote_file(self):
nonexistent_file = self.remote_file(str(uuid.uuid4()))
self.assert_s3_does_not_exist(nonexistent_file)
Expand Down

0 comments on commit 7a0449f

Please sign in to comment.