Merge pull request #160 from glogiotatidis/s3storage

Support uploading bundles to S3.

requirements.txt

@@ -177,3 +177,9 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
+django-storages-redux==1.3.2 \
+    --hash=sha256:df94a582d452c8e7763291d5463b60316bf7f6ffe2c59c4c162514c1bbc39504 \
+    --hash=sha256:84cb0e685ac0401f14d320d3d469b3e5968a7314a93936c86672c2315d8302dc
+boto==2.39.0 \
+    --hash=sha256:c73f43558bbc2c4a438c39019b7b4947ba00573ead23420c8614dde0239167ca \
+    --hash=sha256:950c5bf36691df916b94ebc5679fed07f642030d39132454ec178800d5b6c58a

snippets/base/migrations/0002_auto_20160215_1300.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import snippets.base.fields
+import snippets.base.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='uploadedfile',
+            name='file',
+            field=models.FileField(upload_to=snippets.base.models._generate_filename),
+        ),
+    ]

snippets/base/models.py

@@ -15,6 +15,7 @@
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
+from django.core.files.storage import default_storage
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.db.models.manager import Manager
@@ -28,7 +29,6 @@
 from snippets.base import ENGLISH_COUNTRIES
 from snippets.base.fields import CountryField, LocaleField, RegexField
 from snippets.base.managers import ClientMatchRuleManager, SnippetManager
-from snippets.base.storage import OverwriteStorage
 from snippets.base.util import hashfile
 
 
@@ -126,8 +126,6 @@ class SnippetBundle(object):
     """
     def __init__(self, client):
         self.client = client
-        self.storage = OverwriteStorage()
-
         self._snippets = None
 
     @property
@@ -164,11 +162,11 @@ def expired(self):
 
     @property
     def filename(self):
-        return u'bundles/bundle_{0}.jinja'.format(self.key)
+        return u'bundles/bundle_{0}.html'.format(self.key)
 
     @property
     def url(self):
-        bundle_url = self.storage.url(self.filename)
+        bundle_url = default_storage.url(self.filename)
         site_url = getattr(settings, 'CDN_URL', settings.SITE_URL)
         full_url = urljoin(site_url, bundle_url)
         return full_url
@@ -198,7 +196,7 @@ def generate(self):
 
         if isinstance(bundle_content, unicode):
             bundle_content = bundle_content.encode('utf-8')
-        self.storage.save(self.filename, ContentFile(bundle_content))
+        default_storage.save(self.filename, ContentFile(bundle_content))
         cache.set(self.cache_key, True, settings.SNIPPET_BUNDLE_TIMEOUT)
 
 
@@ -531,7 +529,7 @@ def _generate_filename(instance, filename):
 class UploadedFile(models.Model):
     FILES_ROOT = 'files'  # Directory name inside MEDIA_ROOT
 
-    file = models.FileField(storage=OverwriteStorage(), upload_to=_generate_filename)
+    file = models.FileField(upload_to=_generate_filename)
     name = models.CharField(max_length=255)
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)

snippets/base/storage.py

@@ -1,9 +1,53 @@
+import mimetypes
+from datetime import datetime
+
+from django.conf import settings
 from django.core.files.storage import FileSystemStorage
 
+from boto.utils import ISO8601
+from storages.compat import deconstructible
+from storages.backends.s3boto import S3BotoStorage
+
 
 class OverwriteStorage(FileSystemStorage):
 
     def get_available_name(self, name):
         if self.exists(name):
             self.delete(name)
         return name
+
+
+@deconstructible
+class S3Storage(S3BotoStorage):
+    cache_control_headers = getattr(settings, 'AWS_CACHE_CONTROL_HEADERS', {})
+
+    def _save(self, name, content):
+        cleaned_name = self._clean_name(name)
+        name = self._normalize_name(cleaned_name)
+        headers = self.headers.copy()
+        for filename_start, value in self.cache_control_headers.iteritems():
+            if name.startswith(filename_start):
+                headers['Cache-Control'] = value
+
+        content_type = getattr(content, 'content_type',
+                               mimetypes.guess_type(name)[0] or self.key_class.DefaultContentType)
+
+        # setting the content_type in the key object is not enough.
+        headers.update({'Content-Type': content_type})
+
+        if self.gzip and content_type in self.gzip_content_types:
+            content = self._compress_content(content)
+            headers.update({'Content-Encoding': 'gzip'})
+
+        content.name = cleaned_name
+        encoded_name = self._encode_name(name)
+        key = self.bucket.get_key(encoded_name)
+        if not key:
+            key = self.bucket.new_key(encoded_name)
+        if self.preload_metadata:
+            self._entries[encoded_name] = key
+            key.last_modified = datetime.utcnow().strftime(ISO8601)
+
+        key.set_metadata('Content-Type', content_type)
+        self._save_content(key, content, headers=headers)
+        return cleaned_name

snippets/base/tests/test_models.py

@@ -428,9 +428,10 @@ def test_generate(self):
 
         with patch('snippets.base.models.cache') as cache:
             with patch('snippets.base.models.render_to_string') as render_to_string:
-                with self.settings(SNIPPET_BUNDLE_TIMEOUT=10):
-                    render_to_string.return_value = 'rendered snippet'
-                    bundle.generate()
+                with patch('snippets.base.models.default_storage') as default_storage:
+                    with self.settings(SNIPPET_BUNDLE_TIMEOUT=10):
+                        render_to_string.return_value = 'rendered snippet'
+                        bundle.generate()
 
         render_to_string.assert_called_with('base/fetch_snippets.jinja', {
             'snippet_ids': [s.id for s in [self.snippet1, self.snippet2]],
@@ -439,9 +440,9 @@ def test_generate(self):
             'locale': 'fr',
             'settings': settings,
         })
-        bundle.storage.save.assert_called_with(bundle.filename, ANY)
+        default_storage.save.assert_called_with(bundle.filename, ANY)
         cache.set.assert_called_with(bundle.cache_key, True, 10)
 
         # Check content of saved file.
-        content_file = bundle.storage.save.call_args[0][1]
+        content_file = default_storage.save.call_args[0][1]
         self.assertEqual(content_file.read(), 'rendered snippet')

snippets/settings.py

@@ -176,6 +176,21 @@
 PROD_DETAILS_STORAGE = config('PROD_DETAILS_STORAGE',
                               default='product_details.storage.PDFileStorage')
 
+
 SAML_ENABLE = config('SAML_ENABLE', default=False, cast=bool)
 if SAML_ENABLE:
     from saml.settings import *  # noqa
+
+
+DEFAULT_FILE_STORAGE = config('FILE_STORAGE', 'storages.backends.overwrite.OverwriteStorage')
+# Set to 'storages.backends.s3boto.S3BotoStorage' for S3
+if DEFAULT_FILE_STORAGE == 'snippets.base.storage.S3Storage':
+    AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID')
+    AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY')
+    AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME')
+    # Full list of S3 endpoints http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+    AWS_S3_HOST = config('AWS_S3_HOST')
+    AWS_CACHE_CONTROL_HEADERS = {
+        'files/': 'max-age=900',  # 15 Minutes
+        'bundles/': 'max-age=2592000',  # 1 Month
+    }