diff --git a/requirements.txt b/requirements.txt index cfe8a8306..04c4d3dbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -177,3 +177,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +django-storages-redux==1.3.2 \ + --hash=sha256:df94a582d452c8e7763291d5463b60316bf7f6ffe2c59c4c162514c1bbc39504 \ + --hash=sha256:84cb0e685ac0401f14d320d3d469b3e5968a7314a93936c86672c2315d8302dc +boto==2.39.0 \ + --hash=sha256:c73f43558bbc2c4a438c39019b7b4947ba00573ead23420c8614dde0239167ca \ + --hash=sha256:950c5bf36691df916b94ebc5679fed07f642030d39132454ec178800d5b6c58a diff --git a/snippets/base/migrations/0002_auto_20160215_1300.py b/snippets/base/migrations/0002_auto_20160215_1300.py new file mode 100644 index 000000000..46df0c659 --- /dev/null +++ b/snippets/base/migrations/0002_auto_20160215_1300.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import snippets.base.fields +import snippets.base.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='uploadedfile', + name='file', + field=models.FileField(upload_to=snippets.base.models._generate_filename), + ), + ] diff --git a/snippets/base/models.py b/snippets/base/models.py index cff14b135..9418aca08 100644 --- a/snippets/base/models.py +++ b/snippets/base/models.py @@ -15,6 +15,7 @@ from django.core.cache import cache from django.core.exceptions import ValidationError from django.core.files.base import ContentFile +from django.core.files.storage import default_storage from django.core.urlresolvers import reverse from django.db import models from django.db.models.manager import Manager @@ -28,7 +29,6 @@ from snippets.base import ENGLISH_COUNTRIES from snippets.base.fields import CountryField, LocaleField, RegexField from snippets.base.managers import ClientMatchRuleManager, SnippetManager -from snippets.base.storage import OverwriteStorage from snippets.base.util import hashfile @@ -126,8 +126,6 @@ class SnippetBundle(object): """ def __init__(self, client): self.client = client - self.storage = OverwriteStorage() - self._snippets = None @property @@ -164,11 +162,11 @@ def expired(self): @property def filename(self): - return u'bundles/bundle_{0}.jinja'.format(self.key) + return u'bundles/bundle_{0}.html'.format(self.key) @property def url(self): - bundle_url = self.storage.url(self.filename) + bundle_url = default_storage.url(self.filename) site_url = getattr(settings, 'CDN_URL', settings.SITE_URL) full_url = urljoin(site_url, bundle_url) return full_url @@ -198,7 +196,7 @@ def generate(self): if isinstance(bundle_content, unicode): bundle_content = bundle_content.encode('utf-8') - self.storage.save(self.filename, ContentFile(bundle_content)) + default_storage.save(self.filename, ContentFile(bundle_content)) cache.set(self.cache_key, True, settings.SNIPPET_BUNDLE_TIMEOUT) @@ -531,7 +529,7 @@ def _generate_filename(instance, filename): class UploadedFile(models.Model): FILES_ROOT = 'files' # Directory name inside MEDIA_ROOT - file = models.FileField(storage=OverwriteStorage(), upload_to=_generate_filename) + file = models.FileField(upload_to=_generate_filename) name = models.CharField(max_length=255) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) diff --git a/snippets/base/storage.py b/snippets/base/storage.py index b027b9aea..c80715381 100644 --- a/snippets/base/storage.py +++ b/snippets/base/storage.py @@ -1,5 +1,13 @@ +import mimetypes +from datetime import datetime + +from django.conf import settings from django.core.files.storage import FileSystemStorage +from boto.utils import ISO8601 +from storages.compat import deconstructible +from storages.backends.s3boto import S3BotoStorage + class OverwriteStorage(FileSystemStorage): @@ -7,3 +15,39 @@ def get_available_name(self, name): if self.exists(name): self.delete(name) return name + + +@deconstructible +class S3Storage(S3BotoStorage): + cache_control_headers = getattr(settings, 'AWS_CACHE_CONTROL_HEADERS', {}) + + def _save(self, name, content): + cleaned_name = self._clean_name(name) + name = self._normalize_name(cleaned_name) + headers = self.headers.copy() + for filename_start, value in self.cache_control_headers.iteritems(): + if name.startswith(filename_start): + headers['Cache-Control'] = value + + content_type = getattr(content, 'content_type', + mimetypes.guess_type(name)[0] or self.key_class.DefaultContentType) + + # setting the content_type in the key object is not enough. + headers.update({'Content-Type': content_type}) + + if self.gzip and content_type in self.gzip_content_types: + content = self._compress_content(content) + headers.update({'Content-Encoding': 'gzip'}) + + content.name = cleaned_name + encoded_name = self._encode_name(name) + key = self.bucket.get_key(encoded_name) + if not key: + key = self.bucket.new_key(encoded_name) + if self.preload_metadata: + self._entries[encoded_name] = key + key.last_modified = datetime.utcnow().strftime(ISO8601) + + key.set_metadata('Content-Type', content_type) + self._save_content(key, content, headers=headers) + return cleaned_name diff --git a/snippets/base/tests/test_models.py b/snippets/base/tests/test_models.py index 9c2a0e1db..cb87cbe70 100644 --- a/snippets/base/tests/test_models.py +++ b/snippets/base/tests/test_models.py @@ -428,9 +428,10 @@ def test_generate(self): with patch('snippets.base.models.cache') as cache: with patch('snippets.base.models.render_to_string') as render_to_string: - with self.settings(SNIPPET_BUNDLE_TIMEOUT=10): - render_to_string.return_value = 'rendered snippet' - bundle.generate() + with patch('snippets.base.models.default_storage') as default_storage: + with self.settings(SNIPPET_BUNDLE_TIMEOUT=10): + render_to_string.return_value = 'rendered snippet' + bundle.generate() render_to_string.assert_called_with('base/fetch_snippets.jinja', { 'snippet_ids': [s.id for s in [self.snippet1, self.snippet2]], @@ -439,9 +440,9 @@ def test_generate(self): 'locale': 'fr', 'settings': settings, }) - bundle.storage.save.assert_called_with(bundle.filename, ANY) + default_storage.save.assert_called_with(bundle.filename, ANY) cache.set.assert_called_with(bundle.cache_key, True, 10) # Check content of saved file. - content_file = bundle.storage.save.call_args[0][1] + content_file = default_storage.save.call_args[0][1] self.assertEqual(content_file.read(), 'rendered snippet') diff --git a/snippets/settings.py b/snippets/settings.py index 2bec14809..bbcbc38a0 100644 --- a/snippets/settings.py +++ b/snippets/settings.py @@ -176,6 +176,21 @@ PROD_DETAILS_STORAGE = config('PROD_DETAILS_STORAGE', default='product_details.storage.PDFileStorage') + SAML_ENABLE = config('SAML_ENABLE', default=False, cast=bool) if SAML_ENABLE: from saml.settings import * # noqa + + +DEFAULT_FILE_STORAGE = config('FILE_STORAGE', 'storages.backends.overwrite.OverwriteStorage') +# Set to 'storages.backends.s3boto.S3BotoStorage' for S3 +if DEFAULT_FILE_STORAGE == 'snippets.base.storage.S3Storage': + AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID') + AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY') + AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME') + # Full list of S3 endpoints http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + AWS_S3_HOST = config('AWS_S3_HOST') + AWS_CACHE_CONTROL_HEADERS = { + 'files/': 'max-age=900', # 15 Minutes + 'bundles/': 'max-age=2592000', # 1 Month + }