make compatible with heroku hosting. (#1389)

* initial herokuification

* create custom_storage for manifested pipeline files

Note:
  - @import lines removed from css files cause it was confusing the manifest mixin
  - missing image restored from best guess of what should have been there

* credit where due!

* change elasticsearch providers

* fixup elasticsearch

* move new media storage to it's own prefix

* add migration view for old media files

* update generate_pep_pages to consume results of python/peps#898

* fix admin inline for files/images on pages

* support local and non-local image storage

* make PEP_ARTIFACT_URL configurable via env var for heroku

* herokuify: last fiew page/image fixups

* Jobs rss feed Markup fields should be rendered

Procfile

@@ -0,0 +1,2 @@
+release: python manage.py migrate --noinput
+web: gunicorn pydotorg.wsgi

base-requirements.txt

@@ -16,9 +16,8 @@ icalendar==3.8.4
 chardet2==2.0.3
 # TODO: We may drop 'django-imagekit' completely.
 django-imagekit==4.0.2
-django-haystack==2.8.1
-elasticsearch==1.3.0
-pyelasticsearch==0.6.1
+git+https://github.com/django-haystack/django-haystack.git@802b0f6f4b3b99314453261876a32bac2bbec94f
+elasticsearch>=5,<6
 # TODO: 0.14.0 only supports Django 1.8 and 1.11.
 django-tastypie==0.14.1
 

bin/pre_compile

@@ -0,0 +1 @@
+npm -g install yuglify

custom_storages.py

@@ -0,0 +1,13 @@
+from django.conf import settings
+from django.contrib.staticfiles.storage import ManifestFilesMixin, StaticFilesStorage
+
+from pipeline.storage import PipelineMixin
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class MediaStorage(S3Boto3Storage):
+    location = settings.MEDIAFILES_LOCATION
+
+
+class PipelineManifestStorage(PipelineMixin, ManifestFilesMixin, StaticFilesStorage):
+    pass

dev-requirements.txt

@@ -5,6 +5,7 @@
 factory-boy==2.9.2
 Faker==0.8.1
 tblib==1.3.2
+responses==0.10.5
 
 # Extra stuff required for local dev
 

jobs/feeds.py

@@ -20,6 +20,6 @@ def item_description(self, item):
         """ Description """
         return '\n'.join([
             item.display_location,
-            item.description,
-            item.requirements,
+            item.description.rendered,
+            item.requirements.rendered,
         ])

package.json

@@ -0,0 +1,4 @@
+{
+  "name": "pythondotorg",
+  "description": "Django App behind python.org"
+}

pages/admin.py

@@ -12,21 +12,8 @@
 class PageAdminImageFileWidget(admin.widgets.AdminFileWidget):
 
     def render(self, name, value, attrs=None):
-        """ Fix admin rendering """
         content = super().render(name, value, attrs=None)
-        soup = BeautifulSoup(content, 'lxml')
-
-        # Show useful link/relationship in admin
-        a_href = soup.find('a')
-        if a_href and a_href.attrs['href']:
-            a_href.attrs['href'] = a_href.attrs['href'].replace(settings.MEDIA_ROOT, settings.MEDIA_URL)
-            a_href.string = a_href.text.replace(settings.MEDIA_ROOT, settings.MEDIA_URL)
-
-            if '//' in a_href.attrs['href']:
-                a_href.attrs['href'] = a_href.attrs['href'].replace('//', '/')
-                a_href.string = a_href.text.replace('//', '/')
-
-        return mark_safe(soup)
+        return content
 
 
 class ImageInlineAdmin(admin.StackedInline):

pages/models.py

@@ -90,7 +90,7 @@ def purge_fastly_cache(sender, instance, **kwargs):
 
 
 def page_image_path(instance, filename):
-    return os.path.join(settings.MEDIA_ROOT, instance.page.path, filename)
+    return os.path.join(instance.page.path, filename)
 
 
 class Image(models.Model):

peps/converters.py

@@ -1,4 +1,5 @@
 import functools
+import datetime
 import re
 import os
 
@@ -7,47 +8,48 @@
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files import File
+from django.db.models import Max
 
 from pages.models import Page, Image
 
 PEP_TEMPLATE = 'pages/pep-page.html'
 pep_url = lambda num: 'dev/peps/pep-{}/'.format(num)
 
 
-def check_paths(func):
-    """Ensure that our PEP_REPO_PATH is setup correctly."""
-    @functools.wraps(func)
-    def wrapped(*args, **kwargs):
-        if not hasattr(settings, 'PEP_REPO_PATH'):
-            raise ImproperlyConfigured('No PEP_REPO_PATH in settings')
-        if not os.path.exists(settings.PEP_REPO_PATH):
-            raise ImproperlyConfigured('Path set as PEP_REPO_PATH does not exist')
-        return func(*args, **kwargs)
-    return wrapped
+def get_peps_last_updated():
+    last_update = Page.objects.filter(
+        path__startswith='dev/peps',
+    ).aggregate(Max('updated')).get('updated__max')
+    if last_update is None:
+        return datetime.datetime(
+            1970, 1, 1, tzinfo=datetime.timezone(
+                datetime.timedelta(0)
+            )
+        )
+    return last_update
 
 
-@check_paths
-def convert_pep0():
+def convert_pep0(artifact_path):
     """
     Take existing generated pep-0000.html and convert to something suitable
     for a Python.org Page returns the core body HTML necessary only
     """
-    pep0_path = os.path.join(settings.PEP_REPO_PATH, 'pep-0000.html')
+    pep0_path = os.path.join(artifact_path, 'pep-0000.html')
     pep0_content = open(pep0_path).read()
     data = convert_pep_page(0, pep0_content)
     if data is None:
         return
     return data['content']
 
 
-def get_pep0_page(commit=True):
+def get_pep0_page(artifact_path, commit=True):
     """
     Using convert_pep0 above, create a CMS ready pep0 page and return it
 
     pep0 is used as the directory index, but it's also an actual pep, so we
     return both Page objects.
     """
-    pep0_content = convert_pep0()
+    pep0_content = convert_pep0(artifact_path)
     if pep0_content is None:
         return None, None
     pep0_page, _ = Page.objects.get_or_create(path='dev/peps/')
@@ -88,7 +90,6 @@ def fix_headers(soup, data):
     return soup, data
 
 
-@check_paths
 def convert_pep_page(pep_number, content):
     """
     Handle different formats that pep2html.py outputs
@@ -163,12 +164,12 @@ def convert_pep_page(pep_number, content):
     return data
 
 
-def get_pep_page(pep_number, commit=True):
+def get_pep_page(artifact_path, pep_number, commit=True):
     """
     Given a pep_number retrieve original PEP source text, rst, or html.
     Get or create the associated Page and return it
     """
-    pep_path = os.path.join(settings.PEP_REPO_PATH, 'pep-{}.html'.format(pep_number))
+    pep_path = os.path.join(artifact_path, 'pep-{}.html'.format(pep_number))
     if not os.path.exists(pep_path):
         print("PEP Path '{}' does not exist, skipping".format(pep_path))
         return
@@ -177,7 +178,7 @@ def get_pep_page(pep_number, commit=True):
     if pep_content is None:
         return None
     pep_rst_source = os.path.join(
-        settings.PEP_REPO_PATH, 'pep-{}.rst'.format(pep_number),
+        artifact_path, 'pep-{}.rst'.format(pep_number),
     )
     pep_ext = '.rst' if os.path.exists(pep_rst_source) else '.txt'
     source_link = 'https://github.com/python/peps/blob/master/pep-{}{}'.format(
@@ -198,8 +199,8 @@ def get_pep_page(pep_number, commit=True):
     return pep_page
 
 
-def add_pep_image(pep_number, path):
-    image_path = os.path.join(settings.PEP_REPO_PATH, path)
+def add_pep_image(artifact_path, pep_number, path):
+    image_path = os.path.join(artifact_path, path)
     if not os.path.exists(image_path):
         print("Image Path '{}' does not exist, skipping".format(image_path))
         return
@@ -213,26 +214,15 @@ def add_pep_image(pep_number, path):
     # Find existing images, we have to loop here as we can't use the ORM
     # to query against image__path
     existing_images = Image.objects.filter(page=page)
-    MISSING = False
-    FOUND = False
 
+    FOUND = False
     for image in existing_images:
-        image_root_path = os.path.join(settings.MEDIA_ROOT, page.path, path)
-
-        if image.image.path.endswith(path):
+        if image.image.name.endswith(path):
             FOUND = True
-            # File is missing on disk, recreate
-            if not os.path.exists(image_root_path):
-                MISSING = image
-
             break
 
-    if not FOUND or MISSING:
-        image = None
-        if MISSING:
-            image = MISSING
-        else:
-            image = Image(page=page)
+    if not FOUND:
+        image = Image(page=page)
 
         with open(image_path, 'rb') as image_obj:
             image.image.save(path, File(image_obj))
@@ -243,17 +233,16 @@ def add_pep_image(pep_number, path):
     soup = BeautifulSoup(page.content.raw, 'lxml')
     for img_tag in soup.findAll('img'):
         if img_tag['src'] == path:
-            img_tag['src'] = os.path.join(settings.MEDIA_URL, page.path, path)
+            img_tag['src'] = image.image.url
 
     page.content.raw = str(soup)
     page.save()
 
     return image
 
 
-@check_paths
-def get_peps_rss():
-    rss_feed = os.path.join(settings.PEP_REPO_PATH, 'peps.rss')
+def get_peps_rss(artifact_path):
+    rss_feed = os.path.join(artifact_path, 'peps.rss')
     if not os.path.exists(rss_feed):
         return