Merge pull request #305 from compute-tooling/private-bucket2

Update webapp for private storage bucket

Dockerfile.base

@@ -23,3 +23,6 @@ ADD ./templates /code/templates/
 ADD ./manage.py /code/
 ADD ./static /code/static/
 RUN python manage.py collectstatic --noinput
+
+ADD ./google-creds.json /code/google-creds.json
+ENV GOOGLE_APPLICATION_CREDENTIALS=/code/google-creds.json

docker-compose.yml

@@ -30,7 +30,7 @@ services:
       - DEBUG=True
       - WORKERS=${WORKERS}
       - LOCAL=True
-      - BUCKET=cs-outputs-dev
+      - BUCKET=cs-outputs-dev-private
       - MAILGUN_API_KEY=${MAILGUN_API_KEY}
       - USE_STRIPE=${USE_STRIPE}
       - STRIPE_SECRET=${STRIPE_SECRET}

requirements.txt

@@ -21,4 +21,4 @@ newrelic
 hashids
 django-rest-auth
 django-allauth
-cs-storage>=1.8.1
+cs-storage>=1.11.0

webapp/apps/comp/models.py

@@ -206,6 +206,15 @@ def role(self, user):
 
 
 class SimulationManager(models.Manager):
+    def get_object_from_screenshot(self, output_id, http_404_on_fail=False):
+        query = dict(
+            outputs__outputs__renderable__outputs__contains=[{"id": output_id}],
+        )
+        if http_404_on_fail:
+            return get_object_or_404(Simulation, **query)
+        else:
+            return self.get(**query)
+
     def next_model_pk(self, project):
         curr_max = Simulation.objects.filter(project=project).aggregate(
             models.Max("model_pk")
@@ -685,10 +694,7 @@ def context(self, request=None):
         if self.outputs and self.outputs_version() != "v0":
             output = self.outputs["outputs"]["renderable"]["outputs"][:1]
             if output:
-                output = cs_storage.add_screenshot_links(
-                    {"renderable": {"outputs": output}}
-                )
-                pic = output["renderable"]["outputs"][0]["screenshot"]
+                pic = f"https://{request.get_host()}/data/{output[0]['id']}.png"
 
         return {"owner": self.get_owner(), "title": self.title, "url": url, "pic": pic}
 

webapp/apps/comp/tests/test_asyncviews.py

@@ -315,6 +315,7 @@ def set_sim_description(self, model_pk: int):
 
     def get_paths(self, model_pk: int):
         def fetch_sims(exp_resp: int):
+            output_id = self.sim.outputs["outputs"]["renderable"]["outputs"][0]["id"]
             api_paths = [
                 f"/{self.owner}/{self.title}/api/v1/{model_pk}/remote/",
                 f"/{self.owner}/{self.title}/api/v1/{model_pk}/",
@@ -323,6 +324,7 @@ def fetch_sims(exp_resp: int):
             paths = [
                 f"/{self.owner}/{self.title}/{model_pk}/",
                 f"/{self.owner}/{self.title}/{model_pk}/edit/",
+                f"/storage/screenshots/{output_id}.png",
             ]
             for path in api_paths:
                 resp = self.api_client.get(path)

webapp/apps/comp/tests/test_models.py

@@ -7,6 +7,7 @@
 from django.http import Http404
 from django.contrib import auth
 from django.forms.models import model_to_dict
+from django.http.response import Http404
 from guardian.shortcuts import get_perms
 
 from webapp.apps.users.models import Project, Profile, create_profile_from_user
@@ -213,6 +214,26 @@ def test_get_owner(db, get_inputs, meta_param_dict):
     assert sim.get_owner() == "unsigned"
 
 
+def test_sim_screenshot_query(db, get_inputs, meta_param_dict):
+    modeler = User.objects.get(username="modeler").profile
+    inputs = _submit_inputs("Used-for-testing", get_inputs, meta_param_dict, modeler)
+
+    _, submit_sim = _submit_sim(inputs)
+    sim = submit_sim.submit()
+    sim.status = "SUCCESS"
+    sim.outputs = json.loads(read_outputs("Matchups_v1"))
+    sim.save()
+
+    for output in sim.outputs["outputs"]["renderable"]["outputs"]:
+        assert sim == Simulation.objects.get_object_from_screenshot(output["id"])
+
+    with pytest.raises(Simulation.DoesNotExist):
+        Simulation.objects.get_object_from_screenshot("abc123")
+
+    with pytest.raises(Http404):
+        Simulation.objects.get_object_from_screenshot("abc123", http_404_on_fail=True)
+
+
 def test_sim_permissions(db, get_inputs, meta_param_dict, plus_profile):
     collab = next(gen_collabs(1))
     inputs = _submit_inputs(

webapp/apps/comp/views/__init__.py

@@ -6,6 +6,7 @@
     OutputsView,
     PermissionPendingView,
     PermissionGrantedView,
+    DataView,
 )
 
 from .api import (

webapp/apps/comp/views/api.py

@@ -242,9 +242,15 @@ def get_sim_data(self, user, as_remote, username, title, model_pk):
             if not as_remote:
                 data["outputs"] = cs_storage.read(outputs)
             else:
-                data["outputs"]["outputs"] = cs_storage.add_screenshot_links(
-                    data["outputs"]["outputs"]
-                )
+                if self.request.is_secure():
+                    protocol = "https"
+                else:
+                    protocol = "http"  # local dev.
+                base_url = f"{protocol}://{self.request.get_host()}"
+                for rem_output in data["outputs"]["outputs"]["renderable"]["outputs"]:
+                    rem_output[
+                        "screenshot"
+                    ] = f"{base_url}/storage/screenshots/{rem_output['id']}.png"
             return Response(data, status=status.HTTP_200_OK)
         elif self.object.traceback is not None:
             return Response(data, status=status.HTTP_200_OK)

webapp/apps/comp/views/views.py

@@ -289,3 +289,22 @@ def render_json(self):
             "Content-Disposition"
         ] = f"attachment; filename={self.object.json_filename()}"
         return resp
+
+
+class DataView(View):
+    def get(self, request, *args, **kwargs):
+        data_id = kwargs["data_id"]
+
+        if data_id.endswith(".png"):
+            data_id = data_id[:-4]
+
+        self.object = Simulation.objects.get_object_from_screenshot(
+            data_id, http_404_on_fail=True
+        )
+
+        if not self.object.has_read_access(request.user):
+            raise PermissionDenied()
+
+        pic = cs_storage.read_screenshot(kwargs["data_id"])
+
+        return HttpResponse(pic, content_type="image/png")

webapp/urls.py

@@ -32,6 +32,11 @@
     path("users/", include("webapp.apps.users.urls")),
     path("users/", include("django.contrib.auth.urls")),
     path("billing/", include("webapp.apps.billing.urls")),
+    path(
+        "storage/screenshots/<str:data_id>",
+        compviews.DataView.as_view(),
+        name="data__view",
+    ),
     path("outputs/api/", compviews.OutputsAPIView.as_view(), name="outputs_api"),
     path("inputs/api/", compviews.MyInputsAPIView.as_view(), name="myinputs_api"),
     url(r"^rest-auth/", include("rest_auth.urls")),

workers/requirements.txt

@@ -4,5 +4,5 @@ pytest
 toolz
 boto3
 kubernetes
-cs-storage>=1.10.1
+cs-storage>=1.11.0
 pyyaml

workers/setup.py

@@ -28,7 +28,7 @@
         "dask",
         "distributed",
         "tornado",
-        "cs-storage",
+        "cs-storage>=1.11.0",
         "docker",
     ],
     include_package_data=True,