Merge pull request #367 from compute-tooling/dev

Merge dev branch

.github/workflows/dev-webapp-deploy.yml

@@ -45,13 +45,13 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: '12'
-      
-      - name: Install yarn 
-        run: npm install yarn 
-      
+
+      - name: Install yarn
+        run: npm install yarn
+
       - name: Create app bundle
         run: yarn install && yarn build
-      
+
       # Install pip and pytest
       - name: Install dependencies
         run: |
@@ -88,5 +88,5 @@ jobs:
       - name: Deploy
         run: |
           gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-          cs webapp config -o - --update-db | kubectl apply -f -
+          cs webapp config -o - | kubectl apply -f -
           kubectl get pods -o wide

.github/workflows/workers-deploy.yml

@@ -74,5 +74,5 @@ jobs:
       - name: Deploy
         run: |
           gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-          cs workers svc config -o - --update-dns --update-redis | kubectl apply -f -
+          cs workers svc config -o - --update-dns | kubectl apply -f -
           kubectl get pods -o wide

deploy/cs_deploy/webapp.py

@@ -123,7 +123,11 @@ def write_db(self):
                       type: Directory
         """
         db_deployment = self.db_deployment
-        db_config = webapp_config["db"]
+        db_config = {}
+        for var in ["db", "web-db"]:
+            db_config = webapp_config.get(var, {})
+            if db_config:
+                break
         assert (
             db_config.get("provider") == "volume"
         ), f"Got: {db_config.get('provider', None)}"
@@ -167,8 +171,20 @@ def write_web(self, dev=False):
             web_ir[0]["spec"]["routes"][0]["match"] = f"Host(`{self.host}`)"
             web_ir[1]["spec"]["routes"][0]["match"] = f"Host(`{self.host}`)"
 
-        if webapp_config.get("db", {}).get("provider", "") == "gcp-sql-proxy":
-            spec["containers"].append(webapp_config["db"]["args"][0])
+        db_config = {}
+        for var in ["db", "web-db"]:
+            db_config = webapp_config.get(var, {})
+            if db_config:
+                break
+
+        if db_config.get("provider", "") == "gcp-sql-proxy":
+            spec["containers"].append(db_config["args"][0])
+
+        if webapp_config.get("resources"):
+            spec["containers"][0]["resources"] = webapp_config["resources"]
+
+        if webapp_config.get("replicas"):
+            web_obj["spec"]["replicas"] = webapp_config["replicas"]
 
         self.write_config(self.web_serviceaccount, filename="web-serviceaccount.yaml")
         self.write_config(web_obj, filename="web-deployment.yaml")
@@ -193,8 +209,14 @@ def write_deployment_cleanup_job(self, dev=False):
                     "hostPath": {"path": "/code", "type": "Directory",},
                 }
             ]
-        if webapp_config.get("db", {}).get("provider", "") == "gcp-sql-proxy":
-            spec["containers"].append(webapp_config["db"]["args"][0])
+        db_config = {}
+        for var in ["db", "cronjob-db", "web-db"]:
+            db_config = webapp_config.get(var, {})
+            if db_config:
+                break
+
+        if db_config.get("provider", "") == "gcp-sql-proxy":
+            spec["containers"].append(db_config["args"][0])
 
         self.write_config(job_obj, filename="deployment-cleanup-job.yaml")
 
@@ -213,21 +235,21 @@ def write_secret(self):
 
     def write_config(self, config, filename=None):
         if self.kubernetes_target == "-":
-            sys.stdout.write(yaml.dump(config))
+            sys.stdout.write(yaml.dump(config, width=1000))
             sys.stdout.write("---")
             sys.stdout.write("\n")
         else:
             with open(f"{self.kubernetes_target}/{filename}", "w") as f:
-                f.write(yaml.safe_dump(config))
+                f.write(yaml.safe_dump(config, width=1000))
 
     def write_config_all(self, configs, filename=None):
         if self.kubernetes_target == "-":
-            sys.stdout.write(yaml.safe_dump_all(configs))
+            sys.stdout.write(yaml.safe_dump_all(configs, width=1000))
             sys.stdout.write("---")
             sys.stdout.write("\n")
         else:
             with open(f"{self.kubernetes_target}/{filename}", "w") as f:
-                f.write(yaml.safe_dump_all(configs))
+                f.write(yaml.safe_dump_all(configs, width=1000))
 
 
 def manager_from_args(args: argparse.Namespace):

web-kubernetes/deployment-cleanup.template.yaml

@@ -14,7 +14,14 @@ spec:
           containers:
             - name: web-deployment-cleanup
               image: # Uses web image and is filled in with script.
-              command: ["python", "manage.py", "rm_stale_deployments", "--stale-after", "1800"]
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  trap "touch /tmp/pod/main-terminated" EXIT;
+                  python manage.py rm_stale_deployments --stale-after 1800
+              volumeMounts:
+                - mountPath: /tmp/pod
+                  name: tmp-pod
               resources:
                 requests:
                   memory: 0.5G
@@ -38,13 +45,13 @@ spec:
                     secretKeyRef:
                       name: web-db-secret
                       key: username
-                
+
                 - name: DB_PASS
                   valueFrom:
                     secretKeyRef:
                       name: web-db-secret
                       key: password
-                
+
                 - name: DB_NAME
                   valueFrom:
                     secretKeyRef:
@@ -132,3 +139,7 @@ spec:
           nodeSelector:
             component: web
           restartPolicy: Never
+          volumes:
+          - emptyDir: {}
+            name: tmp-pod
+      backoffLimit: 1

webapp/apps/publish/tests/test_views.py

@@ -245,7 +245,7 @@ def test_get_projects(self, api_client):
         resp = api_client.get("/apps/api/v1/")
         assert resp.status_code == 200
         exp = set(proj.title for proj in Project.objects.all())
-        act = set(proj["title"] for proj in resp.data)
+        act = set(proj["title"] for proj in resp.data["results"])
         assert exp == act
 
     def test_get_private_projects(self, api_client, pro_profile):
@@ -259,7 +259,7 @@ def test_get_private_projects(self, api_client, pro_profile):
         resp = api_client.get("/apps/api/v1/")
         assert resp.status_code == 200
         exp = set(proj.title for proj in Project.objects.filter(is_public=True).all())
-        act = set(proj["title"] for proj in resp.data)
+        act = set(proj["title"] for proj in resp.data["results"])
         assert exp == act
 
         # Test private app not included if user doesn't have write access
@@ -268,23 +268,23 @@ def test_get_private_projects(self, api_client, pro_profile):
         resp = api_client.get("/apps/api/v1/")
         assert resp.status_code == 200
         exp = set(proj.title for proj in Project.objects.filter(is_public=True).all())
-        act = set(proj["title"] for proj in resp.data)
+        act = set(proj["title"] for proj in resp.data["results"])
         assert exp == act
 
         # Test private app included if user has read access
         api_client.force_login(project.owner.user)
         resp = api_client.get("/apps/api/v1/")
         assert resp.status_code == 200
         exp = set(proj.title for proj in Project.objects.all())
-        act = set(proj["title"] for proj in resp.data)
+        act = set(proj["title"] for proj in resp.data["results"])
         assert exp == act
 
         api_client.force_login(collab.user)
         project.assign_role("read", collab.user)
         resp = api_client.get("/apps/api/v1/")
         assert resp.status_code == 200
         exp = set(proj.title for proj in Project.objects.all())
-        act = set(proj["title"] for proj in resp.data)
+        act = set(proj["title"] for proj in resp.data["results"])
         assert exp == act
 
     def test_models_api(self, api_client, test_models):

webapp/apps/publish/views.py

@@ -23,7 +23,7 @@
 )
 from rest_framework.exceptions import PermissionDenied as APIPermissionDenied
 from rest_framework import filters
-
+from rest_framework.pagination import PageNumberPagination
 
 # from webapp.settings import DEBUG
 
@@ -191,7 +191,12 @@ def put(self, request, *args, **kwargs):
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
 
-class ProjectAPIView(APIView):
+class ProjectResultsPagination(PageNumberPagination):
+    page_size = 10
+    max_page_size = 10
+
+
+class ProjectAPIView(generics.ListAPIView):
     authentication_classes = (
         SessionAuthentication,
         BasicAuthentication,
@@ -200,13 +205,11 @@ class ProjectAPIView(APIView):
     )
     api_user = User.objects.get(username="comp-api-user")
 
-    def get(self, request, *args, **kwargs):
-        ser = ProjectSerializer(
-            projects_with_access(self.request.user),
-            many=True,
-            context={"request": request},
-        )
-        return Response(ser.data, status=status.HTTP_200_OK)
+    serializer_class = ProjectSerializer
+    pagination_class = ProjectResultsPagination
+
+    def get_queryset(self):
+        return projects_with_access(self.request.user)
 
     def post(self, request, *args, **kwargs):
         if request.user.is_authenticated:

workers/cs_workers/utils.py

@@ -71,9 +71,27 @@ def get_projects(cs_url, api_token=None, auth_headers=None):
         print("Not using auth")
         headers = {}
     print(f"getting data at {cs_url}")
-    resp = httpx.get(f"{cs_url}/apps/api/v1/", headers=headers)
-    assert resp.status_code == 200, f"Got code {resp.status_code}"
-    return hash_projects(resp.json())
+
+    client = httpx.Client(headers=headers, timeout=5)
+    resp = client.get(f"{cs_url}/apps/api/v1/")
+    assert resp.status_code == 200, f"Got {resp.status_code}, {resp.text}"
+    page = resp.json()
+
+    # BC: Not using pagination yet.
+    if isinstance(page, list):
+        return hash_projects(page)
+
+    results = page["results"]
+    next_url = page["next"]
+    while next_url is not None:
+        resp = client.get(next_url)
+        assert resp.status_code == 200, f"Got {resp.status_code}, {resp.text}"
+        page = resp.json()
+
+        results += page["results"]
+        next_url = page["next"]
+
+    return hash_projects(results)
 
 
 def hash_projects(payload):