Merge pull request #214 from compute-tooling/crit-update

Make updates for new Compute Studio criteria and for use of Google Cloud storage

Dockerfile.base

@@ -11,7 +11,7 @@ ADD ./pytest.ini /code/pytest.ini
 
 # Install dependencies
 RUN conda update conda
-RUN conda install -c conda-forge "python>=3.7.0" pip "paramtools>=0.5.4" "bokeh==1.2.0" --yes
+RUN conda install -c conda-forge "python>=3.7.0" pip "paramtools>=0.5.4" "bokeh==1.2.0" gcsfs --yes
 RUN pip install -r requirements.txt
 
 # Add our code

distributed/api/celery_app/__init__.py

@@ -8,12 +8,20 @@
 from celery.signals import task_postrun
 from celery.result import AsyncResult
 
-import s3like
+import cs_storage
+
+
+try:
+    from cs_config import functions
+except ImportError as ie:
+    if os.environ.get("IS_FLASK", "False") == "True":
+        functions = None
+    else:
+        raise ie
 
 
 COMP_URL = os.environ.get("COMP_URL")
-COMP_API_USER = os.environ.get("COMP_API_USER")
-COMP_API_USER_PASS = os.environ.get("COMP_API_USER_PASS")
+COMP_API_TOKEN = os.environ.get("COMP_API_TOKEN")
 
 CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379")
 CELERY_RESULT_BACKEND = os.environ.get(
@@ -79,12 +87,12 @@ def f(*args, **kwargs):
                     res["model_version"] = "NA"
                     res.update(dict(outputs, **{"version": version}))
                 else:
-                    res["model_version"] = outputs.pop("model_version")
-                    outputs = s3like.write_to_s3like(task_id, outputs)
+                    res["model_version"] = functions.get_version()
+                    outputs = cs_storage.write(task_id, outputs)
                     res.update({"outputs": outputs, "version": version})
             else:
                 res.update(outputs)
-        except Exception as e:
+        except Exception:
             traceback_str = traceback.format_exc()
         finish = time.time()
         if "meta" not in res:
@@ -112,7 +120,7 @@ def post_results(sender=None, headers=None, body=None, **kwargs):
         resp = requests.put(
             f"{COMP_URL}/outputs/api/",
             json=kwargs["retval"],
-            auth=(COMP_API_USER, COMP_API_USER_PASS),
+            headers={"Authorization": f"Token {COMP_API_TOKEN}"},
         )
         print("resp", resp.status_code)
         if resp.status_code == 400:
@@ -122,7 +130,7 @@ def post_results(sender=None, headers=None, body=None, **kwargs):
         resp = requests.put(
             f"{COMP_URL}/inputs/api/",
             json=kwargs["retval"],
-            auth=(COMP_API_USER, COMP_API_USER_PASS),
+            headers={"Authorization": f"Token {COMP_API_TOKEN}"},
         )
         print("resp", resp.status_code)
         if resp.status_code == 400:

distributed/app-deployment.template.yaml

@@ -28,10 +28,6 @@ spec:
         env:
         - name: OUTPUTS_VERSION
           value: v1
-        - name: OBJ_STORAGE_ENDPOINT
-          value: "https://sfo2.digitaloceanspaces.com"
-        - name: OBJ_STORAGE_EDGE
-          value: "https://sfo2.cdn.digitaloceanspaces.com"
         - name: TITLE
           value: "{{TITLE}}"
         - name: OWNER
@@ -45,28 +41,13 @@ spec:
             secretKeyRef:
               name: worker-secret
               key: COMP_URL
-        - name: COMP_API_USER
+        - name: COMP_API_TOKEN
           valueFrom:
             secretKeyRef:
               name: worker-secret
-              key: COMP_API_USER
-        - name: COMP_API_USER_PASS
+              key: COMP_API_TOKEN
+        - name: BUCKET
           valueFrom:
             secretKeyRef:
               name: worker-secret
-              key: COMP_API_USER_PASS
-        - name: OBJ_STORAGE_BUCKET
-          valueFrom:
-            secretKeyRef:
-              name: worker-secret
-              key: OBJ_STORAGE_BUCKET
-        - name: OBJ_STORAGE_ACCESS
-          valueFrom:
-            secretKeyRef:
-              name: worker-secret
-              key: OBJ_STORAGE_ACCESS
-        - name: OBJ_STORAGE_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: worker-secret
-              key: OBJ_STORAGE_SECRET
+              key: BUCKET

distributed/docker-compose.template.yml

@@ -28,17 +28,13 @@ services:
       - AWS_SECRET_ACCESS_KEY={{AWS_SECRET_ACCESS_KEY}}
     depends_on:
       - {{SAFEOWNER}}_{{SAFETITLE}}_tasks
-    env_file:
-      - ./dockerfiles/dev.env
     networks:
       worker_net:
-  
+
   {{SAFEOWNER}}_{{SAFETITLE}}_io:
     image: "{{SAFEOWNER}}_{{SAFETITLE}}_tasks:${TAG}"
     entrypoint: ./celery_io.sh {{OWNER}} {{TITLE}}
     depends_on:
       - {{SAFEOWNER}}_{{SAFETITLE}}_tasks
-    env_file:
-      - ./dockerfiles/dev.env
     networks:
       worker_net:

distributed/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: "3.7"
 services:
   flask:
     build:
@@ -7,16 +7,13 @@ services:
     image: "flask:${TAG}"
     ports:
       - 5050:5050
-    environment:
-      - DEVELOP=${DEVELOP}
-    container_name: flask
     depends_on:
       - redis
       - distributed
     networks:
       worker_net:
 
-####################################################
+  ####################################################
   celerybase:
     build:
       context: ./
@@ -30,7 +27,7 @@ services:
       dockerfile: dockerfiles/Dockerfile
     image: "distributed:latest"
 
-####################################################
+  ####################################################
   redis:
     image: redis
     networks:
@@ -41,4 +38,4 @@ services:
 ####################################################
 
 networks:
-  worker_net:
+  worker_net:

distributed/dockerfiles/Dockerfile

@@ -5,10 +5,11 @@ RUN  apt-get update && apt install libgl1-mesa-glx --yes
 
 RUN mkdir /home/distributed
 RUN mkdir /home/distributed/api
-COPY requirements.txt home/distributed
 
 RUN conda update conda
 RUN conda config --append channels conda-forge
 RUN conda install "python>=3.7" pip
 
+COPY requirements.txt home/distributed
+
 WORKDIR /home/distributed

distributed/dockerfiles/Dockerfile.celerybase

@@ -1,5 +1,5 @@
 ARG TAG
-FROM comporg/distributed
+FROM distributed
 
 ENV CELERY_BROKER_URL redis://redis-master/0
 ENV CELERY_RESULT_BACKEND redis://redis-master/0

distributed/dockerfiles/Dockerfile.flask

@@ -1,5 +1,5 @@
 ARG TAG
-FROM comporg/distributed
+FROM distributed
 
 LABEL build="flask" date="2018-06-13"
 

distributed/dockerfiles/Dockerfile.tasks

@@ -1,5 +1,5 @@
 ARG TAG
-FROM comporg/celerybase
+FROM celerybase
 
 # install packages here
 # install packages necessary for celery,
@@ -17,12 +17,12 @@ ARG BUILD_NUM=0
 
 # Edit Dockerfile here for installing necessary packages, copying files, etc.
 ######################
-ADD ${RAW_REPO_URL}/${BRANCH}/compconfig/install.sh /home
+ADD ${RAW_REPO_URL}/${BRANCH}/cs-config/install.sh /home
 RUN bash /home/install.sh
-RUN pip install "git+${REPO_URL}.git@${BRANCH}#egg=compconfig&subdirectory=compconfig"
-ADD ${RAW_REPO_URL}/${BRANCH}/compconfig/compconfig/tests/test_functions.py /home
-RUN pip install compdevkit>=1.6.0
-RUN py.test /home/test_functions.py -v
+RUN pip install "git+${REPO_URL}.git@${BRANCH}#egg=cs-config&subdirectory=cs-config"
+ADD ${RAW_REPO_URL}/${BRANCH}/cs-config/cs_config/tests/test_functions.py /home
+RUN pip install cs-kit
+RUN py.test /home/test_functions.py -v -s
 ######################
 
 COPY tasks_template.py tasks_template.py

distributed/requirements.txt

@@ -7,4 +7,4 @@ flask
 toolz
 gunicorn
 boto3
-s3like==1.4.0
+cs-storage

distributed/tasks_template.py

@@ -4,10 +4,10 @@
 from api.celery_app import celery_app, task_wrapper
 
 try:
-    import compconfig
+    from cs_config import functions
 except ImportError as ie:
     if os.environ.get("IS_FLASK", "False") == "True":
-        compconfig = None
+        functions = None
     else:
         raise ie
 
@@ -17,25 +17,15 @@
 )
 @task_wrapper
 def inputs_get(self, meta_param_dict):
-    res = compconfig.get_inputs(meta_param_dict)
-    # get ready for upcoming schema change moving from tuples to dicts.
-    if isinstance(res, tuple):
-        res = {"meta_parameters": res[0], "model_parameters": res[1]}
-    return res
+    return functions.get_inputs(meta_param_dict)
 
 
 @celery_app.task(
     name="{{APP_NAME}}.inputs_parse", soft_time_limit=10, bind=True, acks_late=True
 )
 @task_wrapper
 def inputs_parse(self, meta_param_dict, adjustment, errors_warnings):
-    res = compconfig.validate_inputs(meta_param_dict, adjustment, errors_warnings)
-    # get ready for upcoming schema change moving from tuples to dicts.
-    if isinstance(res, tuple):
-        res = {"errors_warnings": res[0], "inputs_file": res[1]}
-    else:
-        res = {"errors_warnings": res}
-    return res
+    return functions.validate_inputs(meta_param_dict, adjustment, errors_warnings)
 
 
 @celery_app.task(
@@ -46,4 +36,4 @@ def inputs_parse(self, meta_param_dict, adjustment, errors_warnings):
 )
 @task_wrapper
 def sim(self, meta_param_dict, adjustment):
-    return compconfig.run_model(meta_param_dict, adjustment)
+    return functions.run_model(meta_param_dict, adjustment)

docker-compose.yml

@@ -1,12 +1,10 @@
-version: '3.7'
+version: "3.7"
 
 services:
   db:
     image: postgres
     volumes:
       - postgres_data:/var/lib/postgresql/data/
-    networks:
-      - distributed_worker_net
   webbase:
     build:
       context: ./
@@ -25,20 +23,13 @@ services:
       - DEBUG=True
       - WORKERS=${WORKERS}:5050
       - LOCAL=True
-      - OBJ_STORAGE_EDGE=https://sfo2.cdn.digitaloceanspaces.com
-      - OBJ_STORAGE_BUCKET=compresults-dev
+      - BUCKET=cs-outputs-dev-1
     ports:
       - "8000:8000"
     container_name: web
     depends_on:
       - db
       - webbase
-    networks:
-      - distributed_worker_net
-
-networks:
-  distributed_worker_net:
-    external: true
 
 volumes:
   postgres_data: