Merge pull request #425 from compute-tooling/dev

C/S 1.9
compute-tooling · May 18, 2021 · 3694640 · 3694640
2 parents 3f4ad1c + 6781037
commit 3694640
Show file tree

Hide file tree

Showing 101 changed files with 3,158 additions and 876 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -19,3 +19,5 @@ coverage.xml
 .git
 redis-data
 db-data
+cluster-db-data
+./workers/cluster-db-data
diff --git a/.github/workflows/dev-workers-deploy.yml b/.github/workflows/dev-workers-deploy.yml
@@ -20,10 +20,10 @@ env:
   REGISTRY_HOSTNAME: gcr.io
 
   PROJECT: ${{ secrets.DEV_GKE_PROJECT }}
-  HOST: dev.compute.studio
   TAG: ${{ github.sha }}
 
   CS_CONFIG: ${{ secrets.DEV_CS_CONFIG }}
+  DEV_WORKERS_VALUES: ${{ secrets.DEV_WORKERS_VALUES }}
 
 jobs:
   setup-build-publish-deploy:
@@ -58,9 +58,10 @@ jobs:
           # Set up docker to authenticate
           gcloud auth configure-docker
 
-      - name: Set cs-config.yaml file.
+      - name: Set cs-config.yaml and values.yaml files.
         run: |
           echo $CS_CONFIG | base64 --decode > cs-config.yaml
+          echo $DEV_WORKERS_VALUES | base64 --decode > ./workers/values.deploy.yaml
 
       - name: Build Docker Images
         run: |
@@ -74,5 +75,12 @@ jobs:
       - name: Deploy
         run: |
           gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-          cs workers svc config -o - --update-dns | kubectl apply -f -
-          kubectl get pods -o wide
+          cd workers
+          helm template cs-workers \
+            --set project=$PROJECT \
+            --set tag=$TAG \
+            --set api.secret_key=$(cs secrets get WORKERS_API_SECRET_KEY) \
+            --set db.password=$(cs secrets get WORKERS_DB_PASSWORD)  \
+            --set redis.password=$(cs secrets get WORKERS_REDIS_PASSWORD) \
+            --namespace workers \
+            -f values.deploy.yaml | kubectl apply -f -
diff --git a/.github/workflows/workers-deploy.yml b/.github/workflows/workers-deploy.yml
@@ -20,10 +20,10 @@ env:
   REGISTRY_HOSTNAME: gcr.io
 
   PROJECT: ${{ secrets.GKE_PROJECT }}
-  HOST: compute.studio
   TAG: ${{ github.sha }}
 
   CS_CONFIG: ${{ secrets.CS_CONFIG }}
+  WORKERS_VALUES: ${{ secrets.DEV_WORKERS_VALUES }}
 
 jobs:
   setup-build-publish-deploy:
@@ -58,9 +58,10 @@ jobs:
           # Set up docker to authenticate
           gcloud auth configure-docker
 
-      - name: Set cs-config.yaml file.
+      - name: Set cs-config.yaml and values.yaml files.
         run: |
           echo $CS_CONFIG | base64 --decode > cs-config.yaml
+          echo $WORKERS_VALUES | base64 --decode > ./workers/values.deploy.yaml
 
       - name: Build Docker Images
         run: |
@@ -74,5 +75,12 @@ jobs:
       - name: Deploy
         run: |
           gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-          cs workers svc config -o - --update-dns | kubectl apply -f -
-          kubectl get pods -o wide
+          cd workers
+          helm template cs-workers \
+            --set project=$PROJECT \
+            --set tag=$TAG \
+            --set api.secret_key=$(cs secrets get WORKERS_API_SECRET_KEY) \
+            --set db.password=$(cs secrets get WORKERS_DB_PASSWORD)  \
+            --set redis.password=$(cs secrets get WORKERS_REDIS_PASSWORD) \
+            --namespace workers \
+            -f values.deploy.yaml | kubectl apply -f -
diff --git a/kind-config.yaml b/kind-config.yaml
@@ -4,8 +4,12 @@ nodes:
   - role: control-plane
   - role: worker
     extraMounts:
-      - hostPath: /home/hankdoupe/compute-studio/redis-data
-        containerPath: /redis-data
+      - hostPath: /home/hankdoupe/compute-studio/redis-queue-data
+        containerPath: /redis-queue-data
+      - hostPath: /home/hankdoupe/compute-studio/workers-db-data
+        containerPath: /workers-db-data
+      - hostPath: /home/hankdoupe/compute-studio
+        containerPath: /code
   - role: worker
   - role: worker
     extraMounts:

diff --git a/pytest.ini b/pytest.ini
@@ -2,3 +2,4 @@
 DJANGO_SETTINGS_MODULE = webapp.settings
 markers =
   register: for the register module.
+  requires_stripe: whether to use stripe or not.
diff --git a/src/Simulation/API.ts b/src/Simulation/API.ts
@@ -43,43 +43,42 @@ export default class API {
     });
   }
 
-  getInputsDetail(): Promise<InputsDetail> {
+  async getInputsDetail(): Promise<InputsDetail> {
     if (!this.modelpk) return;
-    return axios
-      .get(`/${this.owner}/${this.title}/api/v1/${this.modelpk}/edit/`)
-      .then(resp => resp.data);
+    const resp = await axios.get(`/${this.owner}/${this.title}/api/v1/${this.modelpk}/edit/`);
+    return resp.data;
   }
 
-  getInitialValues(): Promise<Inputs> {
-    let data: Inputs;
-    if (!this.modelpk) {
-      return axios.get(`/${this.owner}/${this.title}/api/v1/inputs/`).then(inputsResp => {
-        data = inputsResp.data;
-        return data;
+  async getInputs(meta_parameters?: InputsDetail["meta_parameters"]): Promise<Inputs> {
+    let resp;
+    if (!!meta_parameters) {
+      resp = await axios.post(`/${this.owner}/${this.title}/api/v1/inputs/`, meta_parameters);
+    } else {
+      resp = await axios.get(`/${this.owner}/${this.title}/api/v1/inputs/`);
+    }
+    if (resp.status === 202) {
+      return new Promise(resolve => {
+        setTimeout(async () => resolve(await this.getInputs(meta_parameters)), 2000);
       });
     } else {
-      return axios
-        .get(`/${this.owner}/${this.title}/api/v1/${this.modelpk}/edit/`)
-        .then(detailResp => {
-          return axios
-            .post(`/${this.owner}/${this.title}/api/v1/inputs/`, {
-              meta_parameters: detailResp.data.meta_parameters,
-            })
-            .then(inputsResp => {
-              data = inputsResp.data;
-              data["detail"] = detailResp.data;
-              return data;
-            });
-        });
+      return resp.data;
     }
   }
 
-  resetInitialValues(metaParameters: { [metaParam: string]: any }): Promise<Inputs> {
-    return axios
-      .post(`/${this.owner}/${this.title}/api/v1/inputs/`, metaParameters)
-      .then(response => {
-        return response.data;
+  async resetInitialValues(metaParameters: { [metaParam: string]: any }): Promise<Inputs> {
+    let resp;
+    if (!!metaParameters) {
+      resp = await axios.post(`/${this.owner}/${this.title}/api/v1/inputs/`, metaParameters);
+    } else {
+      resp = await axios.get(`/${this.owner}/${this.title}/api/v1/inputs/`);
+    }
+    if (resp.status === 202) {
+      return new Promise(resolve => {
+        setTimeout(async () => resolve(await this.getInputs(metaParameters)), 2000);
       });
+    } else {
+      return resp.data;
+    }
   }
 
   getAccessStatus(): Promise<AccessStatus> {

diff --git a/src/Simulation/index.tsx b/src/Simulation/index.tsx
@@ -128,79 +128,78 @@ class SimTabs extends React.Component<
     this.handleSubmit = this.handleSubmit.bind(this);
   }
 
-  componentDidMount() {
+  async componentDidMount() {
     this.api.getAccessStatus().then(data => {
       this.setState({
         accessStatus: data,
       });
     });
-    this.api
-      .getInitialValues()
-      .then(data => {
-        const [serverValues, sects, inputs, schema, unknownParams] = convertToFormik(data);
-        let isEmpty = true;
-        for (const msectvals of Object.values(data.detail?.adjustment || {})) {
-          if (Object.keys(msectvals).length > 0) {
-            isEmpty = false;
-          }
-        }
-        let initialValues;
-        if (isEmpty) {
-          const storage = Persist.pop(
-            `${this.props.match.params.owner}/${this.props.match.params.title}/inputs`
-          );
-          // Use values from local storage if available. Default to empty dict from server.
-          initialValues = storage || serverValues;
-        } else {
-          initialValues = serverValues;
-        }
-
-        this.setState({
-          inputs: inputs,
-          initialValues: initialValues,
-          sects: sects,
-          schema: schema,
-          unknownParams: unknownParams,
-          extend: "extend" in data ? data.extend : false,
-        });
-      })
-      .catch(error => {
-        this.setState({ error });
-      });
     if (this.api.modelpk) {
       this.setOutputs();
     }
+    let data: Inputs;
+    if (this.api.modelpk) {
+      const detail = await this.api.getInputsDetail();
+      data = await this.api.getInputs(detail.meta_parameters);
+      data.detail = detail;
+    } else {
+      data = await this.api.getInputs();
+    }
+
+    const [serverValues, sects, inputs, schema, unknownParams] = convertToFormik(data);
+    let isEmpty = true;
+    for (const msectvals of Object.values(data.detail?.adjustment || {})) {
+      if (Object.keys(msectvals).length > 0) {
+        isEmpty = false;
+      }
+    }
+    let initialValues;
+    if (isEmpty) {
+      const storage = Persist.pop(
+        `${this.props.match.params.owner}/${this.props.match.params.title}/inputs`
+      );
+      // Use values from local storage if available. Default to empty dict from server.
+      initialValues = storage || serverValues;
+    } else {
+      initialValues = serverValues;
+    }
+
+    this.setState({
+      inputs: inputs,
+      initialValues: initialValues,
+      sects: sects,
+      schema: schema,
+      unknownParams: unknownParams,
+      extend: "extend" in inputs ? inputs.extend : false,
+    });
   }
 
-  resetInitialValues(metaParameters: InputsDetail["meta_parameters"]) {
+  async resetInitialValues(metaParameters: InputsDetail["meta_parameters"]) {
     this.setState({ resetting: true });
-    this.api
-      .resetInitialValues({
-        meta_parameters: tbLabelSchema.cast(metaParameters),
-      })
-      .then(data => {
-        const [
-          initialValues,
-          sects,
-          { meta_parameters, model_parameters },
-          schema,
-          unknownParams,
-        ] = convertToFormik(data);
-        this.setState(prevState => ({
-          inputs: {
-            ...prevState.inputs,
-            ...{
-              meta_parameters: meta_parameters,
-              model_parameters: model_parameters,
-            },
-          },
-          initialValues: initialValues,
-          sects: sects,
-          schema: schema,
-          unknownParams: unknownParams,
-          resetting: false,
-        }));
-      });
+    const data = await this.api.resetInitialValues({
+      meta_parameters: tbLabelSchema.cast(metaParameters),
+    });
+    const [
+      initialValues,
+      sects,
+      { meta_parameters, model_parameters },
+      schema,
+      unknownParams,
+    ] = convertToFormik(data);
+    this.setState(prevState => ({
+      inputs: {
+        ...prevState.inputs,
+        ...{
+          meta_parameters: meta_parameters,
+          model_parameters: model_parameters,
+        },
+      },
+      initialValues: initialValues,
+      sects: sects,
+      schema: schema,
+      unknownParams: unknownParams,
+      resetting: false,
+    }));
   }
 
   resetAccessStatus() {

diff --git a/webapp/apps/comp/asyncsubmit.py b/webapp/apps/comp/asyncsubmit.py
@@ -128,7 +128,11 @@ def submit(self):
         project = self.sim.project
         tag = str(project.latest_tag)
         self.submitted_id = self.compute.submit_job(
-            project=inputs.project, task_name=actions.SIM, task_kwargs=data, tag=tag,
+            project=inputs.project,
+            task_name=actions.SIM,
+            task_kwargs=data,
+            tag=tag,
+            path_prefix="/api/v1/jobs" if project.cluster.version == "v1" else "",
         )
         print(f"job id: {self.submitted_id}")