Merge pull request #37 from wandb/fix-wandb-job-create

fix(launch): make repo compatible with git sourced jobs
wandb · Jan 19, 2024 · c1497c7 · c1497c7
2 parents fa8fe74 + f7d6f3c
commit c1497c7
Show file tree

Hide file tree

Showing 17 changed files with 44 additions and 15 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,26 +1,52 @@
 version: 2.1
 
 jobs:
-  build:
-    resource_class: medium
+  build_and_push_image:
+    parameters:
+      subdir:
+        type: string
     docker:
-      - image: cimg/base:2023.03
+      - image: cimg/python:3.7
     steps:
-      - setup_remote_docker:
-          docker_layer_caching: false
       - checkout
+      - setup_remote_docker
       - run:
-          name: Build Docker image
+          name: Check for Changes in Subdirectory
           command: |
-            docker buildx create --platform linux/amd64,linux/arm64 --use
-            echo $DOCKERHUB_TOKEN | docker login --username davidwandb --password-stdin
-            ./docker_build.py
+            if git diff --name-only HEAD~1...HEAD | grep '<< parameters.subdir >>'; then
+              echo "Changes detected in << parameters.subdir >>, building Docker image."
+              echo $DOCKERHUB_TOKEN | docker login --username davidwandb --password-stdin
+              docker buildx create --platform linux/amd64,linux/arm64 --use
+              cd << parameters.subdir >>
+              image_name=$(echo "<< parameters.subdir >>" | sed 's/jobs\//job_/g; s/\//_/g')
+              docker buildx build --platform linux/amd64,linux/arm64 -t wandb/$image_name:$CIRCLE_BRANCH --push --file Dockerfile.wandb . 
+            else
+              echo "No changes in << parameters.subdir >>, skipping build."
+            fi
 
 workflows:
-  build:
+  build_and_deploy:
     jobs:
-      - hold:
-          type: approval
-      - build:
-          requires:
-            - hold
+      - build_and_push_image:
+          filters:
+            branches:
+              only: main
+          matrix:
+            parameters:
+              subdir:
+                [
+                  "jobs/deploy_to_nvidia_triton",
+                  "jobs/deploy_to_sagemaker_endpoints",
+                  "jobs/distributed_test",
+                  "jobs/fashion_mnist_train",
+                  "jobs/github_actions_workflow_dispatch",
+                  "jobs/gpu_optimize_with_tensor_rt",
+                  "jobs/hello_world",
+                  "jobs/http_webhook",
+                  "jobs/msft_teams_webhook",
+                  "jobs/openai_evals",
+                  "jobs/sql_query",
+                  "jobs/stable_diffusion_inference",
+                  "jobs/sweep_schedulers/optuna_scheduler",
+                  "jobs/sweep_schedulers/wandb_scheduler",
+                ]
diff --git a/jobs/deploy_to_nvidia_triton/Dockerfile → .../deploy_to_nvidia_triton/Dockerfile.wandb b/jobs/deploy_to_nvidia_triton/Dockerfile → .../deploy_to_nvidia_triton/Dockerfile.wandb
diff --git a/...deploy_to_nvidia_triton/server/Dockerfile → ..._to_nvidia_triton/server/Dockerfile.wandb b/...deploy_to_nvidia_triton/server/Dockerfile → ..._to_nvidia_triton/server/Dockerfile.wandb
diff --git a/.../deploy_to_sagemaker_endpoints/Dockerfile → ...y_to_sagemaker_endpoints/Dockerfile.wandb b/.../deploy_to_sagemaker_endpoints/Dockerfile → ...y_to_sagemaker_endpoints/Dockerfile.wandb
diff --git a/jobs/distributed_test/Dockerfile → jobs/distributed_test/Dockerfile.wandb b/jobs/distributed_test/Dockerfile → jobs/distributed_test/Dockerfile.wandb
diff --git a/jobs/fashion_mnist_train/Dockerfile → jobs/fashion_mnist_train/Dockerfile.wandb b/jobs/fashion_mnist_train/Dockerfile → jobs/fashion_mnist_train/Dockerfile.wandb
diff --git a/...thub_actions_workflow_dispatch/Dockerfile → ...ctions_workflow_dispatch/Dockerfile.wandb b/...thub_actions_workflow_dispatch/Dockerfile → ...ctions_workflow_dispatch/Dockerfile.wandb
diff --git a/jobs/gpu_optimize_with_tensor_rt/Dockerfile → ..._optimize_with_tensor_rt/Dockerfile.wandb b/jobs/gpu_optimize_with_tensor_rt/Dockerfile → ..._optimize_with_tensor_rt/Dockerfile.wandb
diff --git a/jobs/hello_world/Dockerfile → jobs/hello_world/Dockerfile.wandb b/jobs/hello_world/Dockerfile → jobs/hello_world/Dockerfile.wandb
diff --git a/jobs/http_webhook/Dockerfile → jobs/http_webhook/Dockerfile.wandb b/jobs/http_webhook/Dockerfile → jobs/http_webhook/Dockerfile.wandb
diff --git a/jobs/msft_teams_webhook/Dockerfile → jobs/msft_teams_webhook/Dockerfile.wandb b/jobs/msft_teams_webhook/Dockerfile → jobs/msft_teams_webhook/Dockerfile.wandb
diff --git a/jobs/openai_evals/Dockerfile → jobs/openai_evals/Dockerfile.wandb b/jobs/openai_evals/Dockerfile → jobs/openai_evals/Dockerfile.wandb
diff --git a/jobs/sql_query/Dockerfile → jobs/sql_query/Dockerfile.wandb b/jobs/sql_query/Dockerfile → jobs/sql_query/Dockerfile.wandb
diff --git a/jobs/stable_diffusion_inference/Dockerfile → ...able_diffusion_inference/Dockerfile.wandb b/jobs/stable_diffusion_inference/Dockerfile → ...able_diffusion_inference/Dockerfile.wandb
diff --git a/...ep_schedulers/optuna_scheduler/Dockerfile → ...edulers/optuna_scheduler/Dockerfile.wandb b/...ep_schedulers/optuna_scheduler/Dockerfile → ...edulers/optuna_scheduler/Dockerfile.wandb
diff --git a/...eep_schedulers/wandb_scheduler/Dockerfile → ...hedulers/wandb_scheduler/Dockerfile.wandb b/...eep_schedulers/wandb_scheduler/Dockerfile → ...hedulers/wandb_scheduler/Dockerfile.wandb
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,3 @@
+# This is temporarily required to make this repo compatible
+# with the `wandb job create git` command, which until wandb==0.16.3
+# required a requirements.txt to exist in the root of the repository.