Merge branch 'main' into xren/nsys_profiling

xrennvidia · Aug 15, 2024 · 31ac853 · 31ac853
2 parents 2da1e47 + 227c0aa
commit 31ac853
Show file tree

Hide file tree

Showing 23 changed files with 615 additions and 240 deletions.
diff --git a/.github/workflows/_test_template.yml b/.github/workflows/_test_template.yml
@@ -34,22 +34,27 @@ on:
         description: Last 2000 characters of the test step's log
         value: ${{ jobs.main.outputs.log }} 
 jobs:
+
   main:
     runs-on: ${{ inputs.RUNNER }} 
     outputs:
       conclusion: ${{ steps.main.conclusion }}
       log: ${{ steps.main.outputs.log }}
-    permissions:
-      actions: write # Required for cancelling workflows 
     steps:
         - name: Docker system cleanup
           run: |
             docker system prune -a --filter "until=48h" --force
 
+        - name: Docker pull image
+          run: |
+            docker pull nemoci.azurecr.io/nemo_container_${{ github.run_id }}
+
         - id: main
           name: Run main script
           timeout-minutes: ${{ inputs.TIMEOUT }}
           run: |
+            mkdir -p ${{ github.run_id }}
+            cd ${{ github.run_id }}/
             set +e 
             (  
               set -e
@@ -65,9 +70,7 @@ jobs:
             
         - uses: "NVIDIA/NeMo/.github/actions/cancel-workflow@main"
           if: failure() && inputs.IS_OPTIONAL == false
-
         - name: after_script
           if: always() && inputs.AFTER_SCRIPT != ':'
           run: |
-            docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'
-            
+            docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'