Update docker nightly workflow to use GPU runner since these have mor…

…e space (#2586)

* Update workflow to use GPU runner since these have 1TB space

* Update workflow to use GPU runner since these have 1TB space

* clean up built images

* clean up built images

* Use a single docker prune command

.github/workflows/docker-nightly-build.yml

@@ -6,8 +6,15 @@ on:
     - cron: "15 13 * * *"
 jobs:
   nightly:
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, ci-gpu]
     steps:
+      - name: Clean up previous run
+        run: |
+          echo "Cleaning up previous run"
+          ls -la ./
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.??* || true
+          ls -la ./
       - name: Setup Python 3.8
         uses: actions/setup-python@v4
         with:
@@ -22,11 +29,11 @@ jobs:
       - name: Push Docker Nightly
         run: |
           cd docker
-          python docker_nightly.py
+          python docker_nightly.py --cleanup
       - name: Push KServe Docker Nightly
         run: |
           cd kubernetes/kserve
-          python docker_nightly.py
+          python docker_nightly.py --cleanup
 
       - name: Open issue on failure
         if: ${{ failure() && github.event_name  == 'schedule' }}

docker/docker_nightly.py

@@ -23,6 +23,11 @@
         action="store_true",
         help="dry_run will print the commands that will be run without running them",
     )
+    parser.add_argument(
+        "--cleanup",
+        action="store_true",
+        help="Delete all built docker images",
+    )
     args = parser.parse_args()
     dry_run = args.dry_run
     organization = args.organization
@@ -55,3 +60,7 @@
     # Push images with latest tag
     try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run)
     try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run)
+
+    # Cleanup built images
+    if args.cleanup:
+        try_and_handle(f"docker system prune --all --volumes", dry_run)

kubernetes/kserve/docker_nightly.py

@@ -23,6 +23,11 @@
         action="store_true",
         help="dry_run will print the commands that will be run without running them",
     )
+    parser.add_argument(
+        "--cleanup",
+        action="store_true",
+        help="Delete all built docker images",
+    )
     args = parser.parse_args()
     dry_run = args.dry_run
     organization = args.organization
@@ -55,3 +60,7 @@
     # Push images with latest tag
     try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run)
     try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run)
+
+    # Cleanup built images
+    if args.cleanup:
+        try_and_handle(f"docker system prune --all --volumes", dry_run)