Run CUDA tests as part of GPU tests.

Attempt #2.

This runs in continuous mode only.

PiperOrigin-RevId: 689056926

.buildkite/pipeline.yaml

@@ -190,6 +190,16 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *common
+    <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 15 minutes to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"

.buildkite/release.yaml

@@ -67,6 +67,15 @@ steps:
       - make gpu-all-tests
     agents:
       queue: gpu
+  - <<: *common
+    label: ":fish: CUDA tests in compatibility-verifying mode"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around an hour to run.
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
   - <<: *common
     label: ":screwdriver: All GPU Drivers Test"
     commands:

Makefile

@@ -331,6 +331,11 @@ cos-gpu-all-tests: gpu-images cos-gpu-smoke-tests $(RUNTIME_BIN)
 	@$(call sudo,test/gpu:sniffer_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
 .PHONY: cos-gpu-all-tests
 
+cuda-tests: load-gpu_cuda-tests
+	@$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-docker=true)
+	@$(call sudo,test/gpu:cuda_test,--runtime=$(RUNTIME) -test.v $(ARGS))
+.PHONY: cuda-tests
+
 portforward-tests: load-basic_redis load-basic_nginx $(RUNTIME_BIN)
 	@$(call install_runtime,$(RUNTIME),--network=sandbox)
 	@$(call sudo,test/root:portforward_test,--runtime=$(RUNTIME) -test.v $(ARGS))

test/gpu/cuda_test.go

@@ -745,7 +745,7 @@ func TestCUDA(t *testing.T) {
 		durationPct := 100.0 * float64(startedAgo) / float64(testDeadline.Sub(testStart))
 		testLog(t, "[Timing] %d/%d tests (%.1f%%) finished executing. Test started %v ago, deadline in %v (%.1f%%).", testsDone, numTests, donePct, startedAgo.Truncate(time.Second), deadlineIn.Truncate(time.Second), durationPct)
 		if len(failedTests) > 0 {
-			testLog(t, "[Failed] %d test failed: %v", len(failedTests), strings.Join(failedTests, ", "))
+			testLog(t, "[Failed] %d test(s) failed: %v", len(failedTests), strings.Join(failedTests, ", "))
 		}
 		testLog(t, "[Pool] %v", cp.String())
 	}