-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
6405dbb
commit 796705b
Showing
17 changed files
with
611 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| #!/bin/bash | ||
| local OPTIND | ||
| set -euxo pipefail | ||
|
|
||
| print_usage() { | ||
| echo "usage: test_launcher.sh [-i imageName] [-p projectName] [-m metadata]" | ||
| echo " -i <imageName>: which image name to use for the VM" | ||
| echo " -p <imageProject>: which image project to use for the VM" | ||
| echo " -m <metadata>: metadata variables on VM creation; passed directly into gcloud" | ||
| echo " -f <metadataFromFile>: read a metadata value from a file; specified in format key=filePath" | ||
| echo " -n <instanceName>: instance name" | ||
| echo " -z <instanceZone>: instance zone" | ||
| echo " -v <machineType>: type of machine for VM" | ||
| echo " -g <gpuType>: type of GPU to use for the VM" | ||
| echo " -c <gpuCount>: number of GPU(s) to use for the VM" | ||
| exit 1 | ||
| } | ||
|
|
||
| create_vm() { | ||
| if [ -z "$IMAGE_NAME" ]; then | ||
| echo "Empty image name supplied." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "$GPU_TYPE" ]; then | ||
| echo "Empty gpu type supplied." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "$GPU_COUNT" ]; then | ||
| echo "Empty gpu count supplied." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "$MACHINE_TYPE" ]; then | ||
| echo "Empty machine type supplied." | ||
| exit 1 | ||
| fi | ||
|
|
||
| APPEND_METADATA='' | ||
| if ! [ -z "$METADATA" ]; then | ||
| APPEND_METADATA="--metadata ${METADATA}" | ||
| fi | ||
|
|
||
| APPEND_METADATA_FILE='' | ||
| if ! [ -z "$METADATA_FILE" ]; then | ||
| APPEND_METADATA_FILE="--metadata-from-file ${METADATA_FILE}" | ||
| fi | ||
|
|
||
| echo 'Creating VM' ${VM_NAME} 'with image' $IMAGE_NAME | ||
|
|
||
| # check the active account | ||
| gcloud auth list | ||
|
|
||
| gcloud compute instances create $VM_NAME \ | ||
| --maintenance-policy=TERMINATE \ | ||
| --machine-type=$MACHINE_TYPE \ | ||
| --boot-disk-size=$DISK_SIZE_GB \ | ||
| --accelerator=count=$GPU_COUNT,type=$GPU_TYPE \ | ||
| --scopes=cloud-platform \ | ||
| --zone=$ZONE \ | ||
| --image=$IMAGE_NAME \ | ||
| --image-project=$PROJECT_NAME \ | ||
| --shielded-secure-boot $APPEND_METADATA \ | ||
| $APPEND_METADATA_FILE | ||
| } | ||
|
|
||
| IMAGE_NAME='' | ||
| METADATA_FILE='' | ||
| METADATA='' | ||
| PROJECT_NAME='' | ||
| VM_NAME='' | ||
| ZONE='' | ||
| MACHINE_TYPE='' | ||
| GPU_TYPE='' | ||
| GPU_COUNT='' | ||
| DISK_SIZE_GB=100 | ||
|
|
||
|
|
||
| # In getopts, a ':' following a letter means that that flag takes an argument. | ||
| # For example, i: means -i takes an additional argument. | ||
| while getopts 'i:f:m:p:n:z:v:g:c:' flag; do | ||
| case "${flag}" in | ||
| i) IMAGE_NAME=${OPTARG} ;; | ||
| f) METADATA_FILE=${OPTARG} ;; | ||
| m) METADATA=${OPTARG} ;; | ||
| p) PROJECT_NAME=${OPTARG} ;; | ||
| n) VM_NAME=${OPTARG} ;; | ||
| z) ZONE=${OPTARG} ;; | ||
| v) MACHINE_TYPE=${OPTARG} ;; | ||
| g) GPU_TYPE=${OPTARG} ;; | ||
| c) GPU_COUNT=${OPTARG} ;; | ||
| *) print_usage ;; | ||
| esac | ||
| done | ||
|
|
||
| create_vm |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/bin/bash | ||
| set -euo pipefail | ||
| source util/read_serial.sh | ||
|
|
||
| # This test requires the workload to run and printing | ||
| # corresponding messages to the serial console. | ||
| SERIAL_OUTPUT=$(read_serial $1 $2) | ||
| print_serial=false | ||
|
|
||
| if echo $SERIAL_OUTPUT | grep -q 'failed to get the gpu type info' | ||
| then | ||
| echo "- no gpu verified" | ||
| else | ||
| echo "FAILED: gpu not detected" | ||
| echo 'TEST FAILED.' > /workspace/status.txt | ||
| print_serial=true | ||
| fi |
17 changes: 17 additions & 0 deletions
17
launcher/image/test/scripts/gpu/test_gpu_unsupported_gputype.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/bin/bash | ||
| set -euo pipefail | ||
| source util/read_serial.sh | ||
|
|
||
| # This test requires the workload to run and printing | ||
| # corresponding messages to the serial console. | ||
| SERIAL_OUTPUT=$(read_serial $1 $2) | ||
| print_serial=false | ||
|
|
||
| if echo $SERIAL_OUTPUT | grep -q 'unsupported gpu type' | ||
| then | ||
| echo "- unsupported gpu types verified" | ||
| else | ||
| echo "FAILED: gpu type is not supported" | ||
| echo 'TEST FAILED.' > /workspace/status.txt | ||
| print_serial=true | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/bin/bash | ||
| set -euo pipefail | ||
| source util/read_serial.sh | ||
|
|
||
| # This test requires the workload to run and printing | ||
| # corresponding messages to the serial console. | ||
| SERIAL_OUTPUT=$(read_serial $1 $2) | ||
| print_serial=false | ||
|
|
||
| if echo $SERIAL_OUTPUT | grep -q 'Test PASSED' | ||
| then | ||
| echo "- gpu workload running verified" | ||
| else | ||
| echo "FAILED: gpu workload not running" | ||
| echo 'TEST FAILED.' > /workspace/status.txt | ||
| print_serial=true | ||
| fi |
112 changes: 112 additions & 0 deletions
112
launcher/image/test/test_gpu_driver_installation_cloudbuild.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| substitutions: | ||
| '_IMAGE_NAME': '' | ||
| '_IMAGE_PROJECT': '' | ||
| '_CLEANUP': 'true' | ||
| '_VM_NAME_PREFIX': 'cs-gpu-test' | ||
| '_ZONE': 'us-central1-f' | ||
| '_WORKLOAD_IMAGE': 'us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/gpu/cuda-vector-add:latest' | ||
| steps: | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CreateShieldedVMWithSingleGPU | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['create_gpu_vm.sh','-i', '${_IMAGE_NAME}', | ||
| '-p', '${_IMAGE_PROJECT}', | ||
| '-m', 'tee-image-reference=${_WORKLOAD_IMAGE},tee-container-log-redirect=true,tee-install-gpu-driver=true', | ||
| '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}', | ||
| '-z', '${_ZONE}', | ||
| '-v', 'n1-standard-4', | ||
| '-g', 'nvidia-tesla-t4', | ||
| '-c', '1' | ||
| ] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CreateShieldedVMWithMultipleGPU | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['create_gpu_vm.sh','-i', '${_IMAGE_NAME}', | ||
| '-p', '${_IMAGE_PROJECT}', | ||
| '-m', 'tee-image-reference=${_WORKLOAD_IMAGE},tee-container-log-redirect=true,tee-install-gpu-driver=true', | ||
| '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}-mul', | ||
| '-z', '${_ZONE}', | ||
| '-v', 'n1-standard-4', | ||
| '-g', 'nvidia-tesla-t4', | ||
| '-c', '2' | ||
| ] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CreateShieldedVMWithUnsupportedGPU | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['create_gpu_vm.sh','-i', '${_IMAGE_NAME}', | ||
| '-p', '${_IMAGE_PROJECT}', | ||
| '-m', 'tee-image-reference=${_WORKLOAD_IMAGE},tee-container-log-redirect=true,tee-install-gpu-driver=true', | ||
| '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}-unsup', | ||
| '-z', '${_ZONE}', | ||
| '-v', 'n1-standard-4', | ||
| '-g', 'nvidia-tesla-p100', | ||
| '-c', '1' | ||
| ] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CreateShieldedVMWithNoGPU | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['create_vm.sh','-i', '${_IMAGE_NAME}', | ||
| '-p', '${_IMAGE_PROJECT}', | ||
| '-m', 'tee-image-reference=${_WORKLOAD_IMAGE},tee-container-log-redirect=true,tee-install-gpu-driver=true', | ||
| '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}-nogpu', | ||
| '-z', '${_ZONE}', | ||
| ] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: SingleGpuWorkloadTest | ||
| entrypoint: 'bash' | ||
| args: ['scripts/gpu/test_gpu_workload.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: MultipleGpuWorkloadTest | ||
| entrypoint: 'bash' | ||
| args: ['scripts/gpu/test_gpu_workload.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-mul', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: UnsupportedGpuWorkloadTest | ||
| entrypoint: 'bash' | ||
| args: ['scripts/gpu/test_gpu_unsupported_gputype.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-unsup', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: NoGpuWorkloadTest | ||
| entrypoint: 'bash' | ||
| args: ['scripts/gpu/test_gpu_nogpu.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-nogpu', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: SingleGpuCleanUp | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'CLEANUP=$_CLEANUP' | ||
| args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: MultipleGpuCleanUp | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'CLEANUP=$_CLEANUP' | ||
| args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-mul', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: UnsupportedGpuVmCleanUp | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'CLEANUP=$_CLEANUP' | ||
| args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-unsup', '${_ZONE}'] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: NoGpuVmCleanUp | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'CLEANUP=$_CLEANUP' | ||
| args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-nogpu', '${_ZONE}'] | ||
| # Must come after cleanup. | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: NoGpuVmCheckFailure | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['check_failure.sh'] | ||
|
|
||
| # options: | ||
| # pool: | ||
| # name: 'projects/confidential-space-images-dev/locations/us-west1/workerPools/cs-image-build-vpc' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # From current directory: | ||
| # gcloud builds submit --tag us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/gpu/cuda-vector-add:latest --project confidential-space-images-dev | ||
| FROM gcr.io/google_containers/cuda-vector-add:v0.1 | ||
|
|
||
| LABEL "tee.launch_policy.allow_env_override"="ALLOWED_OVERRIDE" | ||
| LABEL "tee.launch_policy.allow_cmd_override"="true" | ||
| LABEL "tee.launch_policy.log_redirect"="always" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package gpu | ||
|
|
||
| const ( | ||
| // InstallationHostDir is the directory where gpu drivers will be installed on the host machine. | ||
| InstallationHostDir = "/var/lib/nvidia" | ||
| // InstallationContainerDir is the directory where gpu drivers will be available on the workload container. | ||
| InstallationContainerDir = "/usr/local/nvidia" | ||
| ) |
Oops, something went wrong.