Skip to content
Nightly integration tests

Nightly integration tests #506

Sign in to view logs

Nightly integration tests

Nightly integration tests #506

Workflow file for this run

.github/workflows/integration_tests.yml at 96267de
name: Nightly integration tests
concurrency:
# only one integration tests workflow to be run at a time, since it involves pushing/cleaning up a test image (same tag).
group: ${{ github.workflow }}${{ github.event.workflow_run.name }}
cancel-in-progress: false
# Run on request and every day at 3 AM UTC
on:
workflow_dispatch:
inputs:
target:
description: 'Target (choose nightly to run like nightly tests)'
required: true
default: 'nightly'
type: choice
options:
- nightly
- anyon
- ionq
- iqm
- oqc
- quantinuum
- nvqc
- orca
- fermioniq
single_test_name:
type: string
required: false
description: 'Single test (e.g., targettests/quantinuum/load_value.cpp). Runs default tests if left blank'
target_machine:
type: string
required: false
description: 'Target machine (e.g., H1-1E).'
cudaq_test_image:
type: string
required: false
default: '' # picked up from repo variable if not provided
description: 'CUDA Quantum image to run the tests in. Default to the latest CUDA Quantum nightly image'
commit_sha:
type: string
required: false
description: 'Commit SHA to pull the code (examples/tests) for testing. Default to the commit associated with the CUDA Quantum docker image if left blank'
cudaq_nvqc_deploy_image:
type: string
required: false
default: '' # same as cudaq_test_image if not provided
description: 'CUDA Quantum image to use for NVQC deployment to NVCF. Default to the latest CUDA Quantum nightly image'
workflow_id:
type: string
required: false
description: 'Workflow Id to retrieve the Python wheel for testing. Default to the wheels produced by the Publishing workflow associated with the latest nightly CUDA Quantum Docker image if left blank'
python_version:
type: choice
required: true
description: 'Python version to run wheel test'
options:
- '3.8'
- '3.9'
- '3.10'
- '3.11'
schedule:
- cron: 0 3 * * *
env:
# NGC nv-quantum organization: pnyjrcojiblh
NGC_QUANTUM_ORG: pnyjrcojiblh
NGC_QUANTUM_TEAM: cuda-quantum
NVQC_FUNCTION_ID: 3bfa0342-7d2a-4f1b-8e81-b6608d28ca7d
# <Backend>:<GPU Type>:<Instance Type>:<Min Instances>:<Max Instances>
NGC_NVQC_DEPLOYMENT_SPEC: GFN:L40S:gl40s_1.br25_2xlarge:1:1
python_version: '3.10'
jobs:
# We need this job purely to choose the container image values because the
# `env` context is unavailable outside of "steps" contexts.
setup:
name: Set variables
runs-on: ubuntu-latest
outputs:
cudaq_test_image: ${{ steps.vars.outputs.cudaq_test_image }}
cudaq_nvqc_deploy_image: ${{ steps.vars.outputs.cudaq_nvqc_deploy_image }}
steps:
- name: Set variables
id: vars
run: |
echo "cudaq_test_image=${{ inputs.cudaq_test_image || vars.cudaq_test_image }}" >> $GITHUB_OUTPUT
echo "cudaq_nvqc_deploy_image=${{ inputs.cudaq_nvqc_deploy_image || vars.cudaq_test_image }}" >> $GITHUB_OUTPUT
metadata:
name: Retrieve commit info
runs-on: ubuntu-latest
needs: setup
container:
image: ${{ needs.setup.outputs.cudaq_test_image }}
options: --user root
outputs:
cudaq_commit: ${{ steps.commit-sha.outputs.sha }}
steps:
- name: Get commit SHA
id: commit-sha
run: |
if [ -n "${{ inputs.commit_sha }}" ]; then
echo "sha=${{ inputs.commit_sha }}" >> $GITHUB_OUTPUT
else
echo "sha=$(cat $CUDA_QUANTUM_PATH/build_info.txt | grep -o 'source-sha: \S*' | cut -d ' ' -f 2)" >> $GITHUB_OUTPUT
fi
build_nvqc_image:
name: Build NVQC deployment image
runs-on: ubuntu-latest
if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
needs: [setup, metadata]
permissions:
contents: read
packages: write
environment: ghcr-deployment
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ needs.metadata.outputs.cudaq_commit }}
fetch-depth: 1
- name: Set up context for buildx
run: |
docker context create builder_context
- name: Set up buildx runner
uses: docker/setup-buildx-action@v3
with:
endpoint: builder_context
- name: Login to NGC container registry
uses: docker/login-action@v3
with:
registry: nvcr.io
username: '$oauthtoken'
password: ${{ secrets.NGC_CREDENTIALS }}
# Log in to GHCR (in case the base image is a local one)
- name: Log in to the GitHub container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Build NVQC image
id: docker_build
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/release/cudaq.nvqc.Dockerfile
build-args: |
base_image=${{ needs.setup.outputs.cudaq_nvqc_deploy_image }}
tags: nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/${{ vars.packages_prefix }}cuda-quantum:nightly
platforms: linux/amd64
provenance: false
push: true
deploy_nvqc_test_function:
name: Deploy NVQC function
runs-on: ubuntu-latest
needs: [metadata, build_nvqc_image]
if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
permissions:
contents: read
# Must have environment protection
environment: ghcr-deployment
outputs:
nvqc_function_version_id: ${{ steps.deploy.outputs.nvqc_function_version_id }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ needs.metadata.outputs.cudaq_commit }}
fetch-depth: 1
- name: Install NGC CLI
uses: ./.github/actions/install-ngc-cli
with:
version: 3.38.0
checksum: 427c67684d792b673b63882a6d0cbb8777815095c0f2f31559c1570a91187388
- name: Deploy NVQC Function
id: deploy
env:
NGC_CLI_API_KEY: ${{ secrets.NGC_CREDENTIALS }}
NGC_CLI_ORG: ${{ env.NGC_QUANTUM_ORG }}
NGC_CLI_TEAM: cuda-quantum
# When a new REST version is introduced, NVQC_REST_PAYLOAD_VERSION needs to be updated in lockstep with the new nightly CUDA Quantum image.
# Otherwise, deployment of the test function will fail.
run: |
# We run with CUDAQ_SER_CODE_EXEC set. The final NVQC deployment may
# or may not have this set, but since we run the client with
# CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1 (below), we need to run
# the CI with CUDAQ_SER_CODE_EXEC=1. If we ever remove
# CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1 below, we can consider
# removing CUDAQ_SER_CODE_EXEC=1.
create_function_result=$(ngc-cli/ngc cloud-function function create \
--container-image nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/cuda-quantum:nightly \
--container-environment-variable NUM_GPUS:1 \
--container-environment-variable NVQC_REST_PAYLOAD_VERSION:1.1 \
--container-environment-variable RUN_AS_NOBODY:1 \
--container-environment-variable CUDAQ_SER_CODE_EXEC:1 \
--api-body-format CUSTOM \
--inference-port 3030 \
--health-uri / \
--inference-url /job \
--name cudaq-nightly-integration-test \
$NVQC_FUNCTION_ID)
version_id=$(echo "$create_function_result" | grep 'Version: \S*' | head -1 | cut -d ':' -f 2 | tr -d ' ')
echo "Create version Id: $version_id"
echo "nvqc_function_version_id=$version_id" >> $GITHUB_OUTPUT
# Deploy it
ngc-cli/ngc cloud-function function deploy create --deployment-specification $NGC_NVQC_DEPLOYMENT_SPEC $NVQC_FUNCTION_ID:$version_id
function_status=DEPLOYING
while [ "$function_status" = "DEPLOYING" ]; do
echo "Waiting for deploying NVQC function version $version_id ..."
sleep 120
function_info=$(ngc-cli/ngc cloud-function function info $NVQC_FUNCTION_ID:$version_id)
function_status=$(echo "$function_info" | grep 'Status: \S*' | head -1 | cut -d ':' -f 2 | tr -d ' ')
done
if [ "$function_status" != "ACTIVE" ]; then
echo "::error:: Failed to deploy NVQC Test Function"
exit 1
fi
hardware_providers_integration_test:
name: Integration test with hardware providers
runs-on: ubuntu-latest
needs: [setup, metadata]
permissions:
contents: read
packages: read
# Must have environment protection
environment: backend-validation
container:
image: ${{ needs.setup.outputs.cudaq_test_image }}
options: --user root
steps:
- name: Get code
uses: actions/checkout@v4
with:
ref: ${{ needs.metadata.outputs.cudaq_commit }}
fetch-depth: 1
- name: Get tests
id: gettests
run: |
if [ -n "${{ inputs.single_test_name }}" ]; then
if [ -e ${{ inputs.single_test_name }} ]; then
echo "testlist=${{ inputs.single_test_name }}" >> $GITHUB_OUTPUT
else
# User's request test does not exit
echo "::error::File ${{ inputs.single_test_name}} not found"
exit 1
fi
else
filelist="targettests/*/*.cpp"
echo "testlist=$filelist" >> $GITHUB_OUTPUT
fi
- name: Setup anyon account
# This step is currently bypassed during nightly runs due to
# maintenance. Restore the if check to the original value when
# maintenance is complete.
#if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'anyon'
if: inputs.target == 'anyon'
run: |
curl -X POST --user "${{ secrets.ANYON_USERNAME }}:${{ secrets.ANYON_PASSWORD }}" -H "Content-Type: application/json" https://api.anyon.cloud:5000/login > credentials.json
id_token=`cat credentials.json | jq -r '."id_token"'`
refresh_token=`cat credentials.json | jq -r '."refresh_token"'`
echo "key: $id_token" > ~/.anyon_config
echo "refresh: $refresh_token" >> ~/.anyon_config
- name: QIR syntax check (Anyon)
# This step is currently bypassed during nightly runs due to
# maintenance. Restore the if check to the original value when
# maintenance is complete.
#if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'anyon'
if: inputs.target == 'anyon'
run: |
echo "### QIR syntax check (Anyon)" >> $GITHUB_STEP_SUMMARY
export CUDAQ_LOG_LEVEL="info"
set +e # Allow script to keep going through errors
test_err_sum=0
for filename in ${{ steps.gettests.outputs.testlist }}; do
case $filename in
targettests/anyon/*)
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
nvq++ -v $filename -DSYNTAX_CHECK --target anyon --anyon-machine telegraph-8q
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
;;
esac
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Setup quantinum account
if: github.event_name == 'schedule' || inputs.target == 'nightly' || inputs.target == 'quantinuum'
run: |
curl -X POST -H "Content Type: application/json" -d '{ "email":"${{ secrets.BACKEND_LOGIN_EMAIL }}","password":"${{ secrets.QUANTINUUM_PASSWORD }}" }' https://qapi.quantinuum.com/v1/login > credentials.json
id_token=`cat credentials.json | jq -r '."id-token"'`
refresh_token=`cat credentials.json | jq -r '."refresh-token"'`
echo "key: $id_token" > ~/.quantinuum_config
echo "refresh: $refresh_token" >> ~/.quantinuum_config
- name: QIR syntax check (Quantinuum)
if: github.event_name == 'schedule' || inputs.target == 'nightly'
run: |
echo "### QIR syntax check (Quantinuum)" >> $GITHUB_STEP_SUMMARY
export CUDAQ_LOG_LEVEL="info"
set +e # Allow script to keep going through errors
test_err_sum=0
for filename in ${{ steps.gettests.outputs.testlist }}; do
case $filename in
targettests/quantinuum/*)
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
nvq++ -v $filename -DSYNTAX_CHECK --target quantinuum --quantinuum-machine H1-1SC
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
;;
esac
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to IonQ Simulator
if: (success() || failure()) && (inputs.target == 'ionq' || github.event_name == 'schedule' || inputs.target == 'nightly')
run: |
echo "### Submit to IonQ Simulator" >> $GITHUB_STEP_SUMMARY
export IONQ_API_KEY='${{ secrets.IONQ_API_KEY }}'
# TODO: remove this flag once https://github.com/NVIDIA/cuda-quantum/issues/512 is addressed.
export CUDAQ_LOG_LEVEL="info"
set +e # Allow script to keep going through errors
test_err_sum=0
for filename in ${{ steps.gettests.outputs.testlist }}; do
case $filename in
targettests/ionq/*)
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
nvq++ -v $filename --target ionq
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
;;
esac
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to IQM Demo server
if: (success() || failure()) && (inputs.target == 'iqm' || github.event_name == 'schedule' || inputs.target == 'nightly')
run: |
# Must install iqm-cortex-cli to authenticate
pip install iqm-cortex-cli
echo "### Submit to IQM Demo server" >> $GITHUB_STEP_SUMMARY
# IQM demo server info is from: https://demo.qc.iqm.fi/cocos/info/
cortex init --config-file ${HOME}/.config/iqm-cortex-cli/config.json --tokens-file ${HOME}/.cache/iqm-cortex-cli/tokens.json --auth-server-url https://demo.qc.iqm.fi/auth --client-id iqm_client --realm cortex --username '${{ secrets.IQM_USER }}'
cortex auth login --username '${{ secrets.IQM_USER }}' --password '${{ secrets.IQM_PASSWORD }}'
echo ":white_check_mark: Successfully installed iqm-cortex-cli and logged in" >> $GITHUB_STEP_SUMMARY
# Use the demo machine, which is Adonis architecture
export IQM_SERVER_URL="https://demo.qc.iqm.fi/cocos"
export CUDAQ_LOG_LEVEL="info"
set +e # Allow script to keep going through errors
test_err_sum=0
for filename in ${{ steps.gettests.outputs.testlist }}; do
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
# Only the following tests are currently supported on IQM
case $filename in
targettests/iqm/*)
nvq++ -DSYNTAX_CHECK --target iqm --iqm-machine Adonis $filename
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
;;
esac
done
set -e # Re-enable exit code error checking
cortex auth logout -f
echo ":white_check_mark: Successfully logged out of IQM" >> $GITHUB_STEP_SUMMARY
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to OQC Sandbox server
# This step is currently bypassed during nightly runs due to
# maintenance. Restore the if check to the original value when
# maintenance is complete.
# if: (success() || failure()) && (inputs.target == 'oqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
if: (success() || failure()) && (inputs.target == 'oqc')
run: |
echo "### Submit to OQC sandbox server" >> $GITHUB_STEP_SUMMARY
export CUDAQ_LOG_LEVEL="info"
export OQC_EMAIL='${{ secrets.BACKEND_LOGIN_EMAIL }}'
export OQC_PASSWORD='${{ secrets.OQC_PASSWORD }}'
set +e # Allow script to keep going through errors
test_err_sum=0
for filename in ${{ steps.gettests.outputs.testlist }}; do
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
# Only the following tests are currently supported on OQC
case $filename in
targettests/oqc/*)
nvq++ -DSYNTAX_CHECK --target oqc $filename
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
;;
esac
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to ORCA test server
if: (success() || failure()) && (inputs.target == 'orca' || github.event_name == 'schedule' || inputs.target == 'nightly')
run: |
echo "### Submit to ORCA server" >> $GITHUB_STEP_SUMMARY
export ORCA_ACCESS_URL='${{ secrets.ORCA_ACCESS_URL }}'
set +e # Allow script to keep going through errors
test_err_sum=0
cpp_tests="docs/sphinx/targets/cpp/orca.cpp docs/sphinx/targets/cpp/orca_mqpu.cpp"
for filename in $cpp_tests; do
[ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
nvq++ --target orca --orca-url $ORCA_ACCESS_URL $filename
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
done
python_tests="docs/sphinx/targets/python/orca.py docs/sphinx/targets/python/orca_mqpu.py"
for filename in $python_tests; do
[ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
python3 $filename 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to Fermioniq test server
if: (success() || failure()) && (inputs.target == 'fermioniq' || github.event_name == 'schedule' || inputs.target == 'nightly')
run: |
echo "### Submit to Fermioniq server" >> $GITHUB_STEP_SUMMARY
export FERMIONIQ_ACCESS_TOKEN_ID='${{ secrets.FERMIONIQ_ACCESS_TOKEN_ID }}'
export FERMIONIQ_ACCESS_TOKEN_SECRET='${{ secrets.FERMIONIQ_ACCESS_TOKEN_SECRET }}'
export FERMIONIQ_PROJECT_ID='c86e2cb8-d776-41ae-8240-d3d55e18b7c3'
set +e # Allow script to keep going through errors
test_err_sum=0
cpp_tests="docs/sphinx/targets/cpp/fermioniq.cpp"
for filename in $cpp_tests; do
[ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
nvq++ \
--target fermioniq \
--fermioniq-project-id $FERMIONIQ_PROJECT_ID \
$filename
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
done
python_tests="docs/sphinx/targets/python/fermioniq.py docs/sphinx/targets/python/fermioniq_observables.py"
for filename in $python_tests; do
[ -e "$filename" ] || echo "::error::Couldn't find file ($filename)"
python3 $filename 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
- name: Submit to ${{ inputs.target }}
# The full set of tests used by this step is currently only supported on
# Quantinuum. The other supported tests are tested by the step above.
# The main point of this special step is to run with a special
# target_machine. It also doesn't use -DSYNTAX_CHECK, so you probably
# don't want to run this on a simple Syntax Check machine.
if: inputs.target == 'quantinuum' && github.event_name == 'workflow_dispatch'
run: |
if ${{inputs.target == 'ionq'}}; then
export IONQ_API_KEY='${{ secrets.IONQ_API_KEY }}'
fi
for filename in ${{ steps.gettests.outputs.testlist }}; do
[ -e "$filename" ] || echo "::error::Couldn't find files ($filename)"
case $filename in
targettests/quantinuum/*)
nvq++ -v $filename --target ${{ inputs.target }} --${{ inputs.target }}-machine ${{ inputs.target_machine }}
CUDAQ_LOG_LEVEL=info ./a.out
;;
esac
done
shell: bash
nvqc_integration_docker_test:
name: NVQC integration test using Docker image
runs-on: ubuntu-latest
if: (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
needs: [setup, metadata, build_nvqc_image, deploy_nvqc_test_function]
permissions:
contents: read
packages: read
# Must have environment protection
environment: ghcr-deployment
container:
image: ${{ needs.setup.outputs.cudaq_test_image }}
options: --user root
steps:
- name: Get code
uses: actions/checkout@v4
with:
ref: ${{ needs.metadata.outputs.cudaq_commit }}
fetch-depth: 1
- name: Submit to NVQC
run: |
echo "### Submit to NVQC" >> $GITHUB_STEP_SUMMARY
export NVQC_API_KEY="${{ secrets.NVQC_SERVICE_KEY }}"
export NVQC_FUNCTION_ID="$NVQC_FUNCTION_ID"
export NVQC_FUNCTION_VERSION_ID="${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
# When overriding the NVQC_FUNCTION_ID to a function that doesn't
# follow the production naming convenvtions, we need to set the
# following environment variable to tell the client that the server
# has all the remote capabilities.
export CUDAQ_CLIENT_REMOTE_CAPABILITY_OVERRIDE=1
set +e # Allow script to keep going through errors
test_err_sum=0
# Test all NVQPP execution tests
for filename in `find targettests/execution/ -name '*.cpp'`; do
echo "$filename"
# Only run tests that require execution (not a syntax-only check)
if grep -q "ifndef SYNTAX_CHECK" "$filename"; then
nvq++ -v $filename --target nvqc
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
fi
done
# Test all remote-sim tests
for filename in `find targettests/Remote-Sim -name '*.cpp'`; do
# unsupport_args and compile_errors are compile error tests
# nested_vectors: Compiler fails to synthesize nested vector parameters (https://github.com/NVIDIA/cuda-quantum/issues/2001)
# qvector_init_from_state, qvector_init_from_state_lazy, test_trotter: New argument synthesis is not executed for nvqc (https://github.com/NVIDIA/cuda-quantum/issues/2146)
if [[ "$filename" != *"unsupport_args"* ]] && [[ "$filename" != *"compile_errors"* ]] && [[ "$filename" != *"nested_vectors"* ]] && [[ "$filename" != *"qvector_init_from_state"* ]] && [[ "$filename" != *"qvector_init_from_state_lazy"* ]] && [[ "$filename" != *"test_trotter"* ]]; then
echo "$filename"
nvqc_config=""
# Look for a --remote-mqpu-auto-launch to determine the number of QPUs
num_qpus=`cat $filename | grep -oP -m 1 '^//\s*RUN:\s*nvq++.+--remote-mqpu-auto-launch\s+\K\S+'`
if [ -n "$num_qpus" ]; then
echo "Intended to run on '$num_qpus' QPUs."
nvqc_config="$nvqc_config --nvqc-nqpus $num_qpus"
fi
nvq++ -v $filename --target nvqc $nvqc_config
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
fi
done
# Test C++ examples with NVQC
for filename in `find examples/cpp/ applications/cpp/ targets/cpp/ -name '*.cpp'`; do
if [[ "$filename" == *"nvqc"* ]]; then
echo "$filename"
nvqc_config=""
# Look for a --nvqc-backend flag to nvq++ in the comment block
nvqc_backend=`sed -e '/^$/,$d' $filename | grep -oP -m 1 '^//\s*nvq++.+--nvqc-backend\s+\K\S+'`
if [ -n "$nvqc_backend" ]; then
echo "Intended for execution on '$nvqc_backend' backend."
nvqc_config="$nvqc_config --nvqc-backend $nvqc_backend"
fi
# Look for a --nvqc-nqpus flag to nvq++ in the comment block
num_qpus=`sed -e '/^$/,$d' $filename | grep -oP -m 1 '^//\s*nvq++.+--nvqc-nqpus\s+\K\S+'`
if [ -n "$num_qpus" ]; then
echo "Intended to run on '$num_qpus' QPUs."
nvqc_config="$nvqc_config --nvqc-nqpus $num_qpus"
fi
nvq++ -v $filename --target nvqc $nvqc_config
test_status=$?
if [ $test_status -eq 0 ]; then
./a.out
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $filename" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
echo ":x: Test failed (failed to compile): $filename" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
fi
done
# Test NVQC Python examples + Python MLIR execution tests (not IR tests)
python3 -m pip install pytest
for ex in `find examples/python python/tests/mlir/target -name '*.py'`; do
filename=$(basename -- "$ex")
filename="${filename%.*}"
echo "Testing $filename:"
if [[ "$ex" == *"nvqc"* ]]; then
# This is an NVQC example
python3 $ex 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
# Only run examples that are not target-specific (e.g., ionq, iqm)
if ! grep -q "set_target" "$ex"; then
# Use --target command line option to run these examples with nvqc
python3 $ex --target nvqc 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
fi
fi
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
shell: bash
nvqc_integration_wheel_test:
name: NVQC integration test using Python wheels
runs-on: ubuntu-latest
if: inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly'
needs: [metadata, build_nvqc_image, deploy_nvqc_test_function]
permissions:
contents: read
# Must have environment protection
environment: ghcr-deployment
steps:
- name: Get code
uses: actions/checkout@v4
with:
ref: ${{ needs.metadata.outputs.cudaq_commit }}
fetch-depth: 1
- name: Install wheel
id: install_wheel
run: |
python_version=${{ inputs.python_version || env.python_version }}
workflow_id=${{ inputs.workflow_id }}
# Helper to get the *valid* Publishing run Id for a commit hash
# Notes: runs that have 'CUDA Quantum Python wheels' jobs skipped are not considered.
function get_publishing_run_id {
# Find all Publishing runs, we'll look into its jobs' status later
if [[ -z "$1" ]]; then
publishing_run_ids=$(gh run list --workflow Publishing --json databaseId --jq .[].databaseId)
else
publishing_run_ids=$(gh run list --commit $1 --workflow Publishing --json databaseId --jq .[].databaseId)
fi
for run_id in $publishing_run_ids ; do
# Look into its jobs: if "CUDA Quantum Python wheels" matrix build was performed,
# then we have multiple jobs, like "CUDA Quantum Python wheels (python_arm64....")
cuda_wheel_build_jobs=$(gh run view $run_id --jq '.jobs.[] | select(.name | startswith("CUDA Quantum Python wheels (python_")).name' --json jobs)
if [ ! -z "$cuda_wheel_build_jobs" ]; then
# This is a valid run that produces wheel artifacts
echo $run_id
break
fi
done
}
if [ -z "${workflow_id}" ]; then
workflow_id=$(get_publishing_run_id ${{ needs.metadata.outputs.cudaq_commit }})
fi
if [ ! -z "$workflow_id" ]; then
echo "Using artifacts from workflow id $workflow_id"
# Allow error when trying to download wheel artifacts since they might be expired.
set +e
gh run download $workflow_id --name "x86_64-py$python_version-wheels"
retVal=$?
set -e
if [ $retVal -ne 0 ]; then
echo "Failed to download wheels artifact from Publishing workflow run Id $workflow_id. Perhaps the artifacts have been expired."
# This is allowed since there might be a period where no Publishing workflow is run (e.g., no PR merged to main).
echo "skipped=true" >> $GITHUB_OUTPUT
exit 0
fi
python_version_filename=$(echo "${python_version//.}")
# Install Python and the wheel
apt-get update && apt-get install -y --no-install-recommends python$python_version python3-pip
wheelfile=$(find . -name "cuda_quantum*cp$python_version_filename*x86_64.whl")
python$python_version -m pip install $wheelfile
echo "skipped=false" >> $GITHUB_OUTPUT
else
echo "Failed to retrieve Publishing workflow run Id for commit ${{ needs.metadata.outputs.cudaq_commit }}"
exit 1
fi
env:
GH_TOKEN: ${{ github.token }}
- name: Test NVQC
if: ${{ ! steps.install_wheel.skipped }}
run: |
echo "### Submit to NVQC from Python wheels" >> $GITHUB_STEP_SUMMARY
python_version=${{ inputs.python_version || env.python_version }}
export NVQC_API_KEY="${{ secrets.NVQC_SERVICE_KEY }}"
export NVQC_FUNCTION_ID="$NVQC_FUNCTION_ID"
export NVQC_FUNCTION_VERSION_ID="${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
set +e # Allow script to keep going through errors
python$python_version -m pip install pytest
test_err_sum=0
for ex in `find examples/python python/tests/mlir/target -name '*.py'`; do
filename=$(basename -- "$ex")
filename="${filename%.*}"
echo "Testing $filename:"
if [[ "$ex" == *"nvqc"* ]]; then
python$python_version $ex 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
else
# Only run examples that are not target-specific (e.g., ionq, iqm)
if ! grep -q "set_target" "$ex"; then
# Use --target command line option to run these examples with nvqc
python$python_version $ex --target nvqc 1> /dev/null
test_status=$?
if [ $test_status -eq 0 ]; then
echo ":white_check_mark: Successfully ran test: $ex" >> $GITHUB_STEP_SUMMARY
else
echo ":x: Test failed (failed to execute): $ex" >> $GITHUB_STEP_SUMMARY
test_err_sum=$((test_err_sum+1))
fi
fi
fi
done
set -e # Re-enable exit code error checking
if [ ! $test_err_sum -eq 0 ]; then
echo "::error::${test_err_sum} tests failed. See step summary for a list of failures"
exit 1
fi
cleanup_nvqc_resources:
name: Cleanup NVQC resources
runs-on: ubuntu-latest
if: (success() || failure()) && (inputs.target == 'nvqc' || github.event_name == 'schedule' || inputs.target == 'nightly')
needs: [build_nvqc_image, deploy_nvqc_test_function, nvqc_integration_docker_test, nvqc_integration_wheel_test]
permissions:
contents: read
# Must have environment protection
environment: ghcr-deployment
steps:
- name: Get code
uses: actions/checkout@v4
- name: Install NGC CLI
uses: ./.github/actions/install-ngc-cli
with:
version: 3.38.0
checksum: 427c67684d792b673b63882a6d0cbb8777815095c0f2f31559c1570a91187388
- name: Cleanup
env:
NGC_CLI_API_KEY: ${{ secrets.NGC_CREDENTIALS }}
NGC_CLI_ORG: ${{ env.NGC_QUANTUM_ORG }}
NGC_CLI_TEAM: cuda-quantum
run: |
echo "Version Id: ${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}"
# Remove deployment (make it inactive)
ngc-cli/ngc cloud-function function deploy remove $NVQC_FUNCTION_ID:${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}
# Remove the function version
ngc-cli/ngc cloud-function function remove $NVQC_FUNCTION_ID:${{ needs.deploy_nvqc_test_function.outputs.nvqc_function_version_id }}
# Remove the docker image
ngc-cli/ngc registry image remove -y nvcr.io/${{ env.NGC_QUANTUM_ORG }}/${{ env.NGC_QUANTUM_TEAM }}/cuda-quantum:nightly