From 81581b6394b87c708159e5e4b0a21da71e98ae51 Mon Sep 17 00:00:00 2001 From: huiyuxie Date: Sun, 2 Feb 2025 12:06:09 -1000 Subject: [PATCH 1/3] Start --- src/TrixiCUDA.jl | 2 +- src/auxiliary/auxiliary.jl | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/TrixiCUDA.jl b/src/TrixiCUDA.jl index b414c442..d2f281e9 100644 --- a/src/TrixiCUDA.jl +++ b/src/TrixiCUDA.jl @@ -62,7 +62,7 @@ export semidiscretizeGPU # Get called every time the package is loaded function __init__() - # Initialize the device properties + # Initialize the device init_device() end diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index 1462b9e4..b05a5f38 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -1,17 +1,17 @@ include("configurators.jl") include("stable.jl") -# Initialize the device properties +# Initialize the device function init_device() + # Get the device properties try - # Consider single GPU for now # TODO: Consider multiple GPUs later device = CUDA.device() - # Get the device properties global MULTIPROCESSOR_COUNT = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) global MAX_THREADS_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK) global MAX_SHARED_MEMORY_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK) + # Catch the errors catch e # Handle the errors if e isa CUDA.CuError @@ -26,4 +26,20 @@ function init_device() global MAX_THREADS_PER_BLOCK = 0 global MAX_SHARED_MEMORY_PER_BLOCK = 0 end + + # Initialize the CUDA streams + try + global STREAM1 = CUDA.CuStream() + global STREAM2 = CUDA.CuStream() + # Catch the errors + catch e + println("Error initializing CUDA streams: ", e) + println("Ensure there are enough GPU resources for stream creation.") + + # Fallback to default stream + global STREAM1 = CUDA.stream() + global STREAM2 = CUDA.stream() + + println("Using the default stream as a fallback.") + end end From 14c3c59833ab34578c8cfea2d66094cb40ec968b Mon Sep 17 00:00:00 2001 From: huiyuxie Date: Sun, 2 Feb 2025 12:39:24 -1000 Subject: [PATCH 2/3] Fix errors --- src/auxiliary/auxiliary.jl | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index b05a5f38..5afb73a5 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -3,20 +3,23 @@ include("stable.jl") # Initialize the device function init_device() - # Get the device properties try # TODO: Consider multiple GPUs later device = CUDA.device() + # Get properties global MULTIPROCESSOR_COUNT = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) global MAX_THREADS_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK) global MAX_SHARED_MEMORY_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK) - # Catch the errors + + # Create CUDA streams + global STREAM1 = CUDA.CuStream() + global STREAM2 = CUDA.CuStream() catch e # Handle the errors if e isa CUDA.CuError println("Error initializing device: ", e.msg) - println("Ensure a CUDA-enabled GPU is available and properly configured.") + println("Ensure a CUDA compatible GPU is available and properly configured.") else println("An unexpected error occurred: ", e) end @@ -25,21 +28,7 @@ function init_device() global MULTIPROCESSOR_COUNT = 0 global MAX_THREADS_PER_BLOCK = 0 global MAX_SHARED_MEMORY_PER_BLOCK = 0 - end - - # Initialize the CUDA streams - try - global STREAM1 = CUDA.CuStream() - global STREAM2 = CUDA.CuStream() - # Catch the errors - catch e - println("Error initializing CUDA streams: ", e) - println("Ensure there are enough GPU resources for stream creation.") - - # Fallback to default stream - global STREAM1 = CUDA.stream() - global STREAM2 = CUDA.stream() - - println("Using the default stream as a fallback.") + global STREAM1 = nothing + global STREAM2 = nothing end end From c1450a8e07569e191f001b7ecdf2943280d77799 Mon Sep 17 00:00:00 2001 From: huiyuxie Date: Sun, 2 Feb 2025 13:00:54 -1000 Subject: [PATCH 3/3] Enhance error handling --- src/auxiliary/auxiliary.jl | 52 ++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/auxiliary/auxiliary.jl b/src/auxiliary/auxiliary.jl index 5afb73a5..634e9e74 100644 --- a/src/auxiliary/auxiliary.jl +++ b/src/auxiliary/auxiliary.jl @@ -7,27 +7,53 @@ function init_device() # TODO: Consider multiple GPUs later device = CUDA.device() - # Get properties - global MULTIPROCESSOR_COUNT = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) - global MAX_THREADS_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK) - global MAX_SHARED_MEMORY_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK) + try + # Get properties + global MULTIPROCESSOR_COUNT = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) + global MAX_THREADS_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK) + global MAX_SHARED_MEMORY_PER_BLOCK = CUDA.attribute(device, CUDA.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK) + catch e + # Handle CUDA errors + println("Error getting device properties: ", e.msg) + println("Ensure that your GPU device properties can be retrieved successfully.") + + # Fall back to set zeros + global MULTIPROCESSOR_COUNT = 0 + global MAX_THREADS_PER_BLOCK = 0 + global MAX_SHARED_MEMORY_PER_BLOCK = 0 + println("Device properties have been set to zero. CUDA operations + will fail because no valid properties are configured.") + end + + try + # Create CUDA streams + global STREAM1 = CUDA.CuStream() + global STREAM2 = CUDA.CuStream() + catch e + # Handle CUDA errors + println("Error initializing CUDA streams: ", e.msg) + println("Ensure there are enough GPU resources available for stream creation.") + + # Fall back to set default streams + global STREAM1 = CUDA.stream() + global STREAM2 = CUDA.stream() + println("Streams have been set to the default CUDA stream. + This will impact performance due to no concurrency.") + end - # Create CUDA streams - global STREAM1 = CUDA.CuStream() - global STREAM2 = CUDA.CuStream() catch e - # Handle the errors + # Handle CUDA errors or not if e isa CUDA.CuError - println("Error initializing device: ", e.msg) + println("Error detecting device: ", e.msg) println("Ensure a CUDA compatible GPU is available and properly configured.") else println("An unexpected error occurred: ", e) end - # Fall back to set default values - global MULTIPROCESSOR_COUNT = 0 - global MAX_THREADS_PER_BLOCK = 0 - global MAX_SHARED_MEMORY_PER_BLOCK = 0 + # Fall back to set nothing + global MULTIPROCESSOR_COUNT = nothing + global MAX_THREADS_PER_BLOCK = nothing + global MAX_SHARED_MEMORY_PER_BLOCK = nothing global STREAM1 = nothing global STREAM2 = nothing end