gpu check

app.py

@@ -10,7 +10,7 @@
 import numpy as np
 from utilsAPI import getAPIURL, getWorkerType
 from utilsAuth import getToken
-from utils import getDataDirectory, checkTime, checkResourceUsage, checkCuda
+from utils import getDataDirectory, checkTime, checkResourceUsage
 
 logging.basicConfig(level=logging.INFO)
 
@@ -57,9 +57,6 @@
     # Check resource usage
     resourceUsage = checkResourceUsage(stop_machine_and_email=True)
     logging.info(json.dumps(resourceUsage))
-    # TODO: not yet functional.
-    # Check GPU device is available
-    # checkCuda()
     logging.info(r.text)
 
     trial = r.json()

mmpose/loop_mmpose.py

@@ -3,13 +3,22 @@
 import logging
 import shutil
 import json
+import torch
 
 from utilsMMpose import detection_inference, pose_inference
 
 logging.basicConfig(level=logging.INFO)
 
 logging.info("Waiting for data...")
 
+def checkCudaPyTorch():
+    if torch.cuda.is_available():
+        num_gpus = torch.cuda.device_count()
+        logging.info(f"Found {num_gpus} GPU(s).")
+    else:
+        logging.info("No GPU detected. Exiting.")
+        raise Exception("No GPU detected. Exiting.")
+
 video_path = "/mmpose/data/video_mmpose.mov"
 output_dir = "/mmpose/data/output_mmpose"
 
@@ -38,6 +47,7 @@
     os.makedirs(output_dir)
 
     try:
+        checkCudaPyTorch()
         # Run human detection.
         pathModelCkptPerson = model_ckpt_person
         bboxPath = os.path.join(output_dir, 'box.pkl')

openpose/loop_openpose.py

@@ -4,8 +4,23 @@
 import shutil
 import ffmpeg
 import json
+import subprocess
 
-#%% 
+logging.basicConfig(level=logging.INFO)
+
+#%%
+def check_cuda_device():
+    try:
+        # Run the nvidia-smi command and capture the output
+        _ = subprocess.check_output(["nvidia-smi"])
+        # If the command ran successfully, assume a CUDA device is present
+        logging.info("A CUDA-capable device is detected.")
+    except subprocess.CalledProcessError as e:
+        # If the command fails, it means no CUDA device is detected
+        logging.info("No CUDA-capable device is detected. Error:", e)
+        raise Exception("No CUDA-capable device is detected.")
+
+#%%
 def getVideoOrientation(videoPath):
 
     meta = ffmpeg.probe(videoPath)
@@ -58,7 +73,6 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
     return cmd_hr
 
 #%% 
-logging.basicConfig(level=logging.INFO)
 
 logging.info("Waiting for data...")
 
@@ -87,6 +101,7 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
     horizontal = getVideoOrientation(video_path)
     cmd_hr = getResolutionCommand(resolutionPoseDetection, horizontal)
 
+    check_cuda_device()
     command = "/openpose/build/examples/openpose/openpose.bin\
         --video {video_path}\
         --display 0\

utils.py

@@ -1480,16 +1480,14 @@ def checkResourceUsage(stop_machine_and_email=True):
 
     return resourceUsage
 
-def checkCuda():
+def checkCudaTF():
     import tensorflow as tf
 
-    if tf.config.experimental.list_physical_devices('GPU'):
-        # Get the list of available GPUs
-        gpus = tf.config.experimental.list_physical_devices('GPU')
+    if tf.config.list_physical_devices('GPU'):
+        gpus = tf.config.list_physical_devices('GPU')
         print(f"Found {len(gpus)} GPU(s).")
-        # You can also print GPU device names and memory limits if needed
         for gpu in gpus:
-            print(f"GPU: {gpu.name}, Memory: {gpu.memory_limit}")
+            print(f"GPU: {gpu.name}")
     else:
         message = "Cuda check failed on an OpenCap machine backend machine: " \
                             + socket.gethostname() + ". It has been stopped."

utilsServer.py

@@ -443,6 +443,7 @@ def runTestSession(pose='all',isDocker=True):
         # trials['failure'] = '698162c8-3980-46e5-a3c5-8d4f081db4c4' # failed trial for testing
     else: # dev trials
         trials['openpose'] = '89d77579-8371-4760-a019-95f2c793622c' # session acd0e19c-6c86-4ba4-95fd-94b97229a926
+        trials['hrnet'] = 'e0e02393-42ee-46d4-9ae1-a6fbb0b89c42' # session 3510c726-a1b8-4de4-a4a2-52b021b4aab2
 
     if pose == 'all':
         trialList = list(trials.values())