update gpu_labeler to label only nodes with gpus

sky/utils/kubernetes/gpu_labeler.py

@@ -89,8 +89,16 @@ def label():
 
         # Iterate over nodes
         nodes = v1.list_node().items
-        # TODO(romilb): Run this only on nodes with GPUs.
+
+        # Get the list of nodes with GPUs
+        gpu_nodes = []
         for node in nodes:
+            if 'nvidia.com/gpu' in node.status.capacity:
+                gpu_nodes.append(node)
+
+        print(f'Found {len(gpu_nodes)} GPU nodes in the cluster')
+
+        for node in gpu_nodes:
             node_name = node.metadata.name
 
             # Modify the job manifest for the current node
@@ -103,12 +111,17 @@ def label():
             # Create the job for this node`
             batch_v1.create_namespaced_job(namespace, job_manifest)
             print(f'Created GPU labeler job for node {node_name}')
-    print('GPU labeling started - this may take a few minutes to complete.'
-          '\nTo check the status of GPU labeling jobs, run '
-          '`kubectl get jobs --namespace=kube-system -l job=sky-gpu-labeler`'
-          '\nYou can check if nodes have been labeled by running '
-          '`kubectl describe nodes` and looking for labels of the format '
-          '`skypilot.co/accelerators: <gpu_name>`. ')
+    if len(gpu_nodes) == 0:
+        print('No GPU nodes found in the cluster. If you have GPU nodes, '
+              'please ensure that they have the label '
+              '`nvidia.com/gpu: <number of GPUs>`')
+    else:
+        print('GPU labeling started - this may take a few minutes to complete.'
+              '\nTo check the status of GPU labeling jobs, run '
+              '`kubectl get jobs -n kube-system -l job=sky-gpu-labeler`'
+              '\nYou can check if nodes have been labeled by running '
+              '`kubectl describe nodes` and looking for labels of the format '
+              '`skypilot.co/accelerators: <gpu_name>`. ')
 
 
 def main():