Skip to content

Commit

Permalink
Merge pull request #25 from rit628/master
Browse files Browse the repository at this point in the history 
Added GPU & Memory util meters with -m flag
  • Loading branch information
@peci1
peci1 authored Mar 23, 2024
2 parents 2c9f026 + ee7940c commit 2b4eeee
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 0 deletions.
28 changes: 28 additions & 0 deletions nvidia-htop.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
parser = argparse.ArgumentParser()
parser.add_argument('-l', '--command-length', default=20, const=100, type=int, nargs='?')
parser.add_argument('-c', '--color', action='store_true')
parser.add_argument('-m', '--meter', action='store_true', help="Shows meters for GPU and VRAM utilization")
parser.add_argument('-u', '--user', default='', help="Limit the list of processes to selected users (comma-separated)")
parser.add_argument('-i', '--id', default='', help="Limit the command to selected GPU IDs (comma-separated)")
# only for testing
Expand All @@ -41,6 +42,7 @@
# parse the command length argument
command_length = args.command_length
color = args.color
meter = args.meter
fake_ps = args.fake_ps
users = set(args.user.split(',')) if len(args.user) > 0 else None

Expand Down Expand Up @@ -71,8 +73,31 @@
lines = [line + '\n' for line in lines_proc[:-1]]
lines += lines_proc[-1]

def add_meters(_lines):
pattern = re.compile(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+/\s+([0-9]+)MiB.*\s([0-9]+)%")
gpu_util_lines = list(filter(lambda tup: pattern.match(tup[1]), enumerate(_lines)))
for i, line in gpu_util_lines:
m = pattern.match(line)
used_mem = int(m.group(1))
total_mem = int(m.group(2))
gpu_util = int(m.group(3)) / 100.0
mem_util = used_mem / float(total_mem)
# Uses empty space underneath gpu & mem util stats for meter placement.
meter_space = re.split(r"\|(?!$)", _lines[i+1])
gpu_util_space, mig, _ = re.split(r"([^ |]+ \|)", meter_space[-1], maxsplit=1)
gpu_meter_space = len(gpu_util_space)-4
mem_meter_space = len(meter_space[2])-4
# Fill gpu and mem util meters proportional to reported utilization
gpu_meter = "|"*round(gpu_util*gpu_meter_space) + " "*round((1-gpu_util)*gpu_meter_space)
mem_meter = "|"*round(mem_util*mem_meter_space) + " "*round((1-mem_util)*mem_meter_space)
meter_space[-1] = f" [{gpu_meter}] {mig}"
meter_space[2] = f" [{mem_meter}] "
_lines[i+1] = '|'.join(meter_space)

return _lines

def colorize(_lines):
_lines = add_meters(_lines) if meter else _lines
# Index of the first content line of the current cell in the nvidia-smi output.
start_idx = 0
for j in range(len(_lines)):
Expand Down Expand Up @@ -118,6 +143,9 @@ def colorize(_lines):
if color:
lines_to_print = colorize(lines_to_print)

elif meter:
lines_to_print = add_meters(lines_to_print)

# we print all but the last line which is the +---+ separator
for line in lines_to_print[:-1]:
print(line)
Expand Down
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-----------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-----------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_L
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_L150
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 1111021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 1114518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-------------------------------------------------------------------------------+
16 changes: 16 additions & 0 deletions test/test_main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,22 @@ def test_no_processes(self):
def test_no_processes_docker(self):
self.do_test('FAKE_STDIN_NO_PROCESSES_DOCKER', 'DESIRED_STDOUT_NO_PROCESSES_DOCKER')

def test_with_meters(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER', call_args=["-m"])

def test_with_meters_color(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_COLOR', call_args=["-m", "-c"])

def test_with_meters_long_pids(self):
self.do_test('FAKE_STDIN_LONG_PIDS', 'DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS', call_args=["-m"], fake_ps='FAKE_PS_LONG_PIDS')

def test_with_meters_long(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L', call_args=["-m", "-l"])

def test_with_meters_very_long(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L150', call_args=["-m", "-l", "150"])



if __name__ == '__main__':
unittest.main()

0 comments on commit 2b4eeee

Please sign in to comment.