diff --git a/nvidia-htop.py b/nvidia-htop.py index af53528..583aba9 100755 --- a/nvidia-htop.py +++ b/nvidia-htop.py @@ -31,6 +31,7 @@ parser = argparse.ArgumentParser() parser.add_argument('-l', '--command-length', default=20, const=100, type=int, nargs='?') parser.add_argument('-c', '--color', action='store_true') +parser.add_argument('-m', '--meter', action='store_true', help="Shows meters for GPU and VRAM utilization") parser.add_argument('-u', '--user', default='', help="Limit the list of processes to selected users (comma-separated)") parser.add_argument('-i', '--id', default='', help="Limit the command to selected GPU IDs (comma-separated)") # only for testing @@ -41,6 +42,7 @@ # parse the command length argument command_length = args.command_length color = args.color +meter = args.meter fake_ps = args.fake_ps users = set(args.user.split(',')) if len(args.user) > 0 else None @@ -71,8 +73,31 @@ lines = [line + '\n' for line in lines_proc[:-1]] lines += lines_proc[-1] +def add_meters(_lines): + pattern = re.compile(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+/\s+([0-9]+)MiB.*\s([0-9]+)%") + gpu_util_lines = list(filter(lambda tup: pattern.match(tup[1]), enumerate(_lines))) + for i, line in gpu_util_lines: + m = pattern.match(line) + used_mem = int(m.group(1)) + total_mem = int(m.group(2)) + gpu_util = int(m.group(3)) / 100.0 + mem_util = used_mem / float(total_mem) + # Uses empty space underneath gpu & mem util stats for meter placement. + meter_space = re.split(r"\|(?!$)", _lines[i+1]) + gpu_util_space, mig, _ = re.split(r"([^ |]+ \|)", meter_space[-1], maxsplit=1) + gpu_meter_space = len(gpu_util_space)-4 + mem_meter_space = len(meter_space[2])-4 + # Fill gpu and mem util meters proportional to reported utilization + gpu_meter = "|"*round(gpu_util*gpu_meter_space) + " "*round((1-gpu_util)*gpu_meter_space) + mem_meter = "|"*round(mem_util*mem_meter_space) + " "*round((1-mem_util)*mem_meter_space) + meter_space[-1] = f" [{gpu_meter}] {mig}" + meter_space[2] = f" [{mem_meter}] " + _lines[i+1] = '|'.join(meter_space) + + return _lines def colorize(_lines): + _lines = add_meters(_lines) if meter else _lines # Index of the first content line of the current cell in the nvidia-smi output. start_idx = 0 for j in range(len(_lines)): @@ -118,6 +143,9 @@ def colorize(_lines): if color: lines_to_print = colorize(lines_to_print) +elif meter: + lines_to_print = add_meters(lines_to_print) + # we print all but the last line which is the +---+ separator for line in lines_to_print[:-1]: print(line) diff --git a/test/DESIRED_STDOUT_NEW_FORMAT_METER b/test/DESIRED_STDOUT_NEW_FORMAT_METER new file mode 100644 index 0000000..8150a81 --- /dev/null +++ b/test/DESIRED_STDOUT_NEW_FORMAT_METER @@ -0,0 +1,26 @@ +Sun Aug 2 13:44:21 2020 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | +| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | +| | [|| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ +| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | +| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | +| | [|||||||| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | +| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | +| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | +| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | +| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | +| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | +| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | ++-----------------------------------------------------------------------------+ diff --git a/test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR b/test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR new file mode 100644 index 0000000..be2a474 --- /dev/null +++ b/test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR @@ -0,0 +1,26 @@ +Sun Aug 2 13:44:21 2020 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | +| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | +| | [|| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ +| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | +| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | +| | [|||||||| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | +| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | +| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | +| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | +| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | +| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | +| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | ++-----------------------------------------------------------------------------+ diff --git a/test/DESIRED_STDOUT_NEW_FORMAT_METER_L b/test/DESIRED_STDOUT_NEW_FORMAT_METER_L new file mode 100644 index 0000000..81ea4b0 --- /dev/null +++ b/test/DESIRED_STDOUT_NEW_FORMAT_METER_L @@ -0,0 +1,26 @@ +Sun Aug 2 13:44:21 2020 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | +| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | +| | [|| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ +| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | +| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | +| | [|||||||| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ + ++-------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | +| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | +| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | +| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | +| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | +| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | +| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/test/DESIRED_STDOUT_NEW_FORMAT_METER_L150 b/test/DESIRED_STDOUT_NEW_FORMAT_METER_L150 new file mode 100644 index 0000000..e58740f --- /dev/null +++ b/test/DESIRED_STDOUT_NEW_FORMAT_METER_L150 @@ -0,0 +1,26 @@ +Sun Aug 2 13:44:21 2020 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | +| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | +| | [|| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ +| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | +| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | +| | [|||||||| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ + ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | +| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | +| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | +| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | +| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | +| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | +| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS b/test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS new file mode 100644 index 0000000..83333a1 --- /dev/null +++ b/test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS @@ -0,0 +1,26 @@ +Sun Aug 2 13:44:21 2020 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | +| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | +| | [|| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ +| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | +| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | +| | [|||||||| ] | [ ] N/A | ++-------------------------------+----------------------+----------------------+ + ++-------------------------------------------------------------------------------+ +| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | +| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | +| 1 1111021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | +| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | +| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | +| 1 1114518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | +| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | ++-------------------------------------------------------------------------------+ diff --git a/test/test_main.py b/test/test_main.py index c6dea28..7216a01 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -50,6 +50,22 @@ def test_no_processes(self): def test_no_processes_docker(self): self.do_test('FAKE_STDIN_NO_PROCESSES_DOCKER', 'DESIRED_STDOUT_NO_PROCESSES_DOCKER') + def test_with_meters(self): + self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER', call_args=["-m"]) + + def test_with_meters_color(self): + self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_COLOR', call_args=["-m", "-c"]) + + def test_with_meters_long_pids(self): + self.do_test('FAKE_STDIN_LONG_PIDS', 'DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS', call_args=["-m"], fake_ps='FAKE_PS_LONG_PIDS') + + def test_with_meters_long(self): + self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L', call_args=["-m", "-l"]) + + def test_with_meters_very_long(self): + self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L150', call_args=["-m", "-l", "150"]) + + if __name__ == '__main__': unittest.main()