From 396cb93fe05f6570947ab13ce06beb879489c93a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 15 Jul 2021 22:45:02 -0400 Subject: [PATCH 1/6] hack in some numactl --- src/plotman/configuration.py | 21 ++++++++++++++++++--- src/plotman/manager.py | 16 +++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index df03e143..c302ea79 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -1,11 +1,13 @@ import contextlib import importlib +import itertools import os import stat import subprocess import tempfile import textwrap -from typing import Dict, Generator, List, Mapping, Optional +from collections.abc import Iterator +from typing import cast, Dict, Generator, Iterable, List, Mapping, Optional import appdirs import attr @@ -292,11 +294,24 @@ def _create_log_path(self, time: pendulum.DateTime, directory: str, group: str) timestamp = time.isoformat(timespec='microseconds').replace(':', '_') return os.path.join(directory, f'{timestamp}.{group}.log') -@attr.frozen +@attr.mutable class Directories: tmp: List[str] dst: Optional[List[str]] = None - tmp2: Optional[str] = None + tmp2: Optional[List[str]] = None + x: Optional[Iterator[str]] = None + y: str = '' + + def next_tmp2(self) -> None: + assert self.tmp2 is not None + if self.x is None: + self.x = itertools.cycle(self.tmp2) + + assert self.x is not None + self.y = next(self.x) + + def get_tmp2(self) -> str: + return self.y def dst_is_tmp(self) -> bool: return self.dst is None and self.tmp2 is None diff --git a/src/plotman/manager.py b/src/plotman/manager.py index a199dacf..a8db8e25 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -116,9 +116,11 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: dst_dirs = dir_cfg.get_dst_directories() + dir_cfg.next_tmp2() + dstdir: str if dir_cfg.dst_is_tmp2(): - dstdir = dir_cfg.tmp2 # type: ignore[assignment] + dstdir = dir_cfg.get_tmp2() elif tmpdir in dst_dirs: dstdir = tmpdir elif dir_cfg.dst_is_tmp(): @@ -151,9 +153,9 @@ def key(key: str) -> job.Phase: '-u', str(plotting_cfg.madmax.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] - if dir_cfg.tmp2 is not None: + if dir_cfg.get_tmp2() is not None: plot_args.append('-2') - plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) + plot_args.append(dir_cfg.get_tmp2() if dir_cfg.get_tmp2().endswith('/') else (dir_cfg.get_tmp2() + '/')) else: if plotting_cfg.chia is None: raise Exception( @@ -170,9 +172,9 @@ def key(key: str) -> job.Phase: plot_args.append('-e') if plotting_cfg.chia.x: plot_args.append('-x') - if dir_cfg.tmp2 is not None: + if dir_cfg.get_tmp2() is not None: plot_args.append('-2') - plot_args.append(dir_cfg.tmp2) + plot_args.append(dir_cfg.get_tmp2()) if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) @@ -222,6 +224,10 @@ def key(key: str) -> job.Phase: creationflags = 0 nice = 15 + node = dir_cfg.get_tmp2().rstrip('/')[-1] + + plot_args = ['numactl', f'--membind={node}', f'--cpunodebind={node}', *plot_args] + with open_log_file: # start_new_sessions to make the job independent of this controlling tty (POSIX only). # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. From 521644313a08693b1b2e9afc7705e54c8c42414f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 15 Jul 2021 22:49:29 -0400 Subject: [PATCH 2/6] "" --- src/plotman/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index c302ea79..2c7f91f3 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -299,7 +299,7 @@ class Directories: tmp: List[str] dst: Optional[List[str]] = None tmp2: Optional[List[str]] = None - x: Optional[Iterator[str]] = None + x: "Optional[Iterator[str]]" = None y: str = '' def next_tmp2(self) -> None: From d181688cadd9dae8d0a6b16ad75be3b01a5c1d3f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 15 Jul 2021 22:50:54 -0400 Subject: [PATCH 3/6] hmm --- src/plotman/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 2c7f91f3..f755f51b 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -299,7 +299,7 @@ class Directories: tmp: List[str] dst: Optional[List[str]] = None tmp2: Optional[List[str]] = None - x: "Optional[Iterator[str]]" = None + x: Optional[Iterator] = None y: str = '' def next_tmp2(self) -> None: From 62423eea1457dd87065e03bbf99e0d716cf24872 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 16 Jul 2021 02:59:43 +0000 Subject: [PATCH 4/6] hack it up, but it runs --- src/plotman/configuration.py | 6 +++--- src/plotman/manager.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index f755f51b..e3672e34 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -7,7 +7,7 @@ import tempfile import textwrap from collections.abc import Iterator -from typing import cast, Dict, Generator, Iterable, List, Mapping, Optional +from typing import Any, cast, Dict, Generator, Iterable, List, Mapping, Optional import appdirs import attr @@ -299,8 +299,8 @@ class Directories: tmp: List[str] dst: Optional[List[str]] = None tmp2: Optional[List[str]] = None - x: Optional[Iterator] = None - y: str = '' + x: Any = None + y: Any = '' def next_tmp2(self) -> None: assert self.tmp2 is not None diff --git a/src/plotman/manager.py b/src/plotman/manager.py index a8db8e25..1272f344 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -226,7 +226,7 @@ def key(key: str) -> job.Phase: node = dir_cfg.get_tmp2().rstrip('/')[-1] - plot_args = ['numactl', f'--membind={node}', f'--cpunodebind={node}', *plot_args] + plot_args = ['numactl', f'--membind={node}', f'--cpunodebind={node}', '--', *plot_args] with open_log_file: # start_new_sessions to make the job independent of this controlling tty (POSIX only). From ddea9b494200e1dd742c91565db2dfa264350d38 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 17 Jul 2021 01:40:57 +0000 Subject: [PATCH 5/6] extra prometheus output --- src/plotman/reporting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 8641d63b..14300100 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -138,6 +138,7 @@ def prometheus_report(jobs: typing.List[job.Job], tmp_prefix: str = '', dst_pref metrics = { 'plotman_plot_phase_major': 'The phase the plot is currently in', 'plotman_plot_phase_minor': 'The part of the phase the plot is currently in', + 'plotman_plot_phase_major_minor': 'major and minor', 'plotman_plot_tmp_usage': 'Tmp dir usage in bytes', 'plotman_plot_mem_usage': 'Memory usage in bytes', 'plotman_plot_user_time': 'Processor time (user) in s', @@ -157,6 +158,7 @@ def prometheus_report(jobs: typing.List[job.Job], tmp_prefix: str = '', dst_pref values = { 'plotman_plot_phase_major': j.progress().major, 'plotman_plot_phase_minor': j.progress().minor, + 'plotman_plot_phase_major_minor': j.progress().major + (j.progress().minor / 10), 'plotman_plot_tmp_usage': j.get_tmp_usage(), 'plotman_plot_mem_usage': j.get_mem_usage(), 'plotman_plot_user_time': j.get_time_user(), From bd9a1d792acd92d74668d3996bb329a005ed6245 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 22 Jul 2021 19:53:51 +0000 Subject: [PATCH 6/6] more --- src/plotman/madmax.py | 2 ++ src/plotman/manager.py | 1 + src/plotman/reporting.py | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index a4a2e026..89994311 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -71,5 +71,7 @@ def latest_command(self) -> CommandProtocol: type=str, default=None) @click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", type=str, default=None) +@click.option("-K", "--rmulti2", help="Thread multiplier for P2 (default = 1)", + type=int, default=1) def _cli_c8121b9() -> None: pass diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 1272f344..e396a4b9 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -148,6 +148,7 @@ def key(key: str) -> job.Phase: ) plot_args = [ plotting_cfg.madmax.executable, + '-K', str(2), '-n', str(1), '-r', str(plotting_cfg.madmax.n_threads), '-u', str(plotting_cfg.madmax.n_buckets), diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 14300100..83aef5eb 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -156,9 +156,9 @@ def prometheus_report(jobs: typing.List[job.Job], tmp_prefix: str = '', dst_pref } label_str = ','.join([f'{k}="{v}"' for k, v in labels.items()]) values = { - 'plotman_plot_phase_major': j.progress().major, - 'plotman_plot_phase_minor': j.progress().minor, - 'plotman_plot_phase_major_minor': j.progress().major + (j.progress().minor / 10), + 'plotman_plot_phase_major': j.progress().major if j.progress().known else 0, + 'plotman_plot_phase_minor': j.progress().minor if j.progress().known else 0, + 'plotman_plot_phase_major_minor': j.progress().major + (j.progress().minor / 10) if j.progress().known else 0, 'plotman_plot_tmp_usage': j.get_tmp_usage(), 'plotman_plot_mem_usage': j.get_mem_usage(), 'plotman_plot_user_time': j.get_time_user(),