From 684396646edcdf78f6bdb749f384b5427e642be0 Mon Sep 17 00:00:00 2001 From: "T.J. Alumbaugh" Date: Mon, 5 Feb 2024 20:08:48 -0700 Subject: [PATCH] Set n_workers for dask LocalCluster - Explicity set n_workers for LocalCluster creation (through Client). If this is not set, for machines with a high CPU count, the number of workers will be much larger than 7. In `client.scatter` with `broadcast=True`, this can cause hangs due to how dask distributed works. It's presumed that this bug is only apparent in more recent runs on machines with high CPU counts, although that is not confirmed. But, this fix does allow the example to run to completion on a machine with CPU_COUNT=128. --- examples/run_og_usa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_og_usa.py b/examples/run_og_usa.py index 1a56e9af..5343db51 100644 --- a/examples/run_og_usa.py +++ b/examples/run_og_usa.py @@ -14,8 +14,8 @@ def main(): # Define parameters to use for multiprocessing - client = Client() num_workers = min(multiprocessing.cpu_count(), 7) + client = Client(n_workers=num_workers) print("Number of workers = ", num_workers) # Directories to save data