diff --git a/config/templates/hpc_benchmark_config_template.yaml b/config/templates/hpc_benchmark_config_template.yaml index 7b227af4..9e180fb9 100644 --- a/config/templates/hpc_benchmark_config_template.yaml +++ b/config/templates/hpc_benchmark_config_template.yaml @@ -36,7 +36,7 @@ parameterset: - {name: tasks_per_node, type: int, _: "TASKS_PER_NODE"} # number of (MPI) tasks per node, accepts a list in the format a,b,c,... for generating multiple runs - {name: threads_per_task, type: int, _: "THREADS_PER_TASK"} # number of threads per task, accepts a list in the format a,b,c,... for generating multiple runs - {name: walltime, type: string, _: "WALLTIME"} # wall time for the job in format hh:mm:ss - - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --hint=nomultithread"} # processor affinity/pinning + - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --threads-per-core=1"} # processor affinity/pinning - name: software_parameters parameter: - {name: simulator, _: "nest-simulator"} # simulator to be installed (plan file must be provided) diff --git a/config/templates/microcircuit_config_template.yaml b/config/templates/microcircuit_config_template.yaml index 283ffb83..dff5d9db 100644 --- a/config/templates/microcircuit_config_template.yaml +++ b/config/templates/microcircuit_config_template.yaml @@ -41,7 +41,7 @@ parameterset: - {name: tasks_per_node, type: int, _: "TASKS_PER_NODE"} # number of (MPI) tasks per node, accepts a list in the format a,b,c,... for generating multiple runs - {name: threads_per_task, type: int, _: "THREADS_PER_TASK"} # number of threads per task, accepts a list in the format a,b,c,... for generating multiple runs - {name: walltime, type: string, _: "WALLTIME"} # wall time for the job in format hh:mm:ss - - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --hint=nomultithread"} # processor affinity/pinning + - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --threads-per-core=1"} # processor affinity/pinning - name: software_parameters parameter: - {name: simulator, _: "nest-simulator"} # simulator to be installed (plan file must be provided) diff --git a/config/templates/multi-area-model_2_config_template.yaml b/config/templates/multi-area-model_2_config_template.yaml index f9260325..f90d7613 100644 --- a/config/templates/multi-area-model_2_config_template.yaml +++ b/config/templates/multi-area-model_2_config_template.yaml @@ -37,7 +37,7 @@ parameterset: - {name: tasks_per_node, type: int, _: "TASKS_PER_NODE"} # number of (MPI) tasks per node, accepts a list in the format a,b,c,... for generating multiple runs - {name: threads_per_task, type: int, _: "THREADS_PER_TASK"} # number of threads per task, accepts a list in the format a,b,c,... for generating multiple runs - {name: walltime, type: string, _: "WALLTIME"} # wall time for the job in format hh:mm:ss - - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --hint=nomultithread"} # processor affinity/pinning + - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --threads-per-core=1"} # processor affinity/pinning - name: software_parameters parameter: - {name: simulator, _: "nest-simulator"} # simulator to be installed (plan file must be provided) diff --git a/config/templates/multi-area-model_3_config_template.yaml b/config/templates/multi-area-model_3_config_template.yaml index 4a926560..d2c68512 100644 --- a/config/templates/multi-area-model_3_config_template.yaml +++ b/config/templates/multi-area-model_3_config_template.yaml @@ -39,7 +39,7 @@ parameterset: - {name: tasks_per_node, type: int, _: "TASKS_PER_NODE"} # number of (MPI) tasks per node, accepts a list in the format a,b,c,... for generating multiple runs - {name: threads_per_task, type: int, _: "THREADS_PER_TASK"} # number of threads per task, accepts a list in the format a,b,c,... for generating multiple runs - {name: walltime, type: string, _: "WALLTIME"} # wall time for the job in format hh:mm:ss - - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --hint=nomultithread"} # processor affinity/pinning + - {name: affinity, type: string, separator: ";", _: "--cpu-bind=verbose,threads --distribution=block:cyclic:fcyclic --threads-per-core=1"} # processor affinity/pinning - name: software_parameters parameter: - {name: simulator, _: "nest-simulator"} # simulator to be installed (plan file must be provided) diff --git a/helpers/helpers.yaml b/helpers/helpers.yaml index 232b2b33..512f147a 100644 --- a/helpers/helpers.yaml +++ b/helpers/helpers.yaml @@ -25,7 +25,6 @@ substituteset: - {source: "#NODES#", dest: $num_nodes} - {source: "#NTASKS#", dest: $num_tasks} - {source: "#NTASKS_PER_NODE#", dest: $tasks_per_node} - - {source: "#CPUS_PER_TASK#", dest: $threads_per_task} - {source: "#TIME#", dest: $walltime} - {source: "#ERRPATH#", dest: $err_file} - {source: "#OUTPATH#", dest: $out_file} @@ -40,7 +39,6 @@ substituteset: - {source: "#NODES#", dest: 1} - {source: "#NTASKS#", dest: 1} - {source: "#NTASKS_PER_NODE#", dest: 1} - - {source: "#CPUS_PER_TASK#", dest: 16} - {source: "#TIME#", dest: "00:15:00"} - {source: "#ERRPATH#", dest: $err_file} - {source: "#OUTPATH#", dest: $out_file} @@ -82,8 +80,11 @@ parameterset: module load $simulator/$version/$variant_$suffix fi export OMP_NUM_THREADS=$threads_per_task + export OMP_DISPLAY_ENV=VERBOSE + export OMP_DISPLAY_AFFINITY=TRUE + export OMP_PROC_BIND=TRUE ${optional_run_command} - srun ${affinity} python ${run_file} ${run_args} + srun --cpus-per-task=${threads_per_task} ${affinity} python ${run_file} ${run_args} srun -n 1 --nodes 1 python ${base_path}/helpers/collect_timer_data.py ${log_path} srun -n 1 --nodes 1 python ${base_path}/helpers/cpu_logging.py ${jube_wp_abspath} cd ${model_path} diff --git a/helpers/job.slurm.in b/helpers/job.slurm.in index 60db555e..f9e49617 100755 --- a/helpers/job.slurm.in +++ b/helpers/job.slurm.in @@ -22,7 +22,6 @@ #SBATCH -e #ERRPATH# #SBATCH --nodes=#NODES# #SBATCH --ntasks-per-node=#NTASKS_PER_NODE# -#SBATCH --cpus-per-task=#CPUS_PER_TASK# #SBATCH --time=#TIME# #SBATCH --exclusive #SBATCH --account=#ACCOUNT#