From 73ccfaaad5f9b1e98c1570ca1f56f84fa21370e6 Mon Sep 17 00:00:00 2001 From: Guillermo Lara Date: Tue, 9 Jul 2024 18:01:34 +0000 Subject: [PATCH 1/5] Add Urania machine --- support/Machines/Urania.yaml | 11 +++++++ support/SubmitScripts/Urania.sh | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 support/Machines/Urania.yaml create mode 100644 support/SubmitScripts/Urania.sh diff --git a/support/Machines/Urania.yaml b/support/Machines/Urania.yaml new file mode 100644 index 000000000000..786d27fcf930 --- /dev/null +++ b/support/Machines/Urania.yaml @@ -0,0 +1,11 @@ +# Distributed under the MIT License. +# See LICENSE.txt for details. + +Machine: + Name: Urania + Description: | + Supercomputer at the Max Planck Computing Data Facilty. + DefaultProcsPerNode: 72 + DefaultQueue: "p.urania" + DefaultTimeLimit: "1-00:00:00" + LaunchCommandSingleNode: ["srun", "-n", "1"] diff --git a/support/SubmitScripts/Urania.sh b/support/SubmitScripts/Urania.sh new file mode 100644 index 000000000000..974ad0056128 --- /dev/null +++ b/support/SubmitScripts/Urania.sh @@ -0,0 +1,56 @@ +{% extends "SubmitTemplateBase.sh" %} + +# Distributed under the MIT License. +# See LICENSE.txt for details. + +# Supercomputer at the Max Planck Computing Data Facility. +# More information: +# https://docs.mpcdf.mpg.de/doc/computing/clusters/systems/Gravitational_Physics_ACR.html + +{% block head %} +{{ super() -}} +#SBATCH -D ./ +#SBATCH --nodes {{ num_nodes | default(1) }} +#SBATCH --ntasks-per-node=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --cpus-per-task=72 +#SBATCH --mem=240000 +#SBATCH -t {{ time_limit | default("1-00:00:00") }} +#SBATCH -p {{ queue | default("p.debug") }} +{% endblock %} + +{% block charm_ppn %} +# One thread for communication +CHARM_PPN=$(expr ${SLURM_CPUS_PER_TASK} - 2) +{% endblock %} + +{% block list_modules %} +# Load compiler and MPI modules with explicit version specifications, +# consistently with the versions used to build the executable. +module purge +module load gcc/11 +module load impi/2021.7 +module load boost/1.79 +module load gsl/1.16 +module load cmake/3.26 +module load hdf5-serial/1.12.2 +module load anaconda/3/2021.11 + +# Load Spack environment +source /u/guilara/repos/spack/share/spack/setup-env.sh +spack env activate env3_spectre_impi + +# Define Charm paths +export CHARM_ROOT=/u/guilara/charm_impi_2/mpi-linux-x86_64-smp +export PATH=$PATH:/u/guilara/charm_impi_2/mpi-linux-x86_64-smp/bin + +# Load python environment +source $SPECTRE_HOME/env/bin/activate +{% endblock %} + +{% block run_command %} +srun -n ${SLURM_NTASKS} ${SPECTRE_EXECUTABLE} \ + --input-file ${SPECTRE_INPUT_FILE} \ + ++ppn ${CHARM_PPN} +pemap 0-34,36-70 +commap 35,71 \ + ${SPECTRE_CHECKPOINT:+ +restart "${SPECTRE_CHECKPOINT}"} +{% endblock %} From fd60a334db6e017598fa86f1ea48dce9ba4d3147 Mon Sep 17 00:00:00 2001 From: Guillermo Lara Date: Tue, 30 Jul 2024 08:46:22 +0000 Subject: [PATCH 2/5] Add no-validate flag to scheduler --- support/Python/Schedule.py | 16 ++++++++++++---- tests/support/Python/Test_Schedule.py | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/support/Python/Schedule.py b/support/Python/Schedule.py index ab3b555420d7..4f78f3e27e7e 100644 --- a/support/Python/Schedule.py +++ b/support/Python/Schedule.py @@ -149,6 +149,7 @@ def schedule( submit: Optional[bool] = None, clean_output: bool = False, force: bool = False, + no_validate=False, extra_params: dict = {}, **kwargs, ) -> Optional[subprocess.CompletedProcess]: @@ -548,10 +549,12 @@ def schedule( force=force, ) - # Validate input file - validate_input_file( - input_file_path.resolve(), executable=executable, work_dir=run_dir - ) + if not no_validate: + # Validate input file + validate_input_file( + input_file_path.resolve(), executable=executable, work_dir=run_dir + ) + # - If the input file may request resubmissions, make sure we have a # segments directory metadata, input_file = yaml.safe_load_all(rendered_input_file) @@ -861,6 +864,11 @@ def scheduler_options(f): "You may also want to use '--clean-output'." ), ) + @click.option( + "--no-validate", + is_flag=True, + help="Skip validation of the input file.", + ) # Scheduling options @click.option( "--scheduler", diff --git a/tests/support/Python/Test_Schedule.py b/tests/support/Python/Test_Schedule.py index 4db7c57c7c32..154d1935305c 100644 --- a/tests/support/Python/Test_Schedule.py +++ b/tests/support/Python/Test_Schedule.py @@ -230,6 +230,7 @@ def test_schedule(self): extra_option="TestOpt", metadata_option="MetaOpt", force=False, + no_validate=False, input_file="InputFile.yaml", input_file_name="InputFile.yaml", input_file_template=str(self.test_dir / "InputFile.yaml"), From a94ce0fef5201b28f46601f1bbdd608a1c66cb05 Mon Sep 17 00:00:00 2001 From: Guillermo Lara Date: Wed, 31 Jul 2024 13:34:39 +0000 Subject: [PATCH 3/5] Change name of option --- support/Python/Schedule.py | 12 +++++++----- tests/support/Python/Test_Schedule.py | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/support/Python/Schedule.py b/support/Python/Schedule.py index 4f78f3e27e7e..f8654a8088a1 100644 --- a/support/Python/Schedule.py +++ b/support/Python/Schedule.py @@ -149,7 +149,7 @@ def schedule( submit: Optional[bool] = None, clean_output: bool = False, force: bool = False, - no_validate=False, + validate: Optional[bool] = None, extra_params: dict = {}, **kwargs, ) -> Optional[subprocess.CompletedProcess]: @@ -320,6 +320,8 @@ def schedule( files in the 'run_dir' before scheduling the run. (Default: 'False') force: Optional. When 'True', overwrite input file and submit script in the 'run_dir' instead of raising an error when they already exist. + validate: Optional. When 'True', validate that the input file is parsed + correctly. When 'False' skip this step. extra_params: Optional. Dictionary of extra parameters passed to input file and submit script templates. Parameters can also be passed as keyword arguments to this function instead. @@ -549,7 +551,7 @@ def schedule( force=force, ) - if not no_validate: + if validate or validate is None: # Validate input file validate_input_file( input_file_path.resolve(), executable=executable, work_dir=run_dir @@ -865,9 +867,9 @@ def scheduler_options(f): ), ) @click.option( - "--no-validate", - is_flag=True, - help="Skip validation of the input file.", + "--validate/--no-validate", + default=None, + help="Validate or skip the validation of the input file.", ) # Scheduling options @click.option( diff --git a/tests/support/Python/Test_Schedule.py b/tests/support/Python/Test_Schedule.py index 154d1935305c..4db7c57c7c32 100644 --- a/tests/support/Python/Test_Schedule.py +++ b/tests/support/Python/Test_Schedule.py @@ -230,7 +230,6 @@ def test_schedule(self): extra_option="TestOpt", metadata_option="MetaOpt", force=False, - no_validate=False, input_file="InputFile.yaml", input_file_name="InputFile.yaml", input_file_template=str(self.test_dir / "InputFile.yaml"), From ba2c34a724037f837504ebefe6022060a370390e Mon Sep 17 00:00:00 2001 From: Guillermo Lara Date: Wed, 31 Jul 2024 13:57:04 +0000 Subject: [PATCH 4/5] Add environment file --- support/Environments/urania.sh | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 support/Environments/urania.sh diff --git a/support/Environments/urania.sh b/support/Environments/urania.sh new file mode 100644 index 000000000000..80a2d9536480 --- /dev/null +++ b/support/Environments/urania.sh @@ -0,0 +1,69 @@ +#!/bin/env sh + +# Distributed under the MIT License. +# See LICENSE.txt for details. + +spectre_setup_charm_paths() { + # Define Charm paths + export CHARM_ROOT=/u/guilara/charm_impi_2/mpi-linux-x86_64-smp + export PATH=$PATH:/u/guilara/charm_impi_2/mpi-linux-x86_64-smp/bin +} + +spectre_load_modules() { + module load gcc/11 + module load impi/2021.7 + module load boost/1.79 + module load gsl/1.16 + module load cmake/3.26 + module load hdf5-serial/1.12.2 + module load anaconda/3/2021.11 + module load paraview/5.10 + # Load Spack environment + source /u/guilara/repos/spack/share/spack/setup-env.sh + spack env activate env3_spectre_impi + # Load python environment + source $SPECTRE_HOME/env/bin/activate +} + +spectre_unload_modules() { + module unload gcc/11 + module unload impi/2021.7 + module unload boost/1.79 + module unload gsl/1.16 + module unload cmake/3.26 + module unload hdf5-serial/1.12.2 + module unload anaconda/3/2021.11 + module unload paraview/5.10 + # Unload Spack environment + spack env deactivate + # Unload python environment + deactivate +} + +spectre_run_cmake() { + if [ -z ${SPECTRE_HOME} ]; then + echo "You must set SPECTRE_HOME to the cloned SpECTRE directory" + return 1 + fi + spectre_load_modules + + cmake -D CMAKE_C_COMPILER=gcc \ + -D CMAKE_CXX_COMPILER=g++ \ + -D CMAKE_Fortran_COMPILER=gfortran \ + -D CHARM_ROOT=$CHARM_ROOT \ + -D CMAKE_BUILD_TYPE=Release \ + -D DEBUG_SYMBOLS=OFF \ + -D BUILD_SHARED_LIBS=ON \ + -D MEMORY_ALLOCATOR=JEMALLOC \ + -D BUILD_PYTHON_BINDINGS=ON \ + -D MACHINE=Urania \ + -D Python_EXECUTABLE=${SPECTRE_HOME}/env/bin/python \ + -D Catch2_DIR=/u/guilara/repos/Catch2/install_dir/lib64/cmake/Catch2 \ + -D MPI_C_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpigcc \ + -D MPI_CXX_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpig++ \ + -D MPI_Fortran_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ +/impi/gcc_11-11.2.0/2021.7.1/bin/mpigfortran \ + -Wno-dev "$@" $SPECTRE_HOME +} From 1438c076821046f5cdcc700b3a8579e0946a452c Mon Sep 17 00:00:00 2001 From: Guillermo Lara Date: Wed, 31 Jul 2024 13:26:48 +0000 Subject: [PATCH 5/5] Clean script Fix validate flag Fix --- support/Environments/urania.sh | 8 +++---- support/Python/Schedule.py | 8 +++---- support/SubmitScripts/Urania.sh | 31 ++++++--------------------- tests/support/Python/Test_Schedule.py | 1 + 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/support/Environments/urania.sh b/support/Environments/urania.sh index 80a2d9536480..1544ca40ee21 100644 --- a/support/Environments/urania.sh +++ b/support/Environments/urania.sh @@ -20,9 +20,10 @@ spectre_load_modules() { module load paraview/5.10 # Load Spack environment source /u/guilara/repos/spack/share/spack/setup-env.sh - spack env activate env3_spectre_impi + source /urania/u/guilara/repos/spack/var/spack/environments\ +/env3_spectre_impi/loads # Load python environment - source $SPECTRE_HOME/env/bin/activate + source /u/guilara/envs/spectre_env } spectre_unload_modules() { @@ -57,8 +58,7 @@ spectre_run_cmake() { -D MEMORY_ALLOCATOR=JEMALLOC \ -D BUILD_PYTHON_BINDINGS=ON \ -D MACHINE=Urania \ - -D Python_EXECUTABLE=${SPECTRE_HOME}/env/bin/python \ - -D Catch2_DIR=/u/guilara/repos/Catch2/install_dir/lib64/cmake/Catch2 \ + -D Catch2_DIR=/u/guilara/repos/Catch2/install_dir/lib64/cmake/Catch2 -D MPI_C_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ /impi/gcc_11-11.2.0/2021.7.1/bin/mpigcc \ -D MPI_CXX_COMPILER=/mpcdf/soft/SLE_15/packages/skylake\ diff --git a/support/Python/Schedule.py b/support/Python/Schedule.py index f8654a8088a1..e5be7775e65f 100644 --- a/support/Python/Schedule.py +++ b/support/Python/Schedule.py @@ -149,7 +149,7 @@ def schedule( submit: Optional[bool] = None, clean_output: bool = False, force: bool = False, - validate: Optional[bool] = None, + validate: Optional[bool] = True, extra_params: dict = {}, **kwargs, ) -> Optional[subprocess.CompletedProcess]: @@ -551,8 +551,8 @@ def schedule( force=force, ) - if validate or validate is None: - # Validate input file + # Validate input file + if validate: validate_input_file( input_file_path.resolve(), executable=executable, work_dir=run_dir ) @@ -868,7 +868,7 @@ def scheduler_options(f): ) @click.option( "--validate/--no-validate", - default=None, + default=True, help="Validate or skip the validation of the input file.", ) # Scheduling options diff --git a/support/SubmitScripts/Urania.sh b/support/SubmitScripts/Urania.sh index 974ad0056128..c6ee572518b2 100644 --- a/support/SubmitScripts/Urania.sh +++ b/support/SubmitScripts/Urania.sh @@ -3,49 +3,32 @@ # Distributed under the MIT License. # See LICENSE.txt for details. -# Supercomputer at the Max Planck Computing Data Facility. -# More information: +# Uranina -- HPC cluster of ACR division of MPI for Grav Physics, housed at the +# Max Planck Computing & Data Facility. # https://docs.mpcdf.mpg.de/doc/computing/clusters/systems/Gravitational_Physics_ACR.html {% block head %} {{ super() -}} -#SBATCH -D ./ #SBATCH --nodes {{ num_nodes | default(1) }} #SBATCH --ntasks-per-node=1 #SBATCH --ntasks-per-core=1 #SBATCH --cpus-per-task=72 -#SBATCH --mem=240000 #SBATCH -t {{ time_limit | default("1-00:00:00") }} -#SBATCH -p {{ queue | default("p.debug") }} +#SBATCH -p {{ queue | default("p.urania") }} {% endblock %} {% block charm_ppn %} -# One thread for communication +# Two thread for communication CHARM_PPN=$(expr ${SLURM_CPUS_PER_TASK} - 2) {% endblock %} {% block list_modules %} # Load compiler and MPI modules with explicit version specifications, # consistently with the versions used to build the executable. -module purge -module load gcc/11 -module load impi/2021.7 -module load boost/1.79 -module load gsl/1.16 -module load cmake/3.26 -module load hdf5-serial/1.12.2 -module load anaconda/3/2021.11 +source ${SPECTRE_HOME}/support/Environments/urania.sh +spectre_load_modules +spectre_setup_charm_paths -# Load Spack environment -source /u/guilara/repos/spack/share/spack/setup-env.sh -spack env activate env3_spectre_impi - -# Define Charm paths -export CHARM_ROOT=/u/guilara/charm_impi_2/mpi-linux-x86_64-smp -export PATH=$PATH:/u/guilara/charm_impi_2/mpi-linux-x86_64-smp/bin - -# Load python environment -source $SPECTRE_HOME/env/bin/activate {% endblock %} {% block run_command %} diff --git a/tests/support/Python/Test_Schedule.py b/tests/support/Python/Test_Schedule.py index 4db7c57c7c32..42b50942af1f 100644 --- a/tests/support/Python/Test_Schedule.py +++ b/tests/support/Python/Test_Schedule.py @@ -230,6 +230,7 @@ def test_schedule(self): extra_option="TestOpt", metadata_option="MetaOpt", force=False, + validate=True, input_file="InputFile.yaml", input_file_name="InputFile.yaml", input_file_template=str(self.test_dir / "InputFile.yaml"),