From c6ff1e2a037d40b4b1bd5367e3aae165a6cf4e02 Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Tue, 6 Feb 2024 15:14:51 -0800 Subject: [PATCH 01/10] Updating Release notes for patch; Editing Release versions. --- RELEASE_NOTES.md | 6 ++++-- cmake/SpheralVersion.cmake | 2 +- docs/conf.py | 4 ++-- docs/conf.py.in | 4 ++-- docs/developer/dev/continuous_deployment.rst | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0ec541b63..a33e0d8d1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,7 @@ -Version vX.Y.Z -- Release date xxxx-xx-xx +Version v2024.01.1 -- Release date 2024-02-xx ============================================== * Important Notes: - * + * This is a patch release for v2024.01.0. Notable changes include: @@ -15,6 +15,8 @@ Notable changes include: * Bug Fixes / improvements: * Fixed bug with ConstantBoundary in the presence of porosity with the new porosity models introduced in v2024.01.00. + * Updating header lists for including Spheral modules in external projects. + * Adding effective viscous pressure back to FSISPH. Version v2024.01.00 -- Release date 2024-01-19 ============================================== diff --git a/cmake/SpheralVersion.cmake b/cmake/SpheralVersion.cmake index 651fab3b2..0e62b972d 100644 --- a/cmake/SpheralVersion.cmake +++ b/cmake/SpheralVersion.cmake @@ -1 +1 @@ -set(SPHERAL_VERSION 2024.01.0) +set(SPHERAL_VERSION 2024.01.1) diff --git a/docs/conf.py b/docs/conf.py index 38bbfd963..c6b2d7ed4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,9 +26,9 @@ author = 'J. Michael Owen' # The short X.Y version -version = '2024.01.0' +version = '2024.01.1' # The full version, including alpha/beta/rc tags -release = '2024.01.0' +release = '2024.01.1' # -- General configuration --------------------------------------------------- diff --git a/docs/conf.py.in b/docs/conf.py.in index 9b2bc4db7..5c6b055e1 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -26,9 +26,9 @@ copyright = '2012, LLNS' author = 'J. Michael Owen' # The short X.Y version -version = '2024.01.0' +version = '2024.01.1' # The full version, including alpha/beta/rc tags -release = '2024.01.0' +release = '2024.01.1' # -- General configuration --------------------------------------------------- diff --git a/docs/developer/dev/continuous_deployment.rst b/docs/developer/dev/continuous_deployment.rst index 7f03ccfbe..dca78ad2d 100644 --- a/docs/developer/dev/continuous_deployment.rst +++ b/docs/developer/dev/continuous_deployment.rst @@ -9,7 +9,7 @@ to their environment. On LC systems we maintain: * ``Spheral/risky`` - * ``Spheral/2024.01.0`` + * ``Spheral/2024.01.1`` * ``Spheral/2023.06.0`` * ``Spheral/2023.03.1`` * ``Spheral/2023.03.0`` (deprecated as of TOSS4 upgrade) From c0697c6b72483ead53de526987f025f2f108e4ba Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Mon, 25 Mar 2024 14:54:30 -0700 Subject: [PATCH 02/10] Moving Spheral python install into lib/pythonX.Y/site-packages dir. --- cmake/InstallTPLs.cmake | 1 + cmake/spheral/SpheralAddLibs.cmake | 2 +- cmake/spheral/SpheralInstallPythonFiles.cmake | 5 +++-- scripts/spheral-setup-venv.in | 10 +++++----- src/Distributed/CMakeLists.txt | 4 ++-- src/PYB11/CMakeLists.txt | 7 ++----- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index f556243ff..05901241f 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -19,6 +19,7 @@ if (NOT ENABLE_CXXONLY) # Find the appropriate Python find_package(Python3 COMPONENTS Interpreter Development) set(PYTHON_EXE ${Python3_EXECUTABLE}) + set(SPHERAL_SITE_PACKAGES_PATH "lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages" ) list(APPEND SPHERAL_BLT_DEPENDS Python3::Python) # Set the PYB11Generator path diff --git a/cmake/spheral/SpheralAddLibs.cmake b/cmake/spheral/SpheralAddLibs.cmake index becce9d52..925957a3c 100644 --- a/cmake/spheral/SpheralAddLibs.cmake +++ b/cmake/spheral/SpheralAddLibs.cmake @@ -233,7 +233,7 @@ function(spheral_add_pybind11_library package_name) target_compile_options(${MODULE_NAME} PRIVATE ${SPHERAL_PYB11_TARGET_FLAGS}) install(TARGETS ${MODULE_NAME} - DESTINATION Spheral + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral ) # Set the r-path of the C++ lib such that it is independent of the build dir when installed diff --git a/cmake/spheral/SpheralInstallPythonFiles.cmake b/cmake/spheral/SpheralInstallPythonFiles.cmake index 78f9081b8..f5cf96408 100644 --- a/cmake/spheral/SpheralInstallPythonFiles.cmake +++ b/cmake/spheral/SpheralInstallPythonFiles.cmake @@ -7,13 +7,14 @@ # Note, if ENABLE_CXXONLY is set, this function does nothing #----------------------------------------------------------------------------------- + function(spheral_install_python_files) if (NOT ENABLE_CXXONLY) install(FILES ${ARGV} - DESTINATION Spheral) + DESTINATION DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral) install(CODE "execute_process( \ - COMMAND ${PYTHON_EXE} -m compileall Spheral \ + COMMAND ${PYTHON_EXE} -m compileall DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral \ WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX})") endif() diff --git a/scripts/spheral-setup-venv.in b/scripts/spheral-setup-venv.in index 38c86a73b..05363f62f 100644 --- a/scripts/spheral-setup-venv.in +++ b/scripts/spheral-setup-venv.in @@ -6,10 +6,10 @@ echo "Installing runtime python libraries ..." cp -r @VIRTUALENV_PYTHONPATH_COPY@ .venv/ &> /dev/null echo "Setup Spheral libraries ..." -cp Spheral.pth .venv/lib/python@Python3_VERSION_MAJOR@.@Python3_VERSION_MINOR@/site-packages/ -mkdir -p .venv/lib/python@Python3_VERSION_MAJOR@.@Python3_VERSION_MINOR@/site-packages/Spheral -cd @CMAKE_INSTALL_PREFIX@/.venv/lib/python@Python3_VERSION_MAJOR@.@Python3_VERSION_MINOR@/site-packages/Spheral -cp --symbolic-link @CMAKE_INSTALL_PREFIX@/Spheral/* . > /dev/null 2>&1 +cp @SPHERAL_SITE_PACKAGES_PATH@/Spheral.pth .venv/@SPHERAL_SITE_PACKAGES_PATH@/ +mkdir -p .venv/@SPHERAL_SITE_PACKAGES_PATH@/Spheral +cd @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@/Spheral +cp --symbolic-link @CMAKE_INSTALL_PREFIX@/@SPHERAL_SITE_PACKAGES_PATH@/Spheral/* . > /dev/null 2>&1 cd - > /dev/null # We need to reconfigure ATS to use our virtual env python otherwise ats will not be able to launch properly. @@ -28,6 +28,6 @@ cp --symbolic-link scripts/lcatstest.sh spheral-lcatstest &> /dev/null cd - > /dev/null echo "Byte-compiling packages in install path ..." -@CMAKE_INSTALL_PREFIX@/spheral -m compileall @CMAKE_INSTALL_PREFIX@/.venv/lib/python@Python3_VERSION_MAJOR@.@Python3_VERSION_MINOR@ +@CMAKE_INSTALL_PREFIX@/spheral -m compileall @CMAKE_INSTALL_PREFIX@/.venv/@SPHERAL_SITE_PACKAGES_PATH@ echo "Done." diff --git a/src/Distributed/CMakeLists.txt b/src/Distributed/CMakeLists.txt index 60f123fb5..74b2f6d48 100644 --- a/src/Distributed/CMakeLists.txt +++ b/src/Distributed/CMakeLists.txt @@ -66,7 +66,7 @@ if (ENABLE_MPI) ) install(FILES mpi_mpi4py.py - DESTINATION Spheral + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral RENAME mpi.py ) @@ -77,7 +77,7 @@ else() # MPI parallel off #---------------------------------------------------------------------------- install(FILES fakempi.py - DESTINATION Spheral + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral RENAME mpi.py ) diff --git a/src/PYB11/CMakeLists.txt b/src/PYB11/CMakeLists.txt index 62349e47d..0bd7616c5 100644 --- a/src/PYB11/CMakeLists.txt +++ b/src/PYB11/CMakeLists.txt @@ -76,10 +76,7 @@ set(PYB11_python_targets ${CMAKE_CURRENT_BINARY_DIR}/SpheralCompiledPackages.py ) -install(FILES ${PYB11_python_targets} - DESTINATION Spheral - ) +spheral_install_python_files(${PYB11_python_targets}) install(FILES Spheral.pth - DESTINATION . - ) + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}) From ede7a75193f6b2447cb16da19ee02e0c2a5bc6c4 Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Mon, 25 Mar 2024 15:50:34 -0700 Subject: [PATCH 03/10] Adding documentation on how to use Docker Dev environments. --- docs/developer/dev/docker_dev_env.rst | 74 +++++++++++++++++++++++++++ docs/developer/development_docs.rst | 1 + 2 files changed, 75 insertions(+) create mode 100644 docs/developer/dev/docker_dev_env.rst diff --git a/docs/developer/dev/docker_dev_env.rst b/docs/developer/dev/docker_dev_env.rst new file mode 100644 index 000000000..90fac3d45 --- /dev/null +++ b/docs/developer/dev/docker_dev_env.rst @@ -0,0 +1,74 @@ +******************************************* +Docker Development Environment +******************************************* + +Spheral builds an up-to-date docker container for every merge-request +on ``develop``. Developers can use this container to do development tasks +on local machines. + +=========================== +Creating a Dev Environment +=========================== + +We will use ``docker dev create`` with our spheral docker image and a +local repository. This will allow us to skip setting up a linux system with +external packages, gives us pre-built TPLs and allows us to edit a cloned +repository from our local machines IDE/text editor.bash:: + + > docker dev create --base-image ghcr.io/llnl/spheral + +This will generate a docker environment with a random name. A name can be +specified with the ``--name`` option. + +**Output** :: + + spheral-recursing_darwin <---- Name of dev environment + Creating Dev Environment "spheral-recursing_darwin" + populating volume from /Users/davis291/Projects/spheral + Creating Dev Environment "spheral-recursing_darwin" + detecting language + Detecting main repo language... + building compose stack + building compose stack + starting compose stack + starting compose stack + Network spheral-recursing_darwin_default Creating + Network spheral-recursing_darwin_default Created + Container spheral-recursing_darwin-app-1 Creating + Container spheral-recursing_darwin-app-1 Created + Container spheral-recursing_darwin-app-1 Starting + Container spheral-recursing_darwin-app-1 Started <---- Name of running container to connect to. + Dev Environment "spheral-recursing_darwin" (5bd37219d27eb68a77ce6fd8fee05a533a52017d8dcc72430867e2471e428e58) is running!% + + +============================= +Connecting to a Dev Container +============================= + +Once the continaer has ben started you can connect directly through the terminal +with the **Container** name (**NOT** the **Dev Environment** name).:: + + > docker exec -it spheral-recursing_darwin-app-1 /bin/bash + root@671dab5d0b00:/home/spheral/workspace/build_docker-gcc/install# + +This drops you into the install location of the ``spheral@develop`` build from +github, this is a fully installed version of the latest ``develop`` spheral. + +.. tip:: + VSCode & Docker Desktop: + * Open **Docker Desktop** and navigate to the **Dev Environment** tab. + * Find the container name and select **OPEN IN VSCODE**. + + +============================= +Development Work +============================= + +Your local Spheral repo is mounted from your local filesystem. You can develop directly from your +IDE or text editor of choice. Then you can compile and run from within the container itself. + +- The local Spheral repository will be mounted in the container at ``/com.docker.devenvironments.code/``. + +- There already exists a full build and install of Spheral at ``develop`` in ``/home/spheral/workspace/build_docker-gcc/install``. + +- An updated host config file can be found at ``/home/spheral/wokspace/docker-gcc.cmake``. diff --git a/docs/developer/development_docs.rst b/docs/developer/development_docs.rst index 76aa90c6c..13401fb9f 100644 --- a/docs/developer/development_docs.rst +++ b/docs/developer/development_docs.rst @@ -9,6 +9,7 @@ Welcome to Spheral's developer documentation. This documentation is a work in pr dev/submodules.rst dev/diagnostic_tools.rst + dev/docker_dev_env.rst dev/release_process.rst dev/continuous_deployment.rst From 4c1d65ff9babe3f00a859edc052021724d52d1ab Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Tue, 26 Mar 2024 13:12:54 -0700 Subject: [PATCH 04/10] Using to determine OS when SYS_TYPE is not available. Changing x86_64 to ubuntu20.04; copying x86_64 to generic dir; Adding concretizer options for generic architecture targeting w/docker builds. --- scripts/devtools/tpl-manager.py | 31 ++++- .../spack/configs/generic/concretizer.yaml | 36 ++++++ .../configs/{x86_64 => generic}/packages.yaml | 11 -- .../configs/ubuntu20.04/concretizer.yaml | 36 ++++++ .../spack/configs/ubuntu20.04/packages.yaml | 113 ++++++++++++++++++ 5 files changed, 211 insertions(+), 16 deletions(-) create mode 100644 scripts/spack/configs/generic/concretizer.yaml rename scripts/spack/configs/{x86_64 => generic}/packages.yaml (91%) create mode 100644 scripts/spack/configs/ubuntu20.04/concretizer.yaml create mode 100644 scripts/spack/configs/ubuntu20.04/packages.yaml diff --git a/scripts/devtools/tpl-manager.py b/scripts/devtools/tpl-manager.py index 4ee543025..62735be7d 100755 --- a/scripts/devtools/tpl-manager.py +++ b/scripts/devtools/tpl-manager.py @@ -109,10 +109,6 @@ def build_deps(args): for s in spec_list: print("** SPEC : {0}".format(s)) - spack_config_dir_opt="" - if "SYS_TYPE" not in os.environ.keys(): - spack_config_dir_opt="--spack-config-dir={0}".format(os.path.join(project_dir, "scripts/spack/configs/x86_64")) - spack_upstream_opt="" if os.path.isdir(args.upstream_dir) and not args.no_upstream: spack_upstream_opt="--upstream {0}".format(args.upstream_dir) @@ -124,17 +120,42 @@ def build_deps(args): # We use uberenv to set up our spack instance with our respective package.yaml files # config.yaml and custom spack packages recipes. print("** Running uberenv...") + prefix_opt="--prefix=" + args.spheral_spack_dir uberenv_project_json_opt="--project-json={0}".format(uberenv_project_json) print("** Spheral Spack Dir : {0}".format(args.spheral_spack_dir)) - sexe("python3 {0} --setup-only {1} {2} {3} {4} {5}".format(uberenv_path, prefix_opt, uberenv_project_json_opt, spack_config_dir_opt, spack_upstream_opt, uberenv_spack_url_opt), echo=True) # We just want to use the spac instance directly to generate our TPLs, we don't want # to have the spack instance take over our environment. os.environ["SPACK_DISABLE_LOCAL_CONFIG"] = "1" spack_cmd=os.path.join(args.spheral_spack_dir, "spack/bin/spack") + spheral_config_dir="scripts/spack/configs/" + spack_config_dir_opt="" + if "SYS_TYPE" not in os.environ.keys(): + # We need to install spack without any configuration files so we can use + # spack arch to determine the OS of the system and later to use spack find + # for generating external package files on external systems. + if sexe("python3 {0} --setup-only {1} {2} {3} {4}".format(uberenv_path, prefix_opt, uberenv_project_json_opt, spack_upstream_opt, uberenv_spack_url_opt), echo=True): sys.exit(1) + + spack_arch_os = sexe("{0} arch -o".format(spack_cmd), ret_output=True)[1].strip() + print("INFO : Detected Operating System :{0}".format(spack_arch_os)) + + spheral_config_dir += spack_arch_os + + spack_config_dir_opt="--spack-config-dir={0}".format(os.path.join(project_dir, spheral_config_dir)) + else: + spheral_config_dir += os.environ["SYS_TYPE"] + + + # Setup spack w/ Uberenv and the appropriate external package/compiler configs. + if sexe("python3 {0} --setup-only {1} {2} {3} {4} {5}".format(uberenv_path, prefix_opt, uberenv_project_json_opt, spack_config_dir_opt, spack_upstream_opt, uberenv_spack_url_opt), echo=True): sys.exit(1) + + # Uberenv doesn't copy the concretizer.yaml options... + if os.path.exists(spheral_config_dir+"/concretizer.yaml"): + sexe("cp {0}/concretizer.yaml {1}".format(spheral_config_dir, os.path.join(args.spheral_spack_dir, "spack/etc/spack/defaults")), echo=True) + # Optionally add a parallel job number for spack builds if args.spack_jobs: spack_cmd += " --jobs={0}".format(args.spack_jobs) diff --git a/scripts/spack/configs/generic/concretizer.yaml b/scripts/spack/configs/generic/concretizer.yaml new file mode 100644 index 000000000..4e6375b59 --- /dev/null +++ b/scripts/spack/configs/generic/concretizer.yaml @@ -0,0 +1,36 @@ +# ------------------------------------------------------------------------- +# This is the default spack configuration file. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing +# `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`, +# or by adding a `concretizer:` section to an environment. +# ------------------------------------------------------------------------- +concretizer: + # Whether to consider installed packages or packages from buildcaches when + # concretizing specs. If `true`, we'll try to use as many installs/binaries + # as possible, rather than building. If `false`, we'll always give you a fresh + # concretization. + reuse: true + # Options that tune which targets are considered for concretization. The + # concretization process is very sensitive to the number targets, and the time + # needed to reach a solution increases noticeably with the number of targets + # considered. + targets: + # Determine whether we want to target specific or generic microarchitectures. + # An example of the first kind might be for instance "skylake" or "bulldozer", + # while generic microarchitectures are for instance "aarch64" or "x86_64_v4". + granularity: generic + # If "false" allow targets that are incompatible with the current host (for + # instance concretize with target "icelake" while running on "haswell"). + # If "true" only allow targets that are compatible with the host. + host_compatible: true + # When "true" concretize root specs of environments together, so that each unique + # package in an environment corresponds to one concrete spec. This ensures + # environments can always be activated. When "false" perform concretization separately + # on each root spec, allowing different versions and variants of the same package in + # an environment. + unify: true diff --git a/scripts/spack/configs/x86_64/packages.yaml b/scripts/spack/configs/generic/packages.yaml similarity index 91% rename from scripts/spack/configs/x86_64/packages.yaml rename to scripts/spack/configs/generic/packages.yaml index 3a3096541..c3b86ba4a 100644 --- a/scripts/spack/configs/x86_64/packages.yaml +++ b/scripts/spack/configs/generic/packages.yaml @@ -4,17 +4,6 @@ packages: - spec: mpich@3.3.2 prefix: /usr buildable: false - #openmpi: - # externals: - # - spec: openmpi@4.1.2 - # prefix: /usr/ - # buildable: false - - #cmake: - # externals: - # - spec: cmake@3.16.3 - # prefix: /usr - # buildable: false # ------ SYSTEM LIBS ------- ncurses: diff --git a/scripts/spack/configs/ubuntu20.04/concretizer.yaml b/scripts/spack/configs/ubuntu20.04/concretizer.yaml new file mode 100644 index 000000000..4e6375b59 --- /dev/null +++ b/scripts/spack/configs/ubuntu20.04/concretizer.yaml @@ -0,0 +1,36 @@ +# ------------------------------------------------------------------------- +# This is the default spack configuration file. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing +# `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`, +# or by adding a `concretizer:` section to an environment. +# ------------------------------------------------------------------------- +concretizer: + # Whether to consider installed packages or packages from buildcaches when + # concretizing specs. If `true`, we'll try to use as many installs/binaries + # as possible, rather than building. If `false`, we'll always give you a fresh + # concretization. + reuse: true + # Options that tune which targets are considered for concretization. The + # concretization process is very sensitive to the number targets, and the time + # needed to reach a solution increases noticeably with the number of targets + # considered. + targets: + # Determine whether we want to target specific or generic microarchitectures. + # An example of the first kind might be for instance "skylake" or "bulldozer", + # while generic microarchitectures are for instance "aarch64" or "x86_64_v4". + granularity: generic + # If "false" allow targets that are incompatible with the current host (for + # instance concretize with target "icelake" while running on "haswell"). + # If "true" only allow targets that are compatible with the host. + host_compatible: true + # When "true" concretize root specs of environments together, so that each unique + # package in an environment corresponds to one concrete spec. This ensures + # environments can always be activated. When "false" perform concretization separately + # on each root spec, allowing different versions and variants of the same package in + # an environment. + unify: true diff --git a/scripts/spack/configs/ubuntu20.04/packages.yaml b/scripts/spack/configs/ubuntu20.04/packages.yaml new file mode 100644 index 000000000..c3b86ba4a --- /dev/null +++ b/scripts/spack/configs/ubuntu20.04/packages.yaml @@ -0,0 +1,113 @@ +packages: + mpich: + externals: + - spec: mpich@3.3.2 + prefix: /usr + buildable: false + +# ------ SYSTEM LIBS ------- + ncurses: + externals: + - spec: ncurses@6.2 + prefix: /usr + buildable: false + readline: + externals: + - spec: readline@8.0 + prefix: /usr + buildable: false + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + buildable: false + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + buildable: false + bzip2: + externals: + - spec: bzip2@1.0.8 + prefix: /usr + buildable: false + expat: + externals: + - spec: expat@2.2.9 + prefix: /usr + buildable: false + gdbm: + externals: + - spec: gdbm@1.18.1 + prefix: /usr + buildable: false + gettext: + externals: + - spec: gettext@0.19.8.1 + prefix: /usr + buildable: false + libffi: + externals: + - spec: libffi@3.3 + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1 + prefix: /usr + buildable: false + ossp-uuid: + externals: + - spec: ossp-uuid@1.6.2 + prefix: /usr + buildable: false + sqlite: + externals: + - spec: sqlite@2.8.17 + prefix: /usr + buildable: false + pkg-config: + externals: + - spec: pkg-config@0.29.1 + prefix: /usr + buildable: false + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + elfutils: + externals: + - spec: elfutils@0.176 + prefix: /usr + buildable: false + tcl: + externals: + - spec: tcl@8.6.9 + prefix: /usr + buildable: false + tk: + externals: + - spec: tk@8.6.9 + prefix: /usr + buildable: false + fontconfig: + externals: + - spec: fontconfig@2.13.1 + prefix: /usr + buildable: false + # netlib-lapack: + # externals: + # - spec: netlib-lapack@3.9.0 + # prefix: /usr + # buildable: false + liblapack64-dev: + externals: + - spec: liblapack64-dev@3.9.0 + prefix: /usr + buildable: false From c6ae5c3ee6d50b6d4bc9f46aea1c8a4da865e254 Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Wed, 27 Mar 2024 18:34:02 -0700 Subject: [PATCH 05/10] Adding documentation on editing packages.yaml files for external users. --- docs/build_guide/external/index.rst | 2 +- docs/build_guide/external/system_packages.rst | 12 +++++++ docs/build_guide/external/ubuntu_update.rst | 14 -------- docs/build_guide/include/quickstart.rst.inc | 9 ++++-- docs/build_guide/include/tpls.rst.inc | 32 +++++++++++++++++++ 5 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 docs/build_guide/external/system_packages.rst delete mode 100644 docs/build_guide/external/ubuntu_update.rst diff --git a/docs/build_guide/external/index.rst b/docs/build_guide/external/index.rst index 46640c394..ca2d65dc2 100644 --- a/docs/build_guide/external/index.rst +++ b/docs/build_guide/external/index.rst @@ -9,7 +9,7 @@ This guide is designed to help external users build and install Spheral on non L :caption: External Building Guide: quickstart.rst - ubuntu_update.rst + system_packages.rst cloning.rst updating.rst tpls.rst diff --git a/docs/build_guide/external/system_packages.rst b/docs/build_guide/external/system_packages.rst new file mode 100644 index 000000000..b812def28 --- /dev/null +++ b/docs/build_guide/external/system_packages.rst @@ -0,0 +1,12 @@ +.. _required system packages: + +Required System Packages +######################## + +This guide assumes the use of an Ubuntu 20.04 system using ``apt`` as the package manager. For other other operating systems please install the corresponding packages as seen below. + +:: + + sudo apt update + sudo apt upgrade + sudo apt install build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev liblapack-dev libbz2-dev locales python python3 unzip libtool wget curl tk-dev diff --git a/docs/build_guide/external/ubuntu_update.rst b/docs/build_guide/external/ubuntu_update.rst deleted file mode 100644 index af4ab14f2..000000000 --- a/docs/build_guide/external/ubuntu_update.rst +++ /dev/null @@ -1,14 +0,0 @@ -Updating Ubuntu -############### - -This guide assumes the use of an Ubuntu 20.04 system and using ``apt`` as the package manager. For other distrobutions please install the corresponding packages. - -.. note:: - Future steps (especially those detailed in :ref:`ex_tpl`) are assuming packages are installed under ``/usr/bin``, ``/usr/lib`` etc. - -:: - - sudo apt update - sudo apt upgrade - sudo apt install build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev liblapack-dev libbz2-dev locales python python3 unzip libtool wget curl tk-dev - diff --git a/docs/build_guide/include/quickstart.rst.inc b/docs/build_guide/include/quickstart.rst.inc index 44b1b1fba..e4b6e01b6 100644 --- a/docs/build_guide/include/quickstart.rst.inc +++ b/docs/build_guide/include/quickstart.rst.inc @@ -12,9 +12,6 @@ Update and install necessary package dependencies. sudo apt upgrade sudo apt install build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev liblapack-dev libbz2-dev locales python python3 unzip libtool wget curl tk-dev -.. warning:: - For alternative Linux distros your mileage may vary, ensure you are installing compatible packages to the ones listed above. - [ex_update_sys-section-end] .. @@ -47,6 +44,12 @@ Build our TPL dependencies from source with the Spheral tpl-management tool (``t .. note:: This command will generate a ``.cmake`` file with the naming convention ``-``. The following commands will refer to this format as ```` for generalization across operating systems and architectures. You will need to substitute the correct format in the following commands. +.. warning:: + For operating systems other than Ubuntu 20.04 you should get an error to the effect of: + ``[ERROR: invalid spack config dir: //scripts/spack/configs/ ]`` + You will **need** to follow the steps outlined in :ref:`ERROR: invalid spack config dir` before proceeding with this quickstart guide. + + [ex_tpl-section-end] [lc_tpl-section-start] diff --git a/docs/build_guide/include/tpls.rst.inc b/docs/build_guide/include/tpls.rst.inc index 909b507ab..3bf5d6de0 100644 --- a/docs/build_guide/include/tpls.rst.inc +++ b/docs/build_guide/include/tpls.rst.inc @@ -66,6 +66,38 @@ Above we are telling ``tpl-manager`` to build our TPLs with the ``gcc`` that is .. note:: Spheral minimally requires a C++14 compliant compiler. +ERROR: invalid spack config dir +=============================== + +If you are trying to run ``tpl-manager.py`` on an operating system other than Ubuntu20.04, you will +see an error to the effect of: +``[ERROR: invalid spack config dir: //scripts/spack/configs/ ]`` + +We define configuration files for Ubuntu20.04, as well as our common LLNL operating systems. +You will need to create a set of files for your own system. + +The configuration files tell spack where the packages installed in :ref:`Required System Packages` +are located and what version they are. We have provided a ``generic`` set of config files to +help in setting this up for you. + +#. Copy the directory ``scripts/spack/configs/generic`` to ``scripts/spack/config/`` (you want to match the name of the directory to the one ``tpl-manager.py`` expects to find). + + +#. For each package within the ``packages.py`` file of your new folder edit the version number to be the same as + what is installed on your system. There are a number of ways to retrieve versions for a given package: + + * Most version numbers should be searchable through your package manager, there are however, some system libraries that may not be managed by your package manager. + + * If the package has an executable, often you can run with some form of ``-V`` or ``--version``. e.g. for mpich:``mpiexec --version`` will report the MPI version. + + * For packages that only provide libraries, often the system library will be symlinked to one with the version as the extension. e.g. ``ls -lha /usr/lib/x86_64_gnu-linux/libreadlines.so`` will show it is symlinked to ``libreadlines.so.8.1``. + + +#. Each package requires the ``prefix:`` of the installation be provided. In most cases ``/usr`` is sufficient. Typically packages installed with a package manager will place files in: ``/usr/bin``, ``/usr/share``, ``/usr/lib``, ``/usr/lib64``, sometimes ``/usr/lib/x86_64.../``. Here the common prefix is ``/usr``. + + * If you are building Spheral on a system where you don't have permissions to run package manager and install to ``/usr``, then you might have installed the system packages somewhere else. In that case, replace the ``prefix:`` path for those given packages as necessary. + + [ex_running_tpl_manager-end] [lc_running_tpl_manager-start] From c6cd3296c8bdc4bdbfd05e35280f08af624380bb Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Wed, 27 Mar 2024 18:34:35 -0700 Subject: [PATCH 06/10] Adding generic config dir. --- scripts/spack/configs/generic/packages.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/spack/configs/generic/packages.yaml b/scripts/spack/configs/generic/packages.yaml index c3b86ba4a..3c10f211c 100644 --- a/scripts/spack/configs/generic/packages.yaml +++ b/scripts/spack/configs/generic/packages.yaml @@ -101,13 +101,8 @@ packages: - spec: fontconfig@2.13.1 prefix: /usr buildable: false - # netlib-lapack: - # externals: - # - spec: netlib-lapack@3.9.0 - # prefix: /usr - # buildable: false - liblapack64-dev: - externals: - - spec: liblapack64-dev@3.9.0 + netlib-lapack: + externals: + - spec: netlib-lapack@3.9.0 prefix: /usr buildable: false From a608bb92c82489b98a035437743ff433ddc916cb Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Wed, 27 Mar 2024 18:35:07 -0700 Subject: [PATCH 07/10] Guarding python file install for CXXONLY builds. --- src/Distributed/CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Distributed/CMakeLists.txt b/src/Distributed/CMakeLists.txt index 74b2f6d48..b7016cdcc 100644 --- a/src/Distributed/CMakeLists.txt +++ b/src/Distributed/CMakeLists.txt @@ -65,10 +65,12 @@ if (ENABLE_MPI) waitAllWithDeadlockDetection.hh ) - install(FILES mpi_mpi4py.py - DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral - RENAME mpi.py - ) + if (NOT ENABLE_CXXONLY) + install(FILES mpi_mpi4py.py + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral + RENAME mpi.py + ) + endif() spheral_add_obj_library(Distributed SPHERAL_OBJ_LIBS) @@ -76,10 +78,12 @@ else() #---------------------------------------------------------------------------- # MPI parallel off #---------------------------------------------------------------------------- - install(FILES fakempi.py - DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral - RENAME mpi.py - ) + if (NOT ENABLE_CXXONLY) + install(FILES fakempi.py + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral + RENAME mpi.py + ) + endif() endif() From e444144e0fe5087a7dc1bd1f82a5f9fd436ac58d Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Wed, 27 Mar 2024 18:39:51 -0700 Subject: [PATCH 08/10] Command fixes for using docker env for developing Spheral. --- docs/developer/dev/docker_dev_env.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/developer/dev/docker_dev_env.rst b/docs/developer/dev/docker_dev_env.rst index 90fac3d45..3618b4125 100644 --- a/docs/developer/dev/docker_dev_env.rst +++ b/docs/developer/dev/docker_dev_env.rst @@ -15,10 +15,11 @@ local repository. This will allow us to skip setting up a linux system with external packages, gives us pre-built TPLs and allows us to edit a cloned repository from our local machines IDE/text editor.bash:: - > docker dev create --base-image ghcr.io/llnl/spheral + > rm /compose-dev.yaml + > docker dev create --base-image ghcr.io/llnl/spheral --name --path -o -This will generate a docker environment with a random name. A name can be -specified with the ``--name`` option. +.. note:: + You need to have **Docker Desktop**, **VSCode** and the **VSCode Dev Environment Extension** installed for this to work. You do not need to use VSCode to access the container, but the extension seems to do some of the lifting for us when setting up the volume to our local repo. **Output** :: From 7ddd2f99726425613d87007f0a1d6298204993c3 Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Wed, 27 Mar 2024 22:42:06 -0700 Subject: [PATCH 09/10] Release notes; Adding --fail-fast option to spack install. --- RELEASE_NOTES.md | 14 ++++++++++++++ scripts/devtools/tpl-manager.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e15d1a167..f853ab37c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,17 @@ +Version vYYYY.MM.p -- Release date YYYY-MM-DD +============================================== + * Important Notes: + * External users of the code will need to supply config files for tpl-manager to find system libraries correctly. Steps to do this are detailed in the external user build guide. + +Notable changes include: + + * New features/ API changes: + + * Build changes / improvements: + * tpl-manager.py will no longer use generic x86_64 configs for non LC systems. Users will be required to supply their own configs for pointing spack at external packages. + + * Bug Fixes / improvements: + Version v2024.01.1 -- Release date 2024-02-17 ============================================== * Important Notes: diff --git a/scripts/devtools/tpl-manager.py b/scripts/devtools/tpl-manager.py index 62735be7d..33402efaf 100755 --- a/scripts/devtools/tpl-manager.py +++ b/scripts/devtools/tpl-manager.py @@ -175,7 +175,7 @@ def build_deps(args): os.environ["LC_ALL"] = "en_US.UTF-8" if not args.no_spec: if sexe("{0} spec --fresh -I {1}@develop%{2}".format(spack_cmd, package_name, s), echo=True) : sys.exit(1) - if sexe("{0} dev-build --fresh --quiet --deprecated -u initconfig {2}@develop%{3} 2>&1 | tee -a \"dev-build-{3}-out.txt\"".format(spack_cmd, os.getcwd(), package_name, s), echo=True) : sys.exit(1) + if sexe("{0} install --fail-fast --fresh --deprecated -u initconfig {2}@develop%{3} 2>&1 | tee -a \"dev-build-{3}-out.txt\"".format(spack_cmd, os.getcwd(), package_name, s), echo=True) : sys.exit(1) if not args.no_clean: sexe("rm dev-build-* spack-build-* spack-configure-args.txt") From 93fa90ebe73f6b0a12806739df42618bbba3efd4 Mon Sep 17 00:00:00 2001 From: mdavis36 Date: Thu, 28 Mar 2024 08:04:37 -0700 Subject: [PATCH 10/10] Split spack install into separate commands, one for tpls and one for generating our host-config file. --- scripts/devtools/tpl-manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/devtools/tpl-manager.py b/scripts/devtools/tpl-manager.py index 33402efaf..ccd06cbe3 100755 --- a/scripts/devtools/tpl-manager.py +++ b/scripts/devtools/tpl-manager.py @@ -170,12 +170,19 @@ def build_deps(args): # Loop through the specs we want TPLs for and build/install/get them as necessary. if not args.init_only: for s in spec_list: + print("** Building TPL's and generating host-config for {0}%{1} ...".format(package_name,s)) os.environ["SPEC"] = s os.environ["LC_ALL"] = "en_US.UTF-8" + if not args.no_spec: if sexe("{0} spec --fresh -I {1}@develop%{2}".format(spack_cmd, package_name, s), echo=True) : sys.exit(1) - if sexe("{0} install --fail-fast --fresh --deprecated -u initconfig {2}@develop%{3} 2>&1 | tee -a \"dev-build-{3}-out.txt\"".format(spack_cmd, os.getcwd(), package_name, s), echo=True) : sys.exit(1) + + # Install only the dependencies for Spheral + if sexe("{0} install --fail-fast --fresh --deprecated --only dependencies {2}@develop%{3} 2>&1 | tee -a \"dev-build-{3}-out.txt\"".format(spack_cmd, os.getcwd(), package_name, s), echo=True) : sys.exit(1) + + # Using dev-build we can have spack generate an init-config with the local source files for spheral. + if sexe("{0} dev-build --fresh --deprecated -u initconfig {2}@develop%{3} 2>&1 | tee -a \"dev-build-{3}-out.txt\"".format(spack_cmd, os.getcwd(), package_name, s), echo=True) : sys.exit(1) if not args.no_clean: sexe("rm dev-build-* spack-build-* spack-configure-args.txt")