Merge branch 'development'

AMReX-Astro · Jul 1, 2022 · 6801988 · 6801988
2 parents f724cd9 + d68821a
commit 6801988
Show file tree

Hide file tree

Showing 213 changed files with 18,298 additions and 6,079 deletions.
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -8,18 +8,18 @@ on:
 
 jobs:
   deploy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Install pandoc and doxygen 
+      - name: Install pandoc and doxygen
         run: |
           sudo apt install pandoc doxygen
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Upgrade pip
         run: |

diff --git a/.github/workflows/gpu_action.yml b/.github/workflows/gpu_action.yml
@@ -24,11 +24,12 @@ jobs:
           sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0 gfortran>=9.3.0 
 
       - name: Install CUDA
+        # from https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=deb_network
         run: |
-          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin
-          sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600
-          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
-          sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /"
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
+          sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
+          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
+          sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
           sudo apt-get update
           sudo apt-get -y install cuda
 
@@ -47,4 +48,4 @@ jobs:
       - name: Compile problems for GPU
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
-          python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -header-filter=Castro -ignore-files="amrex|Microphysics" -gpu
+          python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -header-filter=Castro -ignore-files="amrex|Microphysics" -gpu
diff --git a/.github/workflows/sedov-compare.yml b/.github/workflows/sedov-compare.yml
@@ -0,0 +1,39 @@
+name: Sedov
+
+on: [pull_request]
+jobs:
+  Sedov-3d:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0 gfortran>=9.3.0
+
+      - name: Compile Sedov
+        run: |
+          cd Exec/hydro_tests/Sedov
+          make DEBUG=TRUE USE_MPI=FALSE -j 4
+
+      - name: Run Sedov
+        run: |
+          cd Exec/hydro_tests/Sedov
+          ./Castro3d.gnu.DEBUG.ex inputs.3d.sph max_step=10 amr.max_level=2 amr.plot_files_output=0 amr.checkpoint_files_output=0
+
+      - name: Compare to stored output
+        run: |
+          cd Exec/hydro_tests/Sedov
+          diff grid_diag.out ci-benchmarks/grid_diag.out
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,78 @@
+# 22.06
+
+   * castro.stopping_criterion_field and castro.stopping_criterion_value have
+     been added; these allow you to stop the simulation once a certain threshold
+     has been exceeded (for example, if the temperature gets too hot). (#2209)
+
+   * The option castro.show_center_of_mass has been removed. If castro.v = 1
+     and castro.sum_interval > 0, then the center of mass will automatically
+     be included with the other diagnostic sums that are displayed. (#2176)
+
+   * The option castro.state_in_rotating_frame has been removed. The default
+     behavior continues to be that when rotation is being used, fluid variables
+     are measured with respect to the rotating frame. (#2172)
+
+   * The default for use_pslope has been changed to 0 -- disabling this.
+     use_pslope enables reconstruction that knows about HSE for the PLM
+     (castro.ppm_type = 0) implementation.  Since that method is not the
+     default, it is unlikely that this has been used.  This change is being
+     done to allow for a PPM implementation to be added without changing
+     the default behavior of that method. (#2205)
+
+   * The ``castro.use_pslope`` functionality to well-balance HSE has been
+     extended to PPM (#2202)
+
+# 22.05
+
+   * A new option castro.hydro_memory_footprint_ratio has been added which
+     can help limit the amount of memory used in GPU builds. (#2153)
+
+   * In #1379, for the 21.04 release, Castro added a check that issued
+     an abort if any species mass fraction was found to be invalid (defined
+     by being less than -0.01 or greater than 1.01). This helps us catch
+     unintended code errors that do not properly normalize updates to the
+     species. (This was originally only enabled for CPU builds, but in the
+     22.04 release was extended for GPU builds, as noted below.) However,
+     as observed in #2096, this issue can legitimately be triggered in
+     regions of sharp composition and density gradients as an unavoidable
+     consequence of how the multi-dimensional CTU solver is designed. An
+     example would be a helium shell around a C/O white dwarf at low to
+     moderate spatial resolution. This was causing the code to abort in
+     a couple of science problems, so several improvements were added to
+     the code in this release to address it. In #2121, we turned this
+     situation into a retry after a hydro update rather than an abort,
+     so that the code has more chances to recover by doing an advance
+     with a smaller timestep. However, this will not always allow you
+     to recover, particularly if you are in an area where density resets
+     are occurring and/or you are using castro.limit_fluxes_on_small_dens,
+     so we also added a new option castro.abundance_failure_rho_cutoff in
+     #2124, which allows you to set a density threshold below which invalid
+     mass fractions will be silently ignored (and reset to valid values
+     between 0 and 1). We also turned the invalid mass fraction threshold
+     into a runtime parameter castro.abundance_failure_tolerance (#2131),
+     so that you can optionally loosen or tighten the strictness of this
+     check.
+
+     Since this scenario was sometimes occurring during the reflux step
+     in AMR simulations, we also improved the reflux code to avoid doing
+     the flux correction locally in zones where it would cause an invalid
+     mass fraction (#2123).
+
+     While doing these changes we noticed also that the option
+     castro.limit_fluxes_on_small_dens was sometimes inadvertently
+     aggravating this problem by creating physically implausible fluxes of
+     the species, so we simplified the algorithm to avoid that. (#2134)
+
+   * The option castro.limit_fluxes_on_large_vel has been removed. (#2132)
+
+# 22.04
+
+   * We now abort on GPUs if species do not sum to 1 (#2099)
+
+   * Fixed an issue with monopole gravity where running with multiple
+     MPI ranks in a GPU build could result in an incorrect gravitational
+     field calculation. (#2091)
+
 # 22.02
 
    * Microphysics has moved from Starkiller-Astro to AMReX-Astro.  The

diff --git a/Docs/source/FlowChart.rst b/Docs/source/FlowChart.rst
@@ -63,7 +63,7 @@ The time-integration method used is controlled by
     described above that uses the CTU hydro advection and an ODE
     reaction solve.  Note: because this requires a different set of
     state variables, you must compile with ``USE_SIMPLIFIED_SDC = TRUE`` for this
-    method to work (in particular, this defines ``PRIM_SPECIES_HAVE_SOURCES``).
+    method to work.
 
 .. index:: USE_SIMPLIFIED_SDC, USE_TRUE_SDC
 
@@ -124,8 +124,6 @@ of each step.
    actions are performend (note, we omit the actions taken for a retry,
    which we will describe later):
 
-   -  Sync up the level information to the Fortran-side of Castro.
-
    -  Do any radiation initialization.
 
    -  Set the maximum density used for Poisson gravity tolerances.

diff --git a/Docs/source/Hydrodynamics.rst b/Docs/source/Hydrodynamics.rst
@@ -37,11 +37,11 @@ Some general notes:
 Hydrodynamics Data Structures
 =============================
 
-Within the Fortran routines that implement the hydrodynamics, there are
+Within the routines that implement the hydrodynamics, there are
 several main data structures that hold the state.
 
 -  conserved state: these arrays generally begin with ``u``,
-   e.g., ``uin``, ``uout``. The ``NVAR``
+   e.g., ``uin``, ``uout``. The ``NUM_STATE``
    components for the state data in the array are accessed using
    integer keys defined in :numref:`table:consints`.
 
@@ -199,29 +199,29 @@ several main data structures that hold the state.
       +-----------------------+-----------------------+-----------------------+
       | **variable**          | **quantity**          | **note**              |
       +=======================+=======================+=======================+
-      | ``QGDRHO``            | :math:`\rho`          |                       |
+      | ``GDRHO``             | :math:`\rho`          |                       |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDU``               | :math:`u`             |                       |
+      | ``GDU``               | :math:`u`             |                       |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDV``               | :math:`v`             |                       |
+      | ``GDV``               | :math:`v`             |                       |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDW``               | :math:`w`             |                       |
+      | ``GDW``               | :math:`w`             |                       |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDPRES``            | :math:`p`             | regardless of whether |
+      | ``GDPRES``            | :math:`p`             | regardless of whether |
       |                       |                       | ``RADIATION`` is      |
       |                       |                       | defined,              |
       |                       |                       | this is always just   |
       |                       |                       | the gas pressure      |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDLAMS``            | :math:`{\lambda_f}`   | the starting index    |
+      | ``GDLAMS``            | :math:`{\lambda_f}`   | the starting index    |
       |                       |                       | for the flux          |
       |                       |                       | limiter—there are     |
       |                       |                       | ngroups components    |
       |                       |                       | (defined only if      |
       |                       |                       | ``RADIATION`` is      |
       |                       |                       | defined)              |
       +-----------------------+-----------------------+-----------------------+
-      | ``QDERADS``           | :math:`E_r`           | the starting index    |
+      | ``GDERADS``           | :math:`E_r`           | the starting index    |
       |                       |                       | for the radiation     |
       |                       |                       | energy—there are      |
       |                       |                       | ngroups components    |
@@ -568,7 +568,7 @@ There are four major steps in the hydrodynamics update:
 
 #. Doing the conservative update
 
-.. index:: castro.do_hydro, castro.add_ext_src, castro.do_sponge, castro.normalize_species, castro.spherical_star, castro.show_center_of_mass
+.. index:: castro.do_hydro, castro.add_ext_src, castro.do_sponge, castro.normalize_species, castro.spherical_star
 
 Each of these steps has a variety of runtime parameters that
 affect their behavior. Additionally, there are some general
@@ -597,8 +597,6 @@ runtime parameters for hydrodynamics:
    function is then used to set the values outside the domain in
    implementing the boundary conditions.
 
--  ``castro.show_center_of_mass``: (0 or 1; default: 0)
-
 .. index:: castro.small_dens, castro.small_temp, castro.small_pres
 
 Several floors are imposed on the thermodynamic quantities to prevet unphysical

diff --git a/Docs/source/build_system.rst b/Docs/source/build_system.rst
@@ -45,17 +45,12 @@ Most of these are parameters from AMReX.
 Fortran Support
 ^^^^^^^^^^^^^^^
 
-Many problems can be built without Fortran.  The current exceptions
-are MHD, radiation, and anything using a pynucastro-generated network.
-These parameters control Fortran support:
-
-  * ``USE_FORT_MICROPHYSICS``: if set to ``TRUE``, then Fortran
-    versions of the EOS and burner interface will be compiled.  If you
-    are not using a pynucastro network, then you can probably set this
-    to ``FALSE``.
+Radiation currently needs Fortran support.  All of the other solvers
+and problem set ups do not require Fortran.  Fortran support in AMReX
+is enabled / disabled via:
 
   * ``BL_NO_FORT``: if set to ``TRUE``, then no AMReX Fortran source will be built.
-    This cannot currently be used for the MHD or radiation solvers.
+    This cannot currently be used for the radiation solver.
 
 
 Parallelization and GPUs
@@ -230,21 +225,20 @@ This is the current build system process.
 
 * ``set_variables.py`` is called
 
-  .. index:: set_variables.py, _variables, state_indices_nd.F90, state_indices.H
+  .. index:: set_variables.py, _variables, state_indices.H
 
   * This processes the Castro ``_variables`` file and writes
-    ``state_indices.H`` (and  ``state_indices_nd.F90`` if Fortran is enabled) into the
+    ``state_indices.H`` into the
     ``tmp_build_dir/castro_sources/`` directory.
 
     These are used to define the size of the various state arrays and
     the integer keys to index each state variable.
 
-  * The hook for this is in ``Make.auto_source`` in the build rule for ``state_indices_nd.F90``
+  * The hook for this is in ``Make.auto_source`` in the build rule for ``state_indices.H``
 
   * You can test this portion of the build system by doing ``make test_variables``
 
-* (for ``general_null networks``), ``network_properties.H`` (and
-  ``actual_network.F90`` if Fortran is enabled) is created
+* (for ``general_null networks``), ``network_properties.H`` is created
 
   .. index:: write_network.py
 
@@ -257,7 +251,7 @@ This is the current build system process.
   .. index:: write_probin.py
 
   * This writes the routines that manage the Microphysics runtime
-    parameters: ``extern_parameters.cpp``, ``extern_parameters.H``, and  ``extern.F90``.  This is output in
+    parameters: ``extern_parameters.cpp`` and  ``extern_parameters.H``.  This is output in
     ``tmp_build_dir/castro_sources/``.
 
   * The hook for this is in ``Make.auto_source`` in the rule for ``extern_parameters.H``
@@ -286,7 +280,7 @@ This is the current build system process.
 
   * These headers are output into ``tmp_build_dir/castro_sources/``.
 
-* The Fortran dependencies file is created
+* (if Fortran support is enabled) The Fortran dependencies file is created
 
   * This creates the ``f90.depends`` file in the ``tmp_build_dir``
 

diff --git a/Docs/source/creating_a_problem.rst b/Docs/source/creating_a_problem.rst
@@ -207,7 +207,7 @@ each of these in the main source tree.
    provides the C++ code that defines these new plot variables. It
    does this by adding them to the ``derive_lst``—a list of
    derived variables that Castro knows about. When adding new
-   variables, a descriptive name, Fortran routine that does the
+   variables, a descriptive name, a C++ routine that does the
    deriving, and component of ``StateData`` are specified.
 
    The other two files provide the header and implementation of the