EBs: Compiled by Default, Controlled at Runtime (#4865)

* Build Logic: Enable EBs by Default

* EB: Introduce Runtime Parameter

* Cleaning: Review Weiqun

* Cleaning: EB w/ Clang Tidy

* Formatting for Clarity

Only changes spaces

* CMake: Simplify `add_warpx_test`

EB is not mutually exclusive anymore: we can run non-EB tests
with EB compiled in.

* Azure CI: EB is default-ON now

.azure-pipelines.yml

@@ -36,10 +36,6 @@ jobs:
       cylindrical_rz:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON
         WARPX_RZ_FFT: 'TRUE'
-      # embedded boundaries
-      embedded_boundaries:
-        WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_EB=ON
-        WARPX_RZ_FFT: 'TRUE'
       # single precision
       #single_precision:
       #  WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE

.github/workflows/source/check_inputs.py

@@ -27,7 +27,7 @@
                     # skip lines related to other function arguments
                     # NOTE: update range call to reflect changes
                     #       in the interface of 'add_warpx_test'
-                    for _ in range(3):
+                    for _ in range(2):  # skip over: dims, numprocs
                         next(f)
                     # strip leading whitespaces, remove end-of-line comments
                     testinput = next(f).lstrip().split(" ")[0]

CMakeLists.txt

@@ -68,7 +68,7 @@ include(CMakeDependentOption)
 option(WarpX_APP           "Build the WarpX executable application"     ON)
 option(WarpX_ASCENT        "Ascent in situ diagnostics"                 OFF)
 option(WarpX_CATALYST      "Catalyst in situ diagnostics"               OFF)
-option(WarpX_EB            "Embedded boundary support"                  OFF)
+option(WarpX_EB            "Embedded boundary support"                  ON)
 option(WarpX_LIB           "Build WarpX as a library"                   OFF)
 option(WarpX_MPI           "Multi-node support (message-passing)"       ON)
 option(WarpX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                  ON)

Docs/source/developers/testing.rst

@@ -103,7 +103,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            test_1d_laser_acceleration  # name
            1  # dims
            2  # nprocs
-           OFF  # eb
            inputs_test_1d_laser_acceleration  # inputs
            analysis.py  # analysis
            diags/diag1000100  # output (plotfile)
@@ -118,7 +117,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            test_2d_laser_acceleration_picmi  # name
            2  # dims
            2  # nprocs
-           OFF  # eb
            inputs_test_2d_laser_acceleration_picmi.py  # inputs
            analysis.py  # analysis
            diags/diag1000100  # output (plotfile)
@@ -133,7 +131,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            test_3d_laser_acceleration_restart  # name
            3  # dims
            2  # nprocs
-           OFF  # eb
            inputs_test_3d_laser_acceleration_restart  # inputs
            analysis_default_restart.py  # analysis
            diags/diag1000100  # output (plotfile)
@@ -150,7 +147,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
            test_rz_laser_acceleration_picmi  # name
            RZ  # dims
            2   # nprocs
-           OFF  # eb
            "inputs_test_rz_laser_acceleration_picmi.py --test --dir 1"  # inputs
            analysis.py  # analysis
            diags/diag1/  # output (openPMD time series)

Docs/source/install/cmake.rst

@@ -88,7 +88,7 @@ CMake Option                  Default & Values                             Descr
 ``WarpX_CATALYST``            ON/**OFF**                                   Catalyst in situ visualization
 ``WarpX_COMPUTE``             NOACC/**OMP**/CUDA/SYCL/HIP                  On-node, accelerated computing backend
 ``WarpX_DIMS``                **3**/2/1/RZ                                 Simulation dimensionality. Use ``"1;2;RZ;3"`` for all.
-``WarpX_EB``                  ON/**OFF**                                   Embedded boundary support (not supported in RZ yet)
+``WarpX_EB``                  **ON**/OFF                                   Embedded boundary support (not supported in RZ yet)
 ``WarpX_IPO``                 ON/**OFF**                                   Compile WarpX with interprocedural optimization (aka LTO)
 ``WarpX_LIB``                 ON/**OFF**                                   Build WarpX as a library, e.g., for PICMI Python
 ``WarpX_MPI``                 **ON**/OFF                                   Multi-node support (message-passing)
@@ -269,7 +269,7 @@ Environment Variable          Default & Values                             Descr
 ============================= ============================================ ================================================================
 ``WARPX_COMPUTE``             NOACC/**OMP**/CUDA/SYCL/HIP                  On-node, accelerated computing backend
 ``WARPX_DIMS``                ``"1;2;3;RZ"``                               Simulation dimensionalities (semicolon-separated list)
-``WARPX_EB``                  ON/**OFF**                                   Embedded boundary support (not supported in RZ yet)
+``WARPX_EB``                  **ON**/OFF                                   Embedded boundary support (not supported in RZ yet)
 ``WARPX_MPI``                 ON/**OFF**                                   Multi-node support (message-passing)
 ``WARPX_OPENPMD``             **ON**/OFF                                   openPMD I/O (HDF5, ADIOS)
 ``WARPX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)

Docs/source/usage/parameters.rst

@@ -590,14 +590,15 @@ In WarpX, the embedded boundary can be defined in either of two ways:
             A function of `x`, `y`, `z` that defines the surface of the embedded
             boundary. That surface lies where the function value is 0 ;
             the physics simulation area is where the function value is negative ;
-            the interior of the embeddded boundary is where the function value is positive.
+            the interior of the embedded boundary is where the function value is positive.
 
     - **From an STL file:**
         In that case, you will need to set the following parameters in the input file.
 
         * ``eb2.stl_file`` (`string`)
-            The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``,
-            in order for the file to be read by WarpX.
+            The path to an `STL file <https://en.wikipedia.org/wiki/STL_(file_format)>`__.
+            In addition, you also need to set ``eb2.geom_type = stl``, in order for the file to be read by WarpX.
+            `See the AMReX documentation for more details <https://amrex-codes.github.io/amrex/docs_html/EB.html>`__.
 
 Whether the embedded boundary is defined with an analytical function or an STL file, you can
 additionally define the electric potential at the embedded boundary with an analytical function:

Examples/CMakeLists.txt

@@ -20,7 +20,6 @@ endif()
 # name: unique name of this test
 # dims: 1,2,RZ,3
 # nprocs: 1 or 2 (maybe refactor later on to just depend on WarpX_MPI)
-# eb: needs EB support? (temporary until handled as runtime parameter)
 # inputs: inputs file or PICMI script, WarpX_MPI decides w/ or w/o MPI
 # analysis: analysis script, always run without MPI
 # output: output file(s) to analyze
@@ -30,7 +29,6 @@ function(add_warpx_test
     name
     dims
     nprocs
-    eb
     inputs
     analysis
     output
@@ -42,17 +40,6 @@ function(add_warpx_test
         return()
     endif()
 
-    # cannot run EB tests w/o EB build
-    if(eb AND NOT WarpX_EB)
-        message(WARNING "${name}: cannot run EB tests without EB build")
-        return()
-    endif()
-
-    # do not run no-EB tests w/ EB build
-    if(NOT eb AND WarpX_EB)
-        return()
-    endif()
-
     # cannot run tests with unsupported geometry
     if(NOT dims IN_LIST WarpX_DIMS)
         return()

Examples/Physics_applications/beam_beam_collision/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_3d_beam_beam_collision  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_beam_beam_collision  # inputs
     analysis_default_openpmd_regression.py  # analysis
     diags/diag1/  # output

Examples/Physics_applications/capacitive_discharge/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_1d_background_mcc_picmi  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     "inputs_base_1d_picmi.py --test --pythonsolver"  # inputs
     analysis_1d.py  # analysis
     diags/diag1000050  # output
@@ -16,7 +15,6 @@ add_warpx_test(
     test_1d_dsmc_picmi  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     "inputs_base_1d_picmi.py --test --dsmc"  # inputs
     analysis_dsmc.py  # analysis
     diags/diag1000050  # output
@@ -27,7 +25,6 @@ add_warpx_test(
     test_2d_background_mcc  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_background_mcc  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000050  # output
@@ -39,8 +36,7 @@ add_warpx_test(
 #    test_2d_background_mcc_dp_psp  # name
 #    2  # dims
 #    2  # nprocs
-#    OFF  # eb
-#    inputs_test_2d_background_mcc_dp_psp  # inputs
+##    inputs_test_2d_background_mcc_dp_psp  # inputs
 #    analysis_default_regression.py  # analysis
 #    diags/diag1000050  # output
 #    OFF  # dependency
@@ -50,7 +46,6 @@ add_warpx_test(
     test_2d_background_mcc_picmi  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_background_mcc_picmi.py  # inputs
     analysis_2d.py  # analysis
     diags/diag1000050  # output

Examples/Physics_applications/laser_acceleration/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_1d_laser_acceleration  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_1d_laser_acceleration  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000100  # output
@@ -16,7 +15,6 @@ add_warpx_test(
     test_1d_laser_acceleration_fluid  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_1d_laser_acceleration_fluid  # inputs
     analysis_1d_fluid.py  # analysis
     diags/diag1040000  # output
@@ -27,7 +25,6 @@ add_warpx_test(
     test_1d_laser_acceleration_fluid_boosted  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_1d_laser_acceleration_fluid_boosted  # inputs
     analysis_1d_fluid_boosted.py  # analysis
     diags/diag1000001  # output
@@ -38,7 +35,6 @@ add_warpx_test(
     test_1d_laser_acceleration_picmi  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_1d_laser_acceleration_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000100  # output
@@ -49,7 +45,6 @@ add_warpx_test(
     test_2d_laser_acceleration_boosted  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_laser_acceleration_boosted  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000002  # output
@@ -60,7 +55,6 @@ add_warpx_test(
     test_2d_laser_acceleration_mr  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_laser_acceleration_mr  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000200  # output
@@ -71,7 +65,6 @@ add_warpx_test(
     test_2d_laser_acceleration_mr_picmi  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_laser_acceleration_mr_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000200  # output
@@ -82,7 +75,6 @@ add_warpx_test(
     test_2d_refined_injection  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_refined_injection  # inputs
     analysis_refined_injection.py  # analysis
     diags/diag1000200  # output
@@ -93,7 +85,6 @@ add_warpx_test(
     test_3d_laser_acceleration  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_laser_acceleration  # inputs
     analysis_default_openpmd_regression.py  # analysis
     diags/diag1/  # output
@@ -104,7 +95,6 @@ add_warpx_test(
     test_3d_laser_acceleration_picmi  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_laser_acceleration_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000100  # output
@@ -115,7 +105,6 @@ add_warpx_test(
     test_3d_laser_acceleration_single_precision_comms  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_laser_acceleration_single_precision_comms  # inputs
     analysis_default_openpmd_regression.py  # analysis
     diags/diag1/  # output
@@ -126,7 +115,6 @@ add_warpx_test(
     test_rz_laser_acceleration  # name
     RZ  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_rz_laser_acceleration  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000010  # output
@@ -137,7 +125,6 @@ add_warpx_test(
     test_rz_laser_acceleration_opmd  # name
     RZ  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_rz_laser_acceleration_opmd  # inputs
     analysis_openpmd_rz.py  # analysis
     diags/diag1/  # output
@@ -148,7 +135,6 @@ add_warpx_test(
     test_rz_laser_acceleration_picmi  # name
     RZ  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_rz_laser_acceleration_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000010  # output

Examples/Physics_applications/laser_ion/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_2d_laser_ion_acc  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_laser_ion_acc  # inputs
     analysis_default_openpmd_regression.py  # analysis
     diags/diag1/  # output
@@ -16,7 +15,6 @@ add_warpx_test(
     test_2d_laser_ion_acc_picmi  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_laser_ion_acc_picmi.py  # inputs
     analysis_default_openpmd_regression.py  # analysis
     diags/diag1/  # output

Examples/Physics_applications/plasma_acceleration/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_1d_plasma_acceleration_picmi  # name
     1  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_1d_plasma_acceleration_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1001000  # output
@@ -16,7 +15,6 @@ add_warpx_test(
     test_2d_plasma_acceleration_boosted  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_plasma_acceleration_boosted  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000020  # output
@@ -27,7 +25,6 @@ add_warpx_test(
     test_2d_plasma_acceleration_mr  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_plasma_acceleration_mr  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000400  # output
@@ -38,7 +35,6 @@ add_warpx_test(
     test_2d_plasma_acceleration_mr_momentum_conserving  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_plasma_acceleration_mr_momentum_conserving  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000400  # output
@@ -49,7 +45,6 @@ add_warpx_test(
     test_3d_plasma_acceleration_boosted  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_plasma_acceleration_boosted  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000005  # output
@@ -60,7 +55,6 @@ add_warpx_test(
     test_3d_plasma_acceleration_boosted_hybrid  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_plasma_acceleration_boosted_hybrid  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000025  # output
@@ -71,7 +65,6 @@ add_warpx_test(
     test_3d_plasma_acceleration_mr_picmi  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_plasma_acceleration_mr_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000002  # output
@@ -82,7 +75,6 @@ add_warpx_test(
     test_3d_plasma_acceleration_picmi  # name
     3  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_3d_plasma_acceleration_picmi.py  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000010  # output

Examples/Physics_applications/plasma_mirror/CMakeLists.txt

@@ -5,7 +5,6 @@ add_warpx_test(
     test_2d_plasma_mirror  # name
     2  # dims
     2  # nprocs
-    OFF  # eb
     inputs_test_2d_plasma_mirror  # inputs
     analysis_default_regression.py  # analysis
     diags/diag1000020  # output

Examples/Physics_applications/spacecraft_charging/CMakeLists.txt

@@ -6,7 +6,6 @@ add_warpx_test(
     test_rz_spacecraft_charging_picmi  # name
     RZ  # dims
     2  # nprocs
-    ON  # eb
     inputs_test_rz_spacecraft_charging_picmi.py  # inputs
     analysis.py  # analysis
     diags/diag1/  # output