diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh index 46d441009..87f68f1e0 100755 --- a/.github/scripts/run_tests.sh +++ b/.github/scripts/run_tests.sh @@ -144,9 +144,9 @@ fi # coupling SPECFEM if [ "$TESTDIR" == "EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/" ]; then # turning off mesh output to avoid using too much memory on the test nodes - sed -i "s:^SAVE_MESH_FILES .*:SAVE_MESH_FILES = .false.:" DATA.coarse/Par_file + sed -i "s:^SAVE_MESH_FILES .*:SAVE_MESH_FILES = .false.:" DATA.regional/Par_file sed -i "s:^SAVE_MESH_FILES .*:SAVE_MESH_FILES = .false.:" DATA.local/Par_file - sed -i "s:^CREATE_VTK_FILES .*:CREATE_VTK_FILES = .false.:" DATA.coarse/meshfem3D_files/Mesh_Par_file + sed -i "s:^CREATE_VTK_FILES .*:CREATE_VTK_FILES = .false.:" DATA.regional/meshfem3D_files/Mesh_Par_file sed -i "s:^CREATE_VTK_FILES .*:CREATE_VTK_FILES = .false.:" DATA.local/meshfem3D_files/Mesh_Par_file fi # elastic halfspace, no absorbing diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3ad46720f..8160a45d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -907,7 +907,7 @@ jobs: - name: Run build env: - TESTFLAGS: --with-mpi + TESTFLAGS: --with-mpi --enable-vectorization run: ./.github/scripts/run_build.sh shell: bash diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/CMTSOLUTION b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/CMTSOLUTION similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/CMTSOLUTION rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/CMTSOLUTION diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/Par_file b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/Par_file similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/Par_file rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/Par_file diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/STATIONS b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/STATIONS similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/STATIONS rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/STATIONS diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/Mesh_Par_file b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/Mesh_Par_file similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/Mesh_Par_file rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/Mesh_Par_file diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/interfaces.dat b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/interfaces.dat similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/interfaces.dat rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/interfaces.dat diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/ptopo.xyz.flat.dat b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/ptopo.xyz.flat.dat similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.coarse/meshfem3D_files/ptopo.xyz.flat.dat rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/DATA.regional/meshfem3D_files/ptopo.xyz.flat.dat diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/README.md b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/README.md index ca574fa63..1b7bcac54 100644 --- a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/README.md +++ b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/README.md @@ -1,7 +1,7 @@ # Couple with injection - example of hybrid method run Specfem/Specfem This is a very small example to test the coupling between SPECFEM3D and itself. -In other words, it runs a coarse simulation to create an initial wavefield for a distant event, +In other words, it runs a coarse regional simulation to create an initial wavefield for a distant event, and then re-injects that wavefield at the coupling boundaries of a small-scale local simulation. @@ -11,7 +11,7 @@ This coupling example uses the following steps: 1. create the coupling boundary points of the small-scale local mesh by running the mesher with the small-scale local setup in folder `DATA.local/` -2. run the coarse simulation using DATA.coarse/ to store the wavefield at the boundary points +2. run the coarse regional simulation using DATA.regional/ to store the wavefield at the boundary points into a separate folder `COUPLING_FILES/` 3. run the small-scale local simulation with injecting the wavefield at the boundary points @@ -26,16 +26,16 @@ To launch the full coupling example with the above 3 steps, type: ## Notes -* The coupling example here between coarse and small-scale mesh uses a flat top surface. - This allows the coarse simulation to locate all coupling points at the exact locations. +* The coupling example here between coarse regional and small-scale local mesh uses a flat top surface. + This allows the coarse regional simulation to locate all coupling points at the exact locations. The resulting seismograms for stations inside the small-scale region should therefore become (almost) identical. You can compare the seismograms in the output folder `OUTPUT_FILES/` from the small-scale local simulation, with the - seismograms in `OUTPUT_FILES.coarse/` from the coarse simulation. + seismograms in `OUTPUT_FILES.regional/` from the coarse regional simulation. -* In case coarse and small-scale simulation would use realistic topographic interfaces, there might be a difference +* In case coarse regional and small-scale local simulation would use realistic topographic interfaces, there might be a difference in topography resolution and thus coupling points from the small-scale coupling boundary could be located slightly - outside the coarser mesh. Locating those coupling points would move them closer to the nearest coarse mesh elements. + outside the coarser regional mesh. Locating those coupling points would move them closer to the nearest coarse mesh elements. Storing the wavefield and re-injecting it would then lead to smaller artefacts. To work-around this, one would have to make sure that the top of the small-scale mesh surfaces match exactly @@ -44,13 +44,13 @@ To launch the full coupling example with the above 3 steps, type: For "smoother" incoming wavefields and small discrepencies between surface points, the injected wavefield however might still be good enough. -* To save storage space, we use `NTSTEP_BETWEEN_OUTPUT_SAMPLE == 2` in the `Par_file` of the coarse simulation. +* To save storage space, we use `NTSTEP_BETWEEN_OUTPUT_SAMPLE == 2` in the `Par_file` of the coarse regional simulation. This seems to provide enough time samplings for the injection. One would have to test this out a bit for other examples. In general, the storage of the wavefield components (velocity & traction) over all coupling boundary points and over all time steps can lead to significant file sizes. In any case, the stored wavefield will be (time) interpolated during the preparation stage of the small-scale local simulation, - as the time steps `DT` of the coarse and small-scale simulations are likely different. Time interpolation is done by a + as the time steps `DT` of the coarse regional and small-scale local simulations are likely different. Time interpolation is done by a Catmull-Rom scheme that provides cubic interpolation between time samples. This scheme seems to exhibit a good balance between accuracy and speed for these kind of simulations. Thus, storing the wavefield at lower sampling rates can be fine for many applications. @@ -67,7 +67,7 @@ to see what requirements they have. You can use the script in `utils/scripts/run_get_simulation_topography.py` to download and setup the topography surface (including one "downshifted" surface to optimize the mesh with the in-house mesher). - For example, to download a 3-arc seconds topography for the specified coarse region, you would type: + For example, to download a 3-arc seconds topography for the specified coarse regional region, you would type: ``` ./run_get_simulation_topography.py 7.2 47.5 8.6 48.5 --SRTM=topo3 --toposhift=10000.0 --toposcale=0.0 ``` @@ -100,7 +100,7 @@ to see what requirements they have. ``` ./run_convert_IRIS_EMC_netCDF_2_tomo.py --EMC_file=IRIS_EMC/LSP-Eucrust1.0.nc --mesh_area=7.2,47.5,8.6,48.5 --maximum_depth=80.0 ``` - using the area and depth of the coarse mesh used above. This script will create the needed `tomography_model.xyz` file, which you + using the area and depth of the coarse regional mesh used above. This script will create the needed `tomography_model.xyz` file, which you can for example copy into a `DATA/tomo_files/` folder. Once you have a tomography file, you would adjust the `TOMOGRAPHY_PATH` parameter in `DATA/Par_file`, as well diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_generate_databases.coarse.txt b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_generate_databases.regional.txt similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_generate_databases.coarse.txt rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_generate_databases.regional.txt diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_meshfem3D.coarse.txt b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_meshfem3D.regional.txt similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_meshfem3D.coarse.txt rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_meshfem3D.regional.txt diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_solver.coarse.txt b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_solver.regional.txt similarity index 100% rename from EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_solver.coarse.txt rename to EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/REF_SEIS/output_solver.regional.txt diff --git a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/run_this_example.sh b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/run_this_example.sh index dc8bce591..324a9761a 100755 --- a/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/run_this_example.sh +++ b/EXAMPLES/applications/small_example_coupling_SPECFEM_specfem/run_this_example.sh @@ -33,17 +33,17 @@ rm -rf DATABASES_MPI.local mv -v DATABASES_MPI DATABASES_MPI.local ## -## Step 2 - create wavefield solution from coarse simulation +## Step 2 - create wavefield solution from coarse regional simulation ## echo echo "##############################################" -echo "STEP 2 - coarse simulation" +echo "STEP 2 - regional simulation" echo "##############################################" echo # symbolic link rm -f DATA -ln -s DATA.coarse/ DATA +ln -s DATA.regional/ DATA # mesher ./run_mesher.sh @@ -56,11 +56,11 @@ if [[ $? -ne 0 ]]; then exit 1; fi if [[ $? -ne 0 ]]; then exit 1; fi # backup -rm -rf OUTPUT_FILES.coarse -mv -v OUTPUT_FILES OUTPUT_FILES.coarse +rm -rf OUTPUT_FILES.regional +mv -v OUTPUT_FILES OUTPUT_FILES.regional -rm -rf DATABASES_MPI.coarse -mv -v DATABASES_MPI DATABASES_MPI.coarse +rm -rf DATABASES_MPI.regional +mv -v DATABASES_MPI DATABASES_MPI.regional ## ## Step 3 - run coupled small-scale local simulation diff --git a/src/specfem3D/couple_with_injection.f90 b/src/specfem3D/couple_with_injection.f90 index 9c33cf377..c97a3b25f 100644 --- a/src/specfem3D/couple_with_injection.f90 +++ b/src/specfem3D/couple_with_injection.f90 @@ -3153,7 +3153,7 @@ subroutine couple_with_injection_prepare_specfem_files() open(unit=IIN,file=trim(filename),status='old',action='read',form='unformatted',iostat=ier) if (ier /= 0) then print *,'Error: could not open file ',trim(filename) - print *,' Please check if file exists and was computed with a previous (coarse) forward simulation...' + print *,' Please check if file exists and was computed with a previous (coarse) regional forward simulation...' stop 'Error opening specfem_coupling_solution.bin' endif @@ -3190,6 +3190,9 @@ subroutine couple_with_injection_prepare_specfem_files() write(IMAIN,*) ! estimates array memory tmp_veloc_points_timeseries & tmp_traction_points_timeseries sizeval = 2.d0 * dble(NDIM) * dble(maxval(nb_points_local_per_proc)) * dble(ntimesteps) * dble(CUSTOM_REAL) + ! adds memory for read arrays tmp_veloc & tmp_traction + sizeval = sizeval + 2.d0 * dble(NDIM) * dble(npoints_total) * dble(CUSTOM_REAL) + ! info output write(IMAIN,*) ' required maximum array size per slice = ',sngl(sizeval / 1024.d0 / 1024.d0),'MB' write(IMAIN,*) ' = ',sngl(sizeval / 1024.d0 / 1024.d0 / 1024.d0),'GB' write(IMAIN,*) @@ -3271,6 +3274,9 @@ subroutine couple_with_injection_prepare_specfem_files() endif endif + ! synchronizes + call synchronize_all() + ! creates name for each rank process ! format: {TRACTION_PATH} / proc***_ call create_name_database(prname_trac,myrank,TRACTION_PATH) @@ -3307,6 +3313,9 @@ subroutine couple_with_injection_prepare_specfem_files() if (ier /= 0) call exit_MPI(myrank,'error allocating array 2193') Tract_specfem(:,:) = 0.0_CUSTOM_REAL + ! synchronizes to be sure all processes had enough memory + call synchronize_all() + ! interpolate wavefield solution in time ! ! Sinc (Whittaker-Shannon) interpolation: