Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CarpetX: Change the implementation to Driver. #299

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion BenchX/par/benchmark-carpetx.par
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ CarpetX::boundary_upper_z = "dirichlet"

CarpetX::max_num_levels = $nlevels
CarpetX::regrid_every = 0
CarpetX::regrid_error_threshold = 2.0
Driver::regrid_error_threshold = 2.0

ErrorEstimator::region_shape = "cube"
ErrorEstimator::scale_by_resolution = yes
Expand Down
2 changes: 1 addition & 1 deletion BenchX/par/benchmark-z4c+grhydrox.par
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ CarpetX::boundary_upper_z = "dirichlet"

CarpetX::max_num_levels = $nlevels
CarpetX::regrid_every = 0
CarpetX::regrid_error_threshold = 2.0
Driver::regrid_error_threshold = 2.0

ErrorEstimator::region_shape = "cube"
ErrorEstimator::scale_by_resolution = yes
Expand Down
2 changes: 1 addition & 1 deletion BenchX/par/benchmark-z4c.par
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ CarpetX::boundary_upper_z = "dirichlet"

CarpetX::max_num_levels = $nlevels
CarpetX::regrid_every = 0
CarpetX::regrid_error_threshold = 2.0
Driver::regrid_error_threshold = 2.0

ErrorEstimator::region_shape = "cube"
ErrorEstimator::scale_by_resolution = yes
Expand Down
2 changes: 1 addition & 1 deletion BoxInBox/interface.ccl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

IMPLEMENTS: BoxInBox

INHERITS: CarpetX
INHERITS: Driver, CarpetXRegrid

USES INCLUDE HEADER: defs.hxx
USES INCLUDE HEADER: loop_device.hxx
Expand Down
4 changes: 2 additions & 2 deletions BoxInBox/schedule.ccl
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ SCHEDULE BoxInBox_Setup AS EstimateError AT postinitial
READS: positions
READS: radii
READS: radiixyz
WRITES: CarpetX::regrid_error(interior)
WRITES: CarpetXRegrid::regrid_error(interior)
} "Estimate error"

SCHEDULE BoxInBox_Setup AS EstimateError AT poststep
Expand All @@ -30,5 +30,5 @@ SCHEDULE BoxInBox_Setup AS EstimateError AT poststep
READS: positions
READS: radii
READS: radiixyz
WRITES: CarpetX::regrid_error(interior)
WRITES: CarpetXRegrid::regrid_error(interior)
} "Estimate error"
7 changes: 6 additions & 1 deletion BoxInBox/test/levels1.par
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ ActiveThorns = "
CarpetX
CoordinatesX
IOUtil
Regrid
"

Cactus::cctk_show_schedule = no
Expand Down Expand Up @@ -38,7 +39,11 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"

Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/levels2.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -39,7 +40,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/levels20.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -57,7 +58,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/levels3.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -40,7 +41,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/regions2.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -49,7 +50,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/regions3.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -60,7 +61,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
6 changes: 5 additions & 1 deletion BoxInBox/test/unigrid.par
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ActiveThorns = "
BoxInBox
CarpetX
Regrid
CoordinatesX
IOUtil
"
Expand Down Expand Up @@ -37,7 +38,10 @@ CarpetX::out_norm_vars = "all"
CarpetX::out_norm_omit_unstable = yes

CarpetX::out_tsv_vars = "
CarpetX::regrid_error
CarpetXRegrid::regrid_error
CoordinatesX::vertex_coords
CoordinatesX::cell_coords
"
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
2 changes: 1 addition & 1 deletion CarpetX/configuration.ccl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

REQUIRES AMReX IOUtil MPI yaml_cpp zlib

REQUIRES Arith Loop
REQUIRES Arith Loop Regrid

OPTIONAL ADIOS2
{
Expand Down
13 changes: 1 addition & 12 deletions CarpetX/interface.ccl
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# Interface definition for thorn CarpetX

IMPLEMENTS: CarpetX

INHERITS: IO
IMPLEMENTS: Driver

USES INCLUDE HEADER: silo.hxx

Expand Down Expand Up @@ -166,12 +164,3 @@ CCTK_INT FUNCTION DriverInterpolate(
CCTK_INT ARRAY IN output_array_types,
CCTK_POINTER ARRAY IN output_arrays)
PROVIDES FUNCTION DriverInterpolate WITH CarpetX_DriverInterpolate LANGUAGE C



PUBLIC:

CCTK_REAL regrid_error TYPE=gf CENTERING={ccc} TAGS='checkpoint="no"'
{
regrid_error
} "Regridding condition"
6 changes: 3 additions & 3 deletions CarpetX/par/brill-lindquist-checkpoint.par
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ CarpetX::ncells_x = 64
CarpetX::ncells_y = 64
CarpetX::ncells_z = 64

CarpetX::periodic_x = no
CarpetX::periodic_y = no
CarpetX::periodic_z = no
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
# CarpetX::reflection_x = yes
# CarpetX::reflection_y = yes
# CarpetX::reflection_z = yes
Expand Down
6 changes: 3 additions & 3 deletions CarpetX/par/brill-lindquist-read.par
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ CarpetX::ncells_x = 64
CarpetX::ncells_y = 64
CarpetX::ncells_z = 64

CarpetX::periodic_x = no
CarpetX::periodic_y = no
CarpetX::periodic_z = no
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
# CarpetX::reflection_x = yes
# CarpetX::reflection_y = yes
# CarpetX::reflection_z = yes
Expand Down
6 changes: 3 additions & 3 deletions CarpetX/par/brill-lindquist-recover.par
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ CarpetX::ncells_x = 64
CarpetX::ncells_y = 64
CarpetX::ncells_z = 64

CarpetX::periodic_x = no
CarpetX::periodic_y = no
CarpetX::periodic_z = no
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
# CarpetX::reflection_x = yes
# CarpetX::reflection_y = yes
# CarpetX::reflection_z = yes
Expand Down
6 changes: 3 additions & 3 deletions CarpetX/par/brill-lindquist-write.par
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ CarpetX::ncells_x = 64
CarpetX::ncells_y = 64
CarpetX::ncells_z = 64

CarpetX::periodic_x = no
CarpetX::periodic_y = no
CarpetX::periodic_z = no
Driver::periodic_x = no
Driver::periodic_y = no
Driver::periodic_z = no
# CarpetX::reflection_x = yes
# CarpetX::reflection_y = yes
# CarpetX::reflection_z = yes
Expand Down
15 changes: 10 additions & 5 deletions CarpetX/param.ccl
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,22 @@ CCTK_INT ncells_z "Number of grid cells"

RESTRICTED:

BOOLEAN periodic_x "Periodic"
# Do not use
BOOLEAN periodic "Do not use"
{
} no

BOOLEAN periodic_y "Periodic"
BOOLEAN periodic_x "Do not use"
{
} no
} yes

BOOLEAN periodic_z "Periodic"
BOOLEAN periodic_y "Do not use"
{
} no
} yes

BOOLEAN periodic_z "Do not use"
{
} yes



Expand Down
10 changes: 5 additions & 5 deletions CarpetX/schedule.ccl
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ if (!CCTK_Equals(recover, "no") && *recover_file)

# Error estimation

SCHEDULE CarpetX_InitError AT basegrid
{
LANG: C
WRITES: regrid_error(everywhere)
} "Initialize regridding error to zero"
#SCHEDULE CarpetX_InitError AT basegrid
#{
# LANG: C
# WRITES: CarpetXRegrid::regrid_error(everywhere)
#} "Initialize regridding error to zero"



Expand Down
13 changes: 10 additions & 3 deletions CarpetX/src/driver.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,19 @@ array<array<boundary_t, dim>, 2> get_default_boundaries() {
bool(CCTK_EQUALS(boundary_upper_z, "robin")),
}},
}};
for (int f = 0; f < 2; ++f)
for (int d = 0; d < dim; ++d)
for (int f = 0; f < 2; ++f) {
for (int d = 0; d < dim; ++d) {
std::cout << "f=" << f << " d=" << d << std::endl;
std::cout << "is_symmetry[f][d]=" << is_symmetry[f][d] << std::endl;
std::cout << "is_dirichlet[f][d]=" << is_dirichlet[f][d] << std::endl;
std::cout << "is_neumann[f][d]=" << is_neumann[f][d] << std::endl;
std::cout << "is_robin[f][d]=" << is_robin[f][d] << std::endl;
assert(is_symmetry[f][d] + is_dirichlet[f][d] +
is_linear_extrapolation[f][d] + is_neumann[f][d] +
is_robin[f][d] <=
1);
}
}

array<array<boundary_t, dim>, 2> boundaries;
for (int f = 0; f < 2; ++f)
Expand Down Expand Up @@ -1527,7 +1534,7 @@ void CactusAmrCore::ErrorEst(const int level, amrex::TagBoxArray &tags,
#pragma omp critical
CCTK_VINFO("ErrorEst patch %d level %d", patch, level);

const int gi = CCTK_GroupIndex("CarpetX::regrid_error");
const int gi = CCTK_GroupIndex("CarpetXRegrid::regrid_error");
assert(gi >= 0);
const int vi = 0;
const int tl = 0;
Expand Down
1 change: 0 additions & 1 deletion CarpetX/src/make.code.defn
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ SRCS = \
prolongate_3d_rf2_impl_poly.cxx \
prolongate_3d_rf2_impl_poly_cons3lfb.cxx \
reduction.cxx \
regrid_error.cxx \
schedule.cxx \
task_manager.cxx \
timer.cxx \
Expand Down
4 changes: 2 additions & 2 deletions CarpetX/src/schedule.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2240,7 +2240,7 @@ int SyncGroupsByDirI(const cGH *restrict cctkGH, int numgroups,
CCTK_VINFO("SyncGroups %s", buf.str().c_str());
}

const int gi_regrid_error = CCTK_GroupIndex("CarpetX::regrid_error");
const int gi_regrid_error = CCTK_GroupIndex("CarpetXRegrid::regrid_error");
assert(gi_regrid_error >= 0);

vector<int> groups;
Expand Down Expand Up @@ -2621,7 +2621,7 @@ void Restrict(const cGH *cctkGH, int level, const vector<int> &groups) {
static Timer timer("Restrict");
Interval interval(timer);

const int gi_regrid_error = CCTK_GroupIndex("CarpetX::regrid_error");
const int gi_regrid_error = CCTK_GroupIndex("CarpetXRegrid::regrid_error");
assert(gi_regrid_error >= 0);

for (const auto &patchdata : ghext->patchdata) {
Expand Down
11 changes: 11 additions & 0 deletions CarpetXRegrid/README
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Cactus Code Thorn Regrid
Author(s) : Steven R. Brandt <[email protected]>
Maintainer(s): Steven R. Brandt <[email protected]>
Licence : Cactus
--------------------------------------------------------------------------

1. Purpose

Provide an extra grid function, "regrid_error," to the CarpetX driver.
This grid function is used by CarpetX and AMReX to determine where to
refine the grid.
6 changes: 6 additions & 0 deletions CarpetXRegrid/configuration.ccl
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Configuration definitions for thorn Regrid
REQUIRES AMReX Arith Loop

PROVIDES Regrid
{
}
Loading
Loading