From 5ec3f4e5434fe60d57abc9e9fc4aab38ca369517 Mon Sep 17 00:00:00 2001 From: Alwin Mao Date: Mon, 6 Feb 2023 08:47:26 -0500 Subject: [PATCH 01/10] refactor feedback --- src/particles/feedback_CIC_gpu.cu | 327 +++++++++++++++++++++++++++++- src/utils/gpu.hpp | 1 + 2 files changed, 324 insertions(+), 4 deletions(-) diff --git a/src/particles/feedback_CIC_gpu.cu b/src/particles/feedback_CIC_gpu.cu index a9dad701d..646914f8a 100644 --- a/src/particles/feedback_CIC_gpu.cu +++ b/src/particles/feedback_CIC_gpu.cu @@ -232,7 +232,326 @@ __device__ bool Particle_Is_Alone(Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_ return true; } +__device__ Real Cluster_Feedback_Resolved(Real pos_x, Real pos_y, Real pos_z, + Real xMin, Real yMin, Real zMin, + Real dx, Real dy, Real dz, + int nx_g, int ny_g, int n_ghost, int n_cells, + Real gamma, Real* conserved_device, + short direction, + Real feedback_density, Real feedback_energy) +{ + // For 2x2x2, a particle between 0-0.5 injects onto cell - 1 + int indx_x = (int)floor((pos_x - xMin - 0.5 * dx) / dx); + int indx_y = (int)floor((pos_y - yMin - 0.5 * dy) / dy); + int indx_z = (int)floor((pos_z - zMin - 0.5 * dz) / dz); + + Real cell_center_x = xMin + indx_x * dx + 0.5 * dx; + Real cell_center_y = yMin + indx_y * dy + 0.5 * dy; + Real cell_center_z = zMin + indx_z * dz + 0.5 * dz; + + Real delta_x = 1 - (pos_x - cell_center_x) / dx; + Real delta_y = 1 - (pos_y - cell_center_y) / dy; + Real delta_z = 1 - (pos_z - cell_center_z) / dz; + + Real* density = conserved_device; + Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; + Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; + Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; + Real* energy = &conserved_device[n_cells*grid_enum::Energy]; + Real* gasEnergy = &conserved_device[n_cells*grid_enum::GasEnergy]; + + Real local_dti = 0; + + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + for (int k = 0; k < 2; k++) { + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + Real x_frac = i * (1 - delta_x) + (1 - i) * delta_x; + Real y_frac = j * (1 - delta_y) + (1 - j) * delta_y; + Real z_frac = k * (1 - delta_z) + (1 - k) * delta_z; + + atomicAdd(&density[indx], + x_frac * y_frac * z_frac * feedback_density); + atomicAdd(&gasEnergy[indx], + x_frac * y_frac * z_frac * feedback_energy); + atomicAdd(&energy[indx], + x_frac * y_frac * z_frac * feedback_energy); + + if (direction > 0) { + Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, + momentum_z, energy, indx, dx, dy, dz); + + local_dti = fmax(local_dti, cell_dti); + } + + + } // k loop + } // j loop + } // i loop + + return local_dti; +} + + +__device__ Real Cluster_Feedback_Unresolved(Real pos_x, Real pos_y, Real pos_z, + Real xMin, Real yMin, Real zMin, + Real dx, Real dy, Real dz, + int nx_g, int ny_g, int n_ghost, int n_cells, + Real gamma, Real* conserved_device, + short direction, + Real feedback_density, Real feedback_momentum, + Real n_0) +{ + // For 3x3x3 the index just centers on the cell containing the particle + int indx_x = (int)floor((pos_x - xMin) / dx); + int indx_y = (int)floor((pos_y - yMin) / dy); + int indx_z = (int)floor((pos_z - zMin) / dz); + + Real delta_x = (pos_x - xMin - indx_x * dx) / dx; + Real delta_y = (pos_y - yMin - indx_y * dy) / dy; + Real delta_z = (pos_z - zMin - indx_z * dz) / dz; + + Real local_dti = 0; + + Real* density = conserved_device; + Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; + Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; + Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; + Real* energy = &conserved_device[n_cells*grid_enum::Energy]; + Real* gasEnergy = &conserved_device[n_cells*grid_enum::GasEnergy]; + + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + // index in array of conserved quantities + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + + Real x_frac = + d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); + Real y_frac = + frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); + Real z_frac = + frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + + Real px = x_frac * feedback_momentum; + Real py = y_frac * feedback_momentum; + Real pz = z_frac * feedback_momentum; + Real d = (abs(x_frac) + abs(y_frac) + abs(z_frac)) / 6 * + feedback_density + + n_0 * supernova::MU * MP / DENSITY_UNIT; + + atomicAdd(&momentum_x[indx], px); + atomicAdd(&momentum_y[indx], py); + atomicAdd(&momentum_z[indx], pz); + density[indx] = d; + energy[indx] = (momentum_x[indx] * momentum_x[indx] + + momentum_y[indx] * momentum_y[indx] + + momentum_z[indx] * momentum_z[indx]) / + 2 / density[indx] + + gasEnergy[indx]; + + if (direction > 0) { + Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, + momentum_z, energy, indx, dx, dy, dz); + + local_dti = fmax(local_dti, cell_dti); + } + + + + } // k loop + } // j loop + } // i loop + + return local_dti; + + + +} + + +__device__ void Cluster_Feedback_Helper(part_int_t n_local, + Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, + Real* age_dev, Real* mass_dev, part_int_t* id_dev, + Real xMin, Real yMin, Real zMin, + Real xMax, Real yMax, Real zMax, + Real dx, Real dy, Real dz, + int nx_g, int ny_g, int nz_g, + int n_ghost, + int n_step, + Real t, Real dt, Real* dti, + Real* dev_snr, Real snr_dt, Real time_sn_start, Real time_sn_end, + Real* prev_dens, int* prev_N, + short direction, + Real* s_info, Real* conserved_dev, Real gamma) +{ + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + // Bounds check on particle arrays + if (gtid >= n_local) return; + + Real pos_x = pos_x_dev[gtid]; + Real pos_y = pos_y_dev[gtid]; + Real pos_z = pos_z_dev[gtid]; + bool in_local = (pos_x >= xMin && pos_x < xMax) && + (pos_y >= yMin && pos_y < yMax) && + (pos_z >= zMin && pos_z < zMax); + // Particle is outside bounds, exit + if (!in_local) return; + + int indx_x = (int)floor((pos_x - xMin) / dx); + int indx_y = (int)floor((pos_y - yMin) / dy); + int indx_z = (int)floor((pos_z - zMin) / dz); + bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || + indx_x >= nx_g - 2 * n_ghost || + indx_y >= ny_g - 2 * n_ghost || + indx_z >= nz_g - 2 * n_ghost; + // Ignore this particle, exit + if (ignore) return; + + bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + if (!is_alone) return; + + + // note age_dev is actually the time of birth + Real age = t - age_dev[gtid]; + if (age > time_sn_end) return; + + int N = 0; + if (direction == -1) { + N = -prev_N[gtid]; + } else { + Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, + time_sn_start, time_sn_end) * mass_dev[gtid] * dt; + feedback_prng_t state; + curand_init(42,0,0,&state); + unsigned long long skip = n_step * 10000 + id_dev[gtid]; + skipahead(skip, &state); // provided by curand + N = (int) curand_poisson(&state, average_num_sn); + prev_N[gtid] = N; + } + + // No supernova, exit + if (N == 0) return; + + Real n_0; + if (direction == -1) { + n_0 = prev_dens[gtid]; + } else { + Real* density = conserved_dev; + n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost); + prev_dens[gtid] = n_0; + } + + + mass_dev[gtid] -= N * supernova::MASS_PER_SN; + Real dV = dx * dy * dz; + Real feedback_energy = N * supernova::ENERGY_PER_SN / dV; + Real feedback_density = N * supernova::MASS_PER_SN / dV; + Real feedback_momentum = direction * supernova::FINAL_MOMENTUM * + pow(n_0, -0.17) * pow(fabsf(N), 0.93) / dV; + + + Real shell_radius = supernova::R_SH * pow(n_0, -0.46) * pow(fabsf(N), 0.29); + bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; + + int n_cells = nx_g * ny_g * nz_g; + + Real local_dti = 0.0; + if (is_resolved) { + local_dti = Cluster_Feedback_Resolved( pos_x, pos_y, pos_z, + xMin, yMin, zMin, + dx, dy, dz, + nx_g, ny_g, n_ghost, n_cells, + gamma, conserved_dev, + direction, + feedback_density, feedback_energy); + // inject energy and density + } else { + // inject momentum and density + local_dti = Cluster_Feedback_Unresolved( pos_x, pos_y, pos_z, + xMin, yMin, zMin, + dx, dy, dz, + nx_g, ny_g, n_ghost, n_cells, + gamma, conserved_dev, + direction, + feedback_density, feedback_momentum, + n_0); + + } + + if (direction > 0) atomicMax(dti, local_dti); + + return; + +} + __global__ void Cluster_Feedback_Kernel( + part_int_t n_local, part_int_t* id_dev, Real* pos_x_dev, Real* pos_y_dev, + Real* pos_z_dev, Real* mass_dev, Real* age_dev, Real xMin, Real yMin, + Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, + int nx_g, int ny_g, int nz_g, int n_ghost, Real t, Real dt, Real* dti, + Real* info, Real* density, Real* gasEnergy, Real* energy, Real* momentum_x, + Real* momentum_y, Real* momentum_z, Real gamma, + feedback_prng_t* states, Real* prev_dens, int* prev_N, + short direction, Real* dev_snr, Real snr_dt, Real time_sn_start, + Real time_sn_end, int n_step) +{ + + int tid = threadIdx.x; + //int gtid = blockIdx.x * blockDim.x + tid; + + // for collecting SN feedback information + __shared__ Real s_info[FEED_INFO_N * TPB_FEEDBACK]; + + Cluster_Feedback_Helper(n_local, + pos_x_dev, pos_y_dev, pos_z_dev, + age_dev, mass_dev, id_dev, + xMin, yMin, zMin, + xMax, yMax, zMax, + dx, dy, dz, + nx_g, ny_g, nz_g, + n_ghost, + n_step, + t, dt, dti, + dev_snr, snr_dt, time_sn_start, time_sn_end, + prev_dens, prev_N, + direction, + s_info, density, gamma); + + + __syncthreads(); + + // reduce the info from all the threads in the block + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (tid < s) { + s_info[FEED_INFO_N * tid] += s_info[FEED_INFO_N * (tid + s)]; + s_info[FEED_INFO_N * tid + 1] += s_info[FEED_INFO_N * (tid + s) + 1]; + s_info[FEED_INFO_N * tid + 2] += s_info[FEED_INFO_N * (tid + s) + 2]; + s_info[FEED_INFO_N * tid + 3] += s_info[FEED_INFO_N * (tid + s) + 3]; + s_info[FEED_INFO_N * tid + 4] += s_info[FEED_INFO_N * (tid + s) + 4]; + s_info[FEED_INFO_N * tid + 5] += s_info[FEED_INFO_N * (tid + s) + 5]; + } + __syncthreads(); + } + + if (tid == 0) { + info[FEED_INFO_N * blockIdx.x] = s_info[0]; + info[FEED_INFO_N * blockIdx.x + 1] = s_info[1]; + info[FEED_INFO_N * blockIdx.x + 2] = s_info[2]; + info[FEED_INFO_N * blockIdx.x + 3] = s_info[3]; + info[FEED_INFO_N * blockIdx.x + 4] = s_info[4]; + info[FEED_INFO_N * blockIdx.x + 5] = s_info[5]; + } + +} + +__global__ void Cluster_Feedback_Kernel2( part_int_t n_local, part_int_t* id, Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, Real* mass_dev, Real* age_dev, Real xMin, Real yMin, Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, @@ -560,7 +879,7 @@ __global__ void Cluster_Feedback_Kernel( density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); } - + //printf("INDX DEBUG: n_step: %d id: %d indx: %d \n", n_step, (int) id[gtid], indx); if (indx >= nx_g * ny_g * nz_g) { @@ -570,8 +889,8 @@ __global__ void Cluster_Feedback_Kernel( atomicAdd(&momentum_x[indx], px); atomicAdd(&momentum_y[indx], py); atomicAdd(&momentum_z[indx], pz); - - /* + + density[indx] = d; energy[indx] = (momentum_x[indx] * momentum_x[indx] + momentum_y[indx] * momentum_y[indx] + @@ -579,7 +898,7 @@ __global__ void Cluster_Feedback_Kernel( 2 / density[indx] + gasEnergy[indx]; - */ + // atomicAdd( &energy[indx], e ); // atomicAdd( &density[indx], d ); diff --git a/src/utils/gpu.hpp b/src/utils/gpu.hpp index c7968abab..52acbca24 100644 --- a/src/utils/gpu.hpp +++ b/src/utils/gpu.hpp @@ -107,6 +107,7 @@ static constexpr int maxWarpsPerBlock = 1024 / WARPSIZE; #define curandStateMRG32k3a_t hiprandStateMRG32k3a_t #define curand_init hiprand_init + #define curand hiprand #define curand_poisson hiprand_poisson static void __attribute__((unused)) From 91c41caa4414d2676a23c764fee9392673c6cc43 Mon Sep 17 00:00:00 2001 From: ojwg Date: Wed, 5 Apr 2023 23:10:14 -0400 Subject: [PATCH 02/10] Refactors feedback, adds Starburst99 stellar wind contribution to feedback, and removes the need for the DIRS variable in Makefile. --- Makefile | 14 +- builds/make.type.disk | 6 +- src/analysis/feedback_analysis.h | 2 + src/feedback/feedback.cu | 1353 +++++++++++++++++ src/feedback/feedback.h | 55 + .../starburst99_snr.txt | 0 src/feedback/starburst99_sw.txt | 1007 ++++++++++++ src/global/global.cpp | 6 +- src/global/global.h | 6 +- src/gravity/gravity_restart.cpp | 2 +- src/grid/grid3D.h | 8 - src/main.cpp | 34 +- src/particles/feedback_CIC_gpu.cu | 1159 -------------- src/particles/particles_3D.cpp | 12 +- src/particles/supernova.h | 39 - src/utils/timing_functions.cpp | 4 +- 16 files changed, 2468 insertions(+), 1239 deletions(-) create mode 100644 src/feedback/feedback.cu create mode 100644 src/feedback/feedback.h rename src/{particles => feedback}/starburst99_snr.txt (100%) create mode 100755 src/feedback/starburst99_sw.txt delete mode 100644 src/particles/feedback_CIC_gpu.cu delete mode 100644 src/particles/supernova.h diff --git a/Makefile b/Makefile index 782eaf71c..7a4eb2dce 100644 --- a/Makefile +++ b/Makefile @@ -9,17 +9,11 @@ include builds/make.type.$(TYPE) # CUDA_ARCH defaults to sm_70 if not set in make.host CUDA_ARCH ?= sm_70 -DIRS := src src/analysis src/chemistry_gpu src/cooling src/cooling_grackle src/cosmology \ - src/cpu src/global src/gravity src/gravity/paris src/grid src/hydro \ - src/integrators src/io src/main.cpp src/main_tests.cpp src/mhd\ - src/model src/mpi src/old_cholla src/particles src/reconstruction \ - src/riemann_solvers src/system_tests src/utils src/dust - SUFFIX ?= .$(TYPE).$(MACHINE) -CFILES := $(foreach DIR,$(DIRS),$(wildcard $(DIR)/*.c)) -CPPFILES := $(foreach DIR,$(DIRS),$(wildcard $(DIR)/*.cpp)) -GPUFILES := $(foreach DIR,$(DIRS),$(wildcard $(DIR)/*.cu)) +CFILES := $(shell find src/ -type f -name '*.c') +CPPFILES := $(shell find src/ -type f -name '*.cpp') +GPUFILES := $(shell find src/ -type f -name '*.cu') # Build a list of all potential object files so cleaning works properly CLEAN_OBJS := $(subst .c,.o,$(CFILES)) \ @@ -107,7 +101,7 @@ ifeq ($(findstring -DPARIS,$(DFLAGS)),-DPARIS) endif endif -ifeq ($(findstring -DSUPERNOVA,$(DFLAGS)),-DSUPERNOVA) +ifeq ($(findstring -DFEEDBACK,$(DFLAGS)),-DFEEDBACK) ifdef HIPCONFIG CXXFLAGS += -I$(ROCM_PATH)/include/hiprand -I$(ROCM_PATH)/hiprand/include GPUFLAGS += -I$(ROCM_PATH)/include/hiprand -I$(ROCM_PATH)/hiprand/include diff --git a/builds/make.type.disk b/builds/make.type.disk index a95560cf1..5691853b5 100644 --- a/builds/make.type.disk +++ b/builds/make.type.disk @@ -6,7 +6,9 @@ DFLAGS += -DPARTICLES_GPU DFLAGS += -DPARTICLE_IDS #DFLAGS += -DSINGLE_PARTICLE_MASS DFLAGS += -DPARTICLE_AGE -DFLAGS += -DSUPERNOVA #this flag requires PARTICLE_AGE, PARTICLE_IDS +DFLAGS += -DFEEDBACK #this flag requires PARTICLE_AGE, PARTICLE_IDS +#DFLAGS += -DNO_WIND_FEEDBACK +#DFLAGS += -DNO_SN_FEEDBACK DFLAGS += -DANALYSIS #DFLAGS += -DPARTICLES_KDK @@ -36,7 +38,7 @@ DFLAGS += -DTEMPERATURE_FLOOR DFLAGS += -DCOOLING_GPU #DFLAGS += -DCLOUDY_COOL DFLAGS += -DDE -DFLAGS += -DCPU_TIME +#DFLAGS += -DCPU_TIME DFLAGS += -DAVERAGE_SLOW_CELLS DFLAGS += -DHYDRO_GPU diff --git a/src/analysis/feedback_analysis.h b/src/analysis/feedback_analysis.h index 9b29420f4..5e937996e 100644 --- a/src/analysis/feedback_analysis.h +++ b/src/analysis/feedback_analysis.h @@ -21,6 +21,8 @@ class FeedbackAnalysis Real totalEnergy{0}; Real totalMomentum{0}; Real totalUnresEnergy{0}; + Real totalWindMomentum{0}; + Real totalWindEnergy{0}; FeedbackAnalysis(Grid3D& G); ~FeedbackAnalysis(); diff --git a/src/feedback/feedback.cu b/src/feedback/feedback.cu new file mode 100644 index 000000000..aba2ad6fd --- /dev/null +++ b/src/feedback/feedback.cu @@ -0,0 +1,1353 @@ +#if defined(FEEDBACK) && defined(PARTICLES_GPU) && defined(PARTICLE_AGE) && \ + defined(PARTICLE_IDS) + + #include + #include + #include + #include + + #include + #include + #include + #include + + #include "../global/global.h" + #include "../global/global_cuda.h" + #include "../grid/grid3D.h" + #include "../io/io.h" + #include "feedback.h" + + #define FEED_INFO_N 8 + #define i_RES 1 + #define i_UNRES 2 + #define i_ENERGY 3 + #define i_MOMENTUM 4 + #define i_UNRES_ENERGY 5 + #define i_WIND_MOMENTUM 6 + #define i_WIND_ENERGY 7 + + // the starburst 99 total stellar mass input + // stellar wind momentum fluxes and SN rates + // must be divided by this to get per solar + // mass values. + #define S_99_TOTAL_MASS 1e6 + + #define TPB_FEEDBACK 128 + // seed for poisson random number generator + #define FEEDBACK_SEED 42 + + +namespace feedback +{ + feedback_prng_t* randStates; + part_int_t n_states; + Real *dev_snr, snr_dt, time_sn_start, time_sn_end; + Real *dev_sw_p, *dev_sw_e, sw_dt, time_sw_start, time_sw_end; + int snr_n; +} // namespace feedback + + +#ifndef O_HIP +inline __device__ double atomicMax(double* address, double val) +{ + unsigned long long int* address_as_ull = (unsigned long long int*)address; + unsigned long long int old = *address_as_ull, assumed; + do { + assumed = old; + old = atomicCAS( address_as_ull, assumed, + __double_as_longlong(fmax(val, __longlong_as_double(assumed)))); + } while (assumed != old); + return __longlong_as_double(old); +} +#endif // O_HIP + + +inline __device__ Real Calc_Timestep(Real gamma, Real* density, + Real* momentum_x, Real* momentum_y, Real* momentum_z, Real* energy, + int index, Real dx, Real dy, Real dz) +{ + Real dens = fmax(density[index], DENS_FLOOR); + Real d_inv = 1.0 / dens; + Real vx = momentum_x[index] * d_inv; + Real vy = momentum_y[index] * d_inv; + Real vz = momentum_z[index] * d_inv; + Real P = fmax((energy[index] - 0.5 * dens * (vx * vx + vy * vy + vz * vz)) + * (gamma - 1.0), TINY_NUMBER); + Real cs = sqrt(gamma * P * d_inv); + return fmax(fmax((fabs(vx) + cs) / dx, (fabs(vy) + cs) / dy) + , (fabs(vz) + cs) / dz); +} + +/** the prescription for dividing a scalar quantity between 3x3x3 cells is done + by imagining a 2x2x2 cell volume around the SN. These fractions, then, + represent the linear extent of this volume into the cell in question. For i=0 + this should be 1*1/2. For i=-1 this should be (1-dx)*1/2. For i=+1 this + should be dx*1/2. In the above the 1/2 factor is normalize over 2 + cells/direction. + */ +inline __device__ Real frac(int i, Real dx) +{ + return (-0.5 * i * i - 0.5 * i + 1 + i * dx) * 0.5; +} + + +inline __device__ Real d_fr(int i, Real dx) +{ + return (dx > 0.5) * i * (1 - 2 * dx) + ((i + 1) * dx + 0.5 * (i - 1)) - + 3 * (i - 1) * (i + 1) * (0.5 - dx); +} + + +inline __device__ bool Particle_Is_Alone(Real* pos_x_dev, Real* pos_y_dev, + Real* pos_z_dev, part_int_t n_local, int gtid, Real dx) +{ + Real x0 = pos_x_dev[gtid]; + Real y0 = pos_y_dev[gtid]; + Real z0 = pos_z_dev[gtid]; + // Brute force loop to see if particle is alone + for (int i=0;i dx) continue; + if (abs(y0 - pos_y_dev[i]) > dx) continue; + if (abs(z0 - pos_z_dev[i]) > dx) continue; + // If we made it here, something is too close. + return false; + } + return true; +} + + +inline __device__ Real GetAverageDensity(Real* density, int xi, int yi, int zi, + int nx_grid, int ny_grid, int n_ghost) +{ + Real d_average = 0.0; + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + d_average += density[(xi + n_ghost + i) + (yi + n_ghost + j) * nx_grid + + (zi + n_ghost + k) * nx_grid * ny_grid]; + } + } + } + return d_average / 27; +} + + +inline __device__ Real GetAverageNumberDensity_CGS(Real* density, int xi, int yi, + int zi, int nx_grid, int ny_grid, int n_ghost) +{ + return GetAverageDensity(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * DENSITY_UNIT / + (feedback::MU * MP); +} + + +__global__ void initState_kernel(unsigned int seed, + feedback_prng_t* states) +{ + int id = blockIdx.x * blockDim.x + threadIdx.x; + curand_init(seed, id, 0, &states[id]); +} + +#ifndef NO_SN_FEEDBACK +/** + * @brief Does 2 things: + * -# Read in SN rate data from Starburst 99. If no file exists, assume a + * constant rate. + * -# Initialize the cuRAND state, which is analogous to the concept of + * generators in CPU code. The state object maintains configuration and status + * the cuRAND context for each thread on the GPU. Initialize more than the + * number of local particles since the latter will change through MPI transfers. + * + * @param P pointer to parameters struct. Passes in starburst 99 filename and + * random number gen seed. + * @param n_local number of local particles on the GPU + * @param allocation_factor + */ +void feedback::initState(struct parameters* P, part_int_t n_local, + Real allocation_factor) +{ + chprintf("feedback::initState start\n"); + std::string snr_filename(P->snr_filename); + if (not snr_filename.empty()) { + chprintf("Specified a SNR filename %s.\n", snr_filename.data()); + + // read in array of supernova rate values. + std::ifstream snr_in(snr_filename); + if (!snr_in.is_open()) { + chprintf("ERROR: but couldn't read SNR file.\n"); + exit(-1); + } + + std::vector snr_time; + std::vector snr; + + const int N_HEADER = 7; // S'99 has 7 rows of header information + const char* s99_delim = " "; // S'99 data separator + std::string line; + int line_counter = 0; + + while (snr_in.good()) { + std::getline(snr_in, line); + if (line_counter++ < N_HEADER) continue; // skip header processing + + int i = 0; + char* data = strtok(line.data(), s99_delim); + while (data != nullptr) { + if (i == 0) { + // in the following divide by # years per kyr (1000) + snr_time.push_back(std::stof(std::string(data)) / 1000); + } else if (i == 1) { + snr.push_back(pow(10, std::stof(std::string(data))) * 1000 / S_99_TOTAL_MASS); + } + if (i > 0) + break; // only care about the first 2 items. Once i = 1 can break + // here. + data = strtok(nullptr, s99_delim); + i++; + } + } + + time_sn_end = snr_time[snr_time.size() - 1]; + time_sn_start = snr_time[0]; + // the following is the time interval between data points + // (i.e. assumes regular temporal spacing) + snr_dt = (time_sn_end - time_sn_start) / (snr.size() - 1); + + CHECK(cudaMalloc((void**)&dev_snr, snr.size() * sizeof(Real))); + CHECK(cudaMemcpy(dev_snr, snr.data(), snr.size() * sizeof(Real), + cudaMemcpyHostToDevice)); + + } else { + chprintf("No SN rate file specified. Using constant rate\n"); + time_sn_start = DEFAULT_SN_START; + time_sn_end = DEFAULT_SN_END; + } + + // Now initialize the poisson random number generator state. + n_states = n_local * allocation_factor; + cudaMalloc((void**)&randStates, n_states * sizeof(feedback_prng_t)); + + int ngrid = (n_states + TPB_FEEDBACK - 1) / TPB_FEEDBACK; + dim3 grid(ngrid); + dim3 block(TPB_FEEDBACK); + + hipLaunchKernelGGL(initState_kernel, grid, block, 0, 0, P->prng_seed, + randStates); + CHECK(cudaDeviceSynchronize()); + chprintf("feedback::initState end: n_states=%ld, ngrid=%d, threads=%d\n", + n_states, ngrid, TPB_FEEDBACK); +} +#endif // NO_SN_FEEDBACK + +#ifndef NO_WIND_FEEDBACK +/** + * @brief + * Read in Stellar wind data from Starburst 99. If no file exists, assume a + * constant rate. + * + * + * @param P pointer to parameters struct. Passes in starburst 99 filepath + */ + void feedback::initWindState(struct parameters* P) + { + chprintf("initWindState start\n"); + std::string sw_filename(P->sw_filename); + if (sw_filename.empty()) { + chprintf("must specify a stellar wind file.\n"); + exit(-1); + } + + chprintf("Specified a stellar wind filename %s.\n", sw_filename.data()); + + // read in array of supernova rate values. + std::ifstream sw_in(sw_filename); + if (!sw_in.is_open()) { + chprintf("ERROR: couldn't read stellar wind file.\n"); + exit(-1); + } + + std::vector sw_time; + std::vector sw_p; + std::vector sw_e; + + const int N_HEADER_LINES = 7; // S'99 has 7 rows of header information + const int COL_TIME = 0; + const int COL_POWER = 1; + const int COL_ALL_P_FLUX = 7; + + const char* s99_delim = " "; // S'99 data separator + std::string line; + int line_counter = 0; + + while (sw_in.good()) { + std::getline(sw_in, line); + if (line_counter++ < N_HEADER_LINES) continue; // skip header processing + + int i = 0; + char* data = strtok(line.data(), s99_delim); + while (data != nullptr) { + if (i == COL_TIME) { + // in the following divide by # years per kyr (1000) + sw_time.push_back(std::stof(std::string(data)) / 1000); + } else if (i == COL_POWER) { + sw_e.push_back(std::stof(std::string(data))); + } else if (i == COL_ALL_P_FLUX) { + sw_p.push_back(std::stof(std::string(data))); + } + data = strtok(nullptr, s99_delim); + i++; + } + } + + time_sw_end = sw_time[sw_time.size() - 1]; + time_sw_start = sw_time[0]; + // the following is the time interval between data points + // (i.e. assumes regular temporal spacing) + sw_dt = (time_sw_end - time_sw_start) / (sw_p.size() - 1); + chprintf("wind t_s %.5e, t_e %.5e, delta T %0.5e\n", time_sw_start, time_sw_end, sw_dt); + + CHECK(cudaMalloc((void**)&dev_sw_p, sw_p.size() * sizeof(Real))); + CHECK(cudaMemcpy(dev_sw_p, sw_p.data(), sw_p.size() * sizeof(Real), + cudaMemcpyHostToDevice)); + + CHECK(cudaMalloc((void**)&dev_sw_e, sw_e.size() * sizeof(Real))); + CHECK(cudaMemcpy(dev_sw_e, sw_e.data(), sw_e.size() * sizeof(Real), + cudaMemcpyHostToDevice)); + + chprintf("first 40 stellar wind momentum values:\n"); + for (int i = 0; i < 40; i++) { + chprintf("%0.5e %5f %5f \n", sw_time.at(i), sw_e.at(i), sw_p.at(i)); + } + } + + #endif // NO_WIND_FEEDBACK + + + /** + * @brief Get the Starburst 99 stellar wind momentum flux per solar mass. + * + * @param t cluster age in kyr + * @param dev_sw_p device array of log base 10 momentum flux values in dynes. + * @param sw_dt time interval between table data points in kyr. + * @param t_start cluster age when flux becomes non-negligible (kyr). + * @param t_end cluster age when stellar winds turn off (kyr). + * @return flux (in Cholla force units) per solar mass. + */ +__device__ Real GetWindFlux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, + Real t_end) +{ + if (t < t_start || t >= t_end) return 0; + + int index = (int)((t - t_start) / sw_dt); + Real log_p_dynes = dev_sw_p[index] + + (t - index * sw_dt) * (dev_sw_p[index + 1] - dev_sw_p[index]) / sw_dt; + return pow(10, log_p_dynes) / FORCE_UNIT / S_99_TOTAL_MASS; +} + + +/** + * @brief Get the Starburst 99 stellar wind emitted power per solar mass. + * + * @param t cluster age in kyr + * @param dev_sw_e device array of log base 10 power (erg/s). + * @param sw_dt time interval between table data points in kyr. + * @param t_start cluster age when power becomes non-negligible (kyr). + * @param t_end cluster age when stellar winds turn off (kyr). + * @return power (in Cholla units) per solar mass. + */ + __device__ Real GetWindPower(Real t, Real* dev_sw_e, Real sw_dt, Real t_start, + Real t_end) +{ + if (t < t_start || t >= t_end) return 0; + + int index = (int)((t - t_start) / sw_dt); + Real log_e = dev_sw_e[index] + + (t - index * sw_dt) * (dev_sw_e[index + 1] - dev_sw_e[index]) / sw_dt; + Real e = pow(10, log_e) / (MASS_UNIT * VELOCITY_UNIT * VELOCITY_UNIT) * + TIME_UNIT / S_99_TOTAL_MASS; + return e; +} + + +/** + * @brief Get the mass flux associated with stellar wind momentum flux + * and stellar wind power scaled per cluster mass. + * + * @param flux + * @return mass flux in g/s per solar mass + */ + __device__ Real GetWindMass(Real flux, Real power) +{ + if (flux <= 0 || power <= 0) return 0; + return flux * flux / power / 2; +} + +/** + * @brief returns SNR from starburst 99 (or default analytical rate). + * Time is in kyr. Does a basic interpolation of S'99 table + * values. + * + * @param t The cluster age. + * @param dev_snr device array with rate info + * @param snr_dt time interval between table data. Constant value. + * @param t_start cluster age when SNR is greater than zero. + * @param t_end cluster age when SNR drops to zero. + * @return double number of SNe per kyr per solar mass + */ +__device__ Real GetSNRate(Real t, Real* dev_snr, Real snr_dt, Real t_start, + Real t_end) +{ + if (t < t_start || t >= t_end) return 0; + if (dev_snr == nullptr) return feedback::DEFAULT_SNR; + + int index = (int)((t - t_start) / snr_dt); + return dev_snr[index] + + (t - index * snr_dt) * (dev_snr[index + 1] - dev_snr[index]) / snr_dt; +} + + +/** + * @brief Get an actual number of SNe given the expected number. + * Both the simulation step number and cluster ID is used to + * set the state of the random number generator in a unique and + * deterministic way. + * + * @param ave_num_sn expected number of SN, based on cluster + * age, mass and time step. + * @param n_step sim step number + * @param cluster_id + * @return number of supernovae + */ +inline __device__ int GetNumberOfSNeForCluster(Real ave_num_sn, int n_step, + part_int_t cluster_id) +{ + feedback_prng_t state; + curand_init(FEEDBACK_SEED, 0, 0, &state); + unsigned long long skip = n_step * 10000 + cluster_id; + skipahead(skip, &state); // provided by curand + return (int) curand_poisson(&state, ave_num_sn); +} + + +__device__ Real Apply_Resolved_SN(Real pos_x, Real pos_y, Real pos_z, + Real xMin, Real yMin, Real zMin, Real dx, Real dy, Real dz, + int nx_g, int ny_g, int n_ghost, int n_cells, Real gamma, + Real* conserved_device, short time_direction, + Real feedback_density, Real feedback_energy) +{ + // For 2x2x2, a particle between 0-0.5 injects onto cell - 1 + int indx_x = (int)floor((pos_x - xMin - 0.5 * dx) / dx); + int indx_y = (int)floor((pos_y - yMin - 0.5 * dy) / dy); + int indx_z = (int)floor((pos_z - zMin - 0.5 * dz) / dz); + + Real cell_center_x = xMin + indx_x * dx + 0.5 * dx; + Real cell_center_y = yMin + indx_y * dy + 0.5 * dy; + Real cell_center_z = zMin + indx_z * dz + 0.5 * dz; + + Real delta_x = 1 - (pos_x - cell_center_x) / dx; + Real delta_y = 1 - (pos_y - cell_center_y) / dy; + Real delta_z = 1 - (pos_z - cell_center_z) / dz; + + Real* density = conserved_device; + Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; + Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; + Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; + Real* energy = &conserved_device[n_cells*grid_enum::Energy]; + Real* gasEnergy = &conserved_device[n_cells*grid_enum::GasEnergy]; + + Real local_dti = 0; + + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + for (int k = 0; k < 2; k++) { + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + Real x_frac = i * (1 - delta_x) + (1 - i) * delta_x; + Real y_frac = j * (1 - delta_y) + (1 - j) * delta_y; + Real z_frac = k * (1 - delta_z) + (1 - k) * delta_z; + + atomicAdd(&density[indx], + x_frac * y_frac * z_frac * feedback_density); + atomicAdd(&gasEnergy[indx], + x_frac * y_frac * z_frac * feedback_energy); + atomicAdd(&energy[indx], + x_frac * y_frac * z_frac * feedback_energy); + + if (time_direction > 0) { + Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, + momentum_z, energy, indx, dx, dy, dz); + + local_dti = fmax(local_dti, cell_dti); + } + } // k loop + } // j loop + } // i loop + + return local_dti; +} + + +__device__ Real Apply_Unresolved_SN(Real pos_x, Real pos_y, Real pos_z, + Real xMin, Real yMin, Real zMin, Real dx, Real dy, Real dz, + int nx_g, int ny_g, int n_ghost, int n_cells, Real gamma, + Real* conserved_device, short time_direction, + Real feedback_density, Real feedback_momentum, + Real feedback_energy, int indx_x, int indx_y, int indx_z) +{ + + Real delta_x = (pos_x - xMin - indx_x * dx) / dx; + Real delta_y = (pos_y - yMin - indx_y * dy) / dy; + Real delta_z = (pos_z - zMin - indx_z * dz) / dz; + + Real local_dti = 0; + + Real* density = conserved_device; + Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; + Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; + Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; + Real* energy = &conserved_device[n_cells*grid_enum::Energy]; + Real* gas_energy = &conserved_device[n_cells*grid_enum::GasEnergy]; + + Real x_frac, y_frac, z_frac; + Real mag = 0; + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); + y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); + z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + + mag += sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac); + } + } + } + + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + // index in array of conserved quantities + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + + x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); + y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); + z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + + Real px = x_frac * feedback_momentum; + Real py = y_frac * feedback_momentum; + Real pz = z_frac * feedback_momentum; + Real d = sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac) + / mag * feedback_density; + Real e = sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac) + / mag * feedback_energy; + + atomicAdd(&momentum_x[indx], px); + atomicAdd(&momentum_y[indx], py); + atomicAdd(&momentum_z[indx], pz); + atomicAdd(&energy[indx], e); + atomicAdd(&density[indx], d); + + gas_energy[indx] = energy[indx] - (momentum_x[indx]*momentum_x[indx] + + momentum_y[indx]*momentum_y[indx] + + momentum_z[indx]*momentum_z[indx])/(2*density[indx]); + + if (time_direction > 0) { + Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, + momentum_z, energy, indx, dx, dy, dz); + local_dti = fmax(local_dti, cell_dti); + } + } // k loop + } // j loop + } // i loop + + return local_dti; + +} + + +__device__ Real Apply_Wind(Real pos_x, Real pos_y, Real pos_z, Real xMin, + Real yMin, Real zMin, Real dx, Real dy, Real dz, int nx_g, + int ny_g, int n_ghost, int n_cells, Real gamma, + Real* conserved_device, short time_direction, + Real feedback_density, Real feedback_momentum, + Real feedback_energy, int n_step, part_int_t id, int loop, + int indx_x, int indx_y, int indx_z) +{ + + Real delta_x = (pos_x - xMin - indx_x * dx) / dx; + Real delta_y = (pos_y - yMin - indx_y * dy) / dy; + Real delta_z = (pos_z - zMin - indx_z * dz) / dz; + + Real local_dti = 0; + Real f_energy, x_frac, y_frac, z_frac, f_dens; + + Real* density = conserved_device; + Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; + Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; + Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; + Real* energy = &conserved_device[n_cells*grid_enum::Energy]; + Real* gas_energy = &conserved_device[n_cells*grid_enum::GasEnergy]; + + // loop over the 27 cells to add up all the allocated feedback + // momentum magnitudes. For each cell allocate density and + // energy based on the ratio of allocated momentum to this overall sum. + Real mag = 0; + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); + y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); + z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + + mag += sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac); + } + } + } + + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + // index in array of conserved quantities + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + + x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); + y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); + z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + + Real px = x_frac * feedback_momentum; + Real py = y_frac * feedback_momentum; + Real pz = z_frac * feedback_momentum; + f_dens = sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac) + / mag * feedback_density; + f_energy = sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac) + / mag * feedback_energy; + + atomicAdd(&density[indx] , f_dens); + atomicAdd(&momentum_x[indx], px); + atomicAdd(&momentum_y[indx], py); + atomicAdd(&momentum_z[indx], pz); + atomicAdd( &energy[indx], f_energy); + + gas_energy[indx] = energy[indx] - (momentum_x[indx]*momentum_x[indx] + + momentum_y[indx]*momentum_y[indx] + + momentum_z[indx]*momentum_z[indx])/(2*density[indx]); + + /* + energy[indx] = ( momentum_x[indx] * momentum_x[indx] + + momentum_y[indx] * momentum_y[indx] + + momentum_z[indx] * momentum_z[indx] ) / + 2 / density[indx] + gasEnergy[indx]; + */ + if (time_direction > 0) { + Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, + momentum_z, energy, indx, dx, dy, dz); + local_dti = fmax(local_dti, cell_dti); + /* + s_info[FEED_INFO_N * threadIdx.x + i_WIND_ENERGY] += + (energy[indx] - priorEnergy) * dx * dy * dz; + + if (threadIdx.x == 0) { + kernel_printf("WF: mom: %.4e, pe: %.4e, e: %.4e, sinfo: %.7e\n", + feedback_momentum, priorEnergy, energy[indx], + s_info[FEED_INFO_N * threadIdx.x + i_WIND_ENERGY]); + } + */ + } + /* + if (threadIdx.x == 0 && i == -1 && j == 0 && k == 1) { + if (time_direction > 0) { + kernel_printf("~~~~ WF: n_s: %d, id: %lld, l: %d [%d, %d, %d], fe: %.5e, px: %.5e, py: %.5e, pz: %.5e, %.5e\n", + n_step, id, loop, i, j, k, f_energy, px, py, pz, fd); + } else { + kernel_printf("~~~~ -WF: n_s: %d, id: %lld, l: %d [%d, %d, %d], fe: %.5e, px: %.5e, py: %.5e, pz: %.5e, %.5e\n", + n_step, id, loop, i, j, k, f_energy, px, py, pz, fd); + } + } + */ + } // k loop + } // j loop + } // i loop + + return local_dti; +} + + +__device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, + Real* mass_dev, part_int_t* id_dev, Real xMin, Real yMin, Real zMin, + Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, int nx_g, + int ny_g, int nz_g, int n_ghost, int n_step, Real t, Real dt, Real* dti, + Real* dev_snr, Real snr_dt, Real time_sn_start, Real time_sn_end, + Real* prev_dens, short time_direction, Real* s_info, Real* conserved_dev, + Real gamma, int loop, int indx_x, int indx_y, int indx_z) +{ + + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + + Real dV = dx * dy * dz; + Real feedback_density, feedback_momentum, feedback_energy; + Real local_dti = 0.0; + int n_cells = nx_g * ny_g * nz_g; + + Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, + time_sn_end) * mass_dev[gtid] * dt; + int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]) + * time_direction; + /* + if (gtid == 0) { + kernel_printf("SNUMBER n_step: %d, id: %lld, N: %d\n", n_step, id_dev[gtid], N); + } + */ + + // no sense doing anything if there was no SN + if (N != 0) { + Real n_0; + if (time_direction == -1) { + n_0 = prev_dens[gtid]; + } else { + Real* density = conserved_dev; + n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost); + prev_dens[gtid] = n_0; + s_info[FEED_INFO_N * tid] = 1. * N; + } + + feedback_energy = N * feedback::ENERGY_PER_SN / dV; + feedback_density = N * feedback::MASS_PER_SN / dV; + + Real shell_radius = feedback::R_SH * pow(n_0, -0.46) * pow(fabsf(N), 0.29); + bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; + + if (is_resolved) { + // inject energy and density + if (time_direction > 0) { + s_info[FEED_INFO_N * tid + i_RES] = 1.0; + s_info[FEED_INFO_N * tid + i_ENERGY] = feedback_energy * dV; + } + local_dti = Apply_Resolved_SN(pos_x, pos_y, pos_z, + xMin, yMin, zMin, + dx, dy, dz, + nx_g, ny_g, n_ghost, n_cells, + gamma, conserved_dev, + time_direction, + feedback_density, feedback_energy); + } else { + // inject momentum and density + feedback_momentum = time_direction * feedback::FINAL_MOMENTUM * + pow(n_0, -0.17) * pow(fabsf(N), 0.93) / dV / sqrt(3.0); + if (time_direction > 0) { + s_info[FEED_INFO_N * tid + i_UNRES] = time_direction * 1.0; + s_info[FEED_INFO_N * tid + i_MOMENTUM] = feedback_momentum * dV * sqrt(3.0); + s_info[FEED_INFO_N * tid + i_UNRES_ENERGY] = feedback_energy * dV; + } + local_dti = Apply_Unresolved_SN( pos_x, pos_y, pos_z, + xMin, yMin, zMin, + dx, dy, dz, + nx_g, ny_g, n_ghost, n_cells, + gamma, conserved_dev, + time_direction, + feedback_density, feedback_momentum, feedback_energy, + indx_x, indx_y, indx_z); + + } + } + + if (time_direction > 0) atomicMax(dti, local_dti); +} + + +__device__ void Wind_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, + Real* mass_dev, part_int_t* id_dev, Real xMin, Real yMin, Real zMin, + Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, int nx_g, + int ny_g, int nz_g, int n_ghost, int n_step, Real t, Real dt, Real* dti, + Real* dev_sw_p, Real* dev_sw_e, Real sw_dt, Real time_sw_start, + Real time_sw_end, short time_direction, Real* s_info, Real* conserved_dev, + Real gamma, int loop, int indx_x, int indx_y, int indx_z) +{ + + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + + Real dV = dx * dy * dz; + Real feedback_density, feedback_momentum, feedback_energy; + Real local_dti = 0.0; + int n_cells = nx_g * ny_g * nz_g; + + + if (age < 0 || age > time_sw_end) return; + feedback_momentum = GetWindFlux(age, dev_sw_p, sw_dt, time_sw_start, time_sw_end); + // no sense in proceeding if there is no feedback. + if (feedback_momentum == 0) return; + feedback_energy = GetWindPower(age, dev_sw_e, sw_dt, time_sw_start, time_sw_end); + feedback_density = GetWindMass(feedback_momentum, feedback_energy); + + // feedback_momentum now becomes momentum component along one direction. + feedback_momentum *= mass_dev[gtid] * dt / dV / sqrt(3.0) * time_direction; + feedback_density *= mass_dev[gtid] * dt / dV * time_direction; + feedback_energy *= mass_dev[gtid] * dt / dV * time_direction; + + /* TODO refactor into separate kernel call + if (time_direction > 0) { + mass_dev[gtid] -= feedback_density * dV; + }*/ + + if (time_direction > 0) { + // we log net momentum, not momentum density, and magnitude (not the + // component along a direction) + s_info[FEED_INFO_N * tid + i_WIND_MOMENTUM] = + feedback_momentum * dV * sqrt(3.0); + s_info[FEED_INFO_N * tid + i_WIND_ENERGY] = feedback_energy * dV; + } + + //if (tid == 0 ) { + /* + if (abs(feedback_momentum) > 0) { + if (feedback_momentum > 0) { + kernel_printf("WIND: %.5e yrs, %3f log dynes, %f log power \n", + age * 1000, + log10(abs(feedback_momentum)/mass_dev[gtid]*S_99_TOTAL_MASS*dV*sqrt(3.0)*FORCE_UNIT/dt), + log10(abs(feedback_energy)/mass_dev[gtid]*S_99_TOTAL_MASS*dV* + MASS_UNIT*VELOCITY_UNIT*VELOCITY_UNIT/TIME_UNIT/dt)); + } else { + kernel_printf("-WIND: %.5e yrs, %3f log dynes, %f log power \n", + age * 1000, + log10(abs(feedback_momentum)/mass_dev[gtid]*S_99_TOTAL_MASS*dV*sqrt(3.0)*FORCE_UNIT/dt), + log10(abs(feedback_energy)/mass_dev[gtid]*S_99_TOTAL_MASS*dV* + MASS_UNIT*VELOCITY_UNIT*VELOCITY_UNIT/TIME_UNIT/dt)); + } + */ + /* + if (time_direction > 0) { + kernel_printf(" WIND: n_s: %d, id: %lld, l: %d, p: %.9e, e: %.9e, d: %.9e\n", + n_step, id_dev[gtid], loop, feedback_momentum, feedback_energy, feedback_density); + } else { + kernel_printf(" -WIND: n_s: %d, id: %lld, l: %d, p: %.9e, e: %.9e, d: %.9e\n", + n_step, id_dev[gtid], loop, feedback_momentum, feedback_energy, feedback_density); + + } + */ + //} + + local_dti = Apply_Wind( pos_x, pos_y, pos_z, xMin, yMin, zMin, dx, dy, + dz, nx_g, ny_g, n_ghost, n_cells, gamma, conserved_dev, time_direction, + feedback_density, feedback_momentum, feedback_energy, n_step, id_dev[gtid], + loop, indx_x, indx_y, indx_z); + + if (time_direction > 0) atomicMax(dti, local_dti); +} + + +__device__ void Cluster_Feedback_Helper( part_int_t n_local, Real* pos_x_dev, + Real* pos_y_dev, Real* pos_z_dev, Real* age_dev, Real* mass_dev, + part_int_t* id_dev, Real xMin, Real yMin, Real zMin, Real xMax, Real yMax, + Real zMax, Real dx, Real dy, Real dz, int nx_g, int ny_g, int nz_g, + int n_ghost, int n_step, Real t, Real dt, Real* dti, Real* dev_snr, + Real snr_dt, Real time_sn_start, Real time_sn_end, Real* prev_dens, + Real* dev_sw_p, Real* dev_sw_e, Real sw_dt, Real time_sw_start, + Real time_sw_end, short time_direction, Real* s_info, Real* conserved_dev, + Real gamma, int loop ) +{ + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + // Bounds check on particle arrays + if (gtid >= n_local) return; + + Real pos_x = pos_x_dev[gtid]; + Real pos_y = pos_y_dev[gtid]; + Real pos_z = pos_z_dev[gtid]; + bool in_local = (pos_x >= xMin && pos_x < xMax) && + (pos_y >= yMin && pos_y < yMax) && + (pos_z >= zMin && pos_z < zMax); + // Particle is outside bounds, exit + if (!in_local) return; + + int indx_x = (int)floor((pos_x - xMin) / dx); + int indx_y = (int)floor((pos_y - yMin) / dy); + int indx_z = (int)floor((pos_z - zMin) / dz); + bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || + indx_x >= nx_g - 2 * n_ghost || + indx_y >= ny_g - 2 * n_ghost || + indx_z >= nz_g - 2 * n_ghost; + // Ignore this particle, exit + if (ignore) return; + + bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + if (!is_alone) return; + + // note age_dev is actually the time of birth + Real age = t - age_dev[gtid]; + + bool is_sn_feedback = false; + bool is_wd_feedback = false; + #ifndef NO_SN_FEEDBACK + is_sn_feedback = true; + #endif + #ifndef NO_WIND_FEEDBACK + is_wd_feedback = true; + #endif + + // when applying different types of feedback, undoing the step requires + // reverising the order + if (time_direction > 0) { + if (is_sn_feedback) SN_Feedback(pos_x, pos_y, pos_z, age, mass_dev, id_dev, + xMin, yMin, zMin, xMax, yMax, zMax, dx, dy, dz, nx_g, + ny_g, nz_g, n_ghost, n_step, t, dt, dti, + dev_snr, snr_dt, time_sn_start, time_sn_end, + prev_dens, time_direction, s_info, conserved_dev, + gamma, loop, indx_x, indx_y, indx_z); + if (is_wd_feedback) Wind_Feedback(pos_x, pos_y, pos_z, age, mass_dev, id_dev, + xMin, yMin, zMin, xMax, yMax, zMax, dx, dy, dz, nx_g, + ny_g, nz_g, n_ghost, n_step, t, dt, dti, dev_sw_p, + dev_sw_e, sw_dt, time_sw_start, time_sw_end, + time_direction, s_info, conserved_dev, gamma, loop, + indx_x, indx_y, indx_z); + } else { + if (is_wd_feedback) Wind_Feedback(pos_x, pos_y, pos_z, age, mass_dev, id_dev, + xMin, yMin, zMin, xMax, yMax, zMax, dx, dy, dz, nx_g, + ny_g, nz_g, n_ghost, n_step, t, dt, dti, dev_sw_p, + dev_sw_e, sw_dt, time_sw_start, time_sw_end, + time_direction, s_info, conserved_dev, gamma, loop, + indx_x, indx_y, indx_z); + if (is_sn_feedback) SN_Feedback(pos_x, pos_y, pos_z, age, mass_dev, id_dev, + xMin, yMin, zMin, xMax, yMax, zMax, dx, dy, dz, nx_g, + ny_g, nz_g, n_ghost, n_step, t, dt, dti, + dev_snr, snr_dt, time_sn_start, time_sn_end, + prev_dens, time_direction, s_info, conserved_dev, + gamma, loop, indx_x, indx_y, indx_z); + } + + return; +} + + +__global__ void Cluster_Feedback_Kernel( + part_int_t n_local, part_int_t* id_dev, Real* pos_x_dev, Real* pos_y_dev, + Real* pos_z_dev, Real* mass_dev, Real* age_dev, Real xMin, Real yMin, + Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, + int nx_g, int ny_g, int nz_g, int n_ghost, Real t, Real dt, Real* dti, + Real* info, Real* density, Real gamma, Real* prev_dens, + short time_direction, Real* dev_snr, Real snr_dt, Real time_sn_start, + Real time_sn_end, Real* dev_sw_p, Real* dev_sw_e, Real sw_dt, + Real time_sw_start, Real time_sw_end, int n_step, int loop) +{ + + int tid = threadIdx.x; + + // for collecting SN feedback information + __shared__ Real s_info[FEED_INFO_N * TPB_FEEDBACK]; + s_info[FEED_INFO_N * tid] = 0; // number of supernovae + s_info[FEED_INFO_N * tid + 1] = 0; // number of resolved events + s_info[FEED_INFO_N * tid + 2] = 0; // number of unresolved events + s_info[FEED_INFO_N * tid + 3] = 0; // resolved energy + s_info[FEED_INFO_N * tid + 4] = 0; // unresolved momentum + s_info[FEED_INFO_N * tid + 5] = 0; // unresolved KE added via momentum + s_info[FEED_INFO_N * tid + 6] = 0; // wind momentum + s_info[FEED_INFO_N * tid + 7] = 0; // wind energy added + + Cluster_Feedback_Helper(n_local, + pos_x_dev, pos_y_dev, pos_z_dev, age_dev, mass_dev, id_dev, + xMin, yMin, zMin, xMax, yMax, zMax, dx, dy, dz, nx_g, ny_g, nz_g, + n_ghost, n_step, t, dt, dti, dev_snr, snr_dt, time_sn_start, + time_sn_end, prev_dens, dev_sw_p, dev_sw_e, sw_dt, time_sw_start, + time_sw_end,time_direction, s_info, density, gamma, loop); + + __syncthreads(); + + // reduce the info from all the threads in the block + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (tid < s) { + s_info[FEED_INFO_N * tid] += s_info[FEED_INFO_N * (tid + s)]; + s_info[FEED_INFO_N * tid + 1] += s_info[FEED_INFO_N * (tid + s) + 1]; + s_info[FEED_INFO_N * tid + 2] += s_info[FEED_INFO_N * (tid + s) + 2]; + s_info[FEED_INFO_N * tid + 3] += s_info[FEED_INFO_N * (tid + s) + 3]; + s_info[FEED_INFO_N * tid + 4] += s_info[FEED_INFO_N * (tid + s) + 4]; + s_info[FEED_INFO_N * tid + 5] += s_info[FEED_INFO_N * (tid + s) + 5]; + s_info[FEED_INFO_N * tid + 6] += s_info[FEED_INFO_N * (tid + s) + 6]; + s_info[FEED_INFO_N * tid + 7] += s_info[FEED_INFO_N * (tid + s) + 7]; + } + __syncthreads(); + } + + // atomicAdd reduces across all blocks + if (tid == 0) { + atomicAdd(info , s_info[0]); + atomicAdd(info + 1, s_info[1]); + atomicAdd(info + 2, s_info[2]); + atomicAdd(info + 3, s_info[3]); + atomicAdd(info + 4, s_info[4]); + atomicAdd(info + 5, s_info[5]); + atomicAdd(info + 6, s_info[6]); + atomicAdd(info + 7, s_info[7]); + } +} + + +__global__ void Adjust_Cluster_Mass_Kernel(part_int_t n_local, Real* pos_x_dev, + Real* pos_y_dev, Real* pos_z_dev, Real* age_dev, Real* mass_dev, + part_int_t* id_dev, Real xMin, Real yMin, Real zMin, Real xMax, Real yMax, + Real zMax, Real dx, Real dy, Real dz, int nx_g, int ny_g, int nz_g, + int n_ghost, int n_step, Real t, Real dt, Real* dev_snr, + Real snr_dt, Real time_sn_start, Real time_sn_end, + Real* dev_sw_p, Real* dev_sw_e, Real sw_dt, Real time_sw_start, + Real time_sw_end) +{ + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + // Bounds check on particle arrays + if (gtid >= n_local) return; + + Real pos_x = pos_x_dev[gtid]; + Real pos_y = pos_y_dev[gtid]; + Real pos_z = pos_z_dev[gtid]; + bool in_local = (pos_x >= xMin && pos_x < xMax) && + (pos_y >= yMin && pos_y < yMax) && + (pos_z >= zMin && pos_z < zMax); + // Particle is outside bounds, exit + if (!in_local) return; + + int indx_x = (int)floor((pos_x - xMin) / dx); + int indx_y = (int)floor((pos_y - yMin) / dy); + int indx_z = (int)floor((pos_z - zMin) / dz); + bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || + indx_x >= nx_g - 2 * n_ghost || + indx_y >= ny_g - 2 * n_ghost || + indx_z >= nz_g - 2 * n_ghost; + // Ignore this particle, exit + if (ignore) return; + + bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + if (!is_alone) return; + + Real age = t - age_dev[gtid]; + + #ifndef NO_SN_FEEDBACK + Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, + time_sn_end) * mass_dev[gtid] * dt; + int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]); + mass_dev[gtid] -= N * feedback::MASS_PER_SN; + #endif + + #ifndef NO_WIND_FEEDBACK + Real feedback_momentum = GetWindFlux(age, dev_sw_p, sw_dt, time_sw_start, + time_sw_end); + Real feedback_energy = GetWindPower(age, dev_sw_e, sw_dt, time_sw_start, + time_sw_end); + Real feedback_mass_rate = GetWindMass(feedback_momentum, feedback_energy); + + mass_dev[gtid] -= feedback_mass_rate * dt; + #endif +} + + +__device__ void Set_Average_Density(int indx_x, int indx_y, int indx_z, + int nx_g, int ny_g, int n_ghost, + Real* density, Real ave_dens) +{ + for (int i = -1; i < 2; i++) { + for (int j = -1; j < 2; j++) { + for (int k = -1; k < 2; k++) { + int indx = (indx_x + i + n_ghost) + + (indx_y + j + n_ghost) * nx_g + + (indx_z + k + n_ghost) * nx_g * ny_g; + + density[indx] = ave_dens; + } + } + } +} + + +__global__ void Set_Ave_Density_Kernel( + part_int_t n_local, Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, + Real* mass_dev, Real* age_dev, part_int_t* id_dev, Real xMin, Real yMin, + Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, + int nx_g, int ny_g, int nz_g, int n_ghost, Real t, Real dt, Real* density, + Real* dev_snr, Real snr_dt, Real time_sn_start, Real time_sn_end, + Real time_sw_start, Real time_sw_end, int n_step) +{ + int tid = threadIdx.x; + int gtid = blockIdx.x * blockDim.x + tid; + // Bounds check on particle arrays + if (gtid >= n_local) return; + + Real pos_x = pos_x_dev[gtid]; + Real pos_y = pos_y_dev[gtid]; + Real pos_z = pos_z_dev[gtid]; + bool in_local = (pos_x >= xMin && pos_x < xMax) && + (pos_y >= yMin && pos_y < yMax) && + (pos_z >= zMin && pos_z < zMax); + // Particle is outside bounds, exit + if (!in_local) return; + + int indx_x = (int)floor((pos_x - xMin) / dx); + int indx_y = (int)floor((pos_y - yMin) / dy); + int indx_z = (int)floor((pos_z - zMin) / dz); + bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || + indx_x >= nx_g - 2 * n_ghost || + indx_y >= ny_g - 2 * n_ghost || + indx_z >= nz_g - 2 * n_ghost; + // Ignore this particle, exit + if (ignore) return; + + bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + if (!is_alone) return; + + bool is_sn_feedback = false; + bool is_wind_feedback = false; + #ifndef NO_SN_FEEDBACK + is_sn_feedback = true; + #endif + #ifndef NO_WIND_FEEDBACK + is_wind_feedback = true; + #endif + + Real ave_dens; + Real age = t - age_dev[gtid]; + if (is_wind_feedback) { + if (time_sw_start <= age && age <= time_sw_end) { + ave_dens = GetAverageDensity(density, indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost); + Set_Average_Density(indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost, density, ave_dens); + // since we've set the average density, no need to keep + // checking whether we should do so. + return; + } + } + if (is_sn_feedback) { + if (time_sn_start <= age && age <= time_sn_end) { + Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, + time_sn_end) * mass_dev[gtid] * dt; + int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]); + /* + if (gtid == 0) { + kernel_printf("AVEDENS n_step: %d, id: %lld, N: %d\n", n_step, id_dev[gtid], N); + }*/ + Real n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost); + Real shell_radius = feedback::R_SH * pow(n_0, -0.46) * pow(N, 0.29); + bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; + + // resolved SN feedback does not average densities. + if (!is_resolved && N > 0) { + ave_dens = n_0 * feedback::MU * MP / DENSITY_UNIT; + Set_Average_Density(indx_x, indx_y, indx_z, + nx_g, ny_g, n_ghost, density, ave_dens); + } + } + } +} + +/** + * @brief Stellar feedback function (SNe and stellar winds) + * + * @param G + * @param analysis + * @return Real + */ +Real feedback::Cluster_Feedback(Grid3D& G, FeedbackAnalysis& analysis) +{ + #ifdef CPU_TIME + G.Timer.Feedback.Start(); + #endif + + if (G.H.dt == 0) return 0.0; + + Real h_dti = 0.0; + int time_direction, ngrid; + Real h_info[FEED_INFO_N] = {0, 0, 0, 0, 0, 0, 0, 0}; + Real info[FEED_INFO_N]; + Real *d_dti, *d_info; + // require d_prev_dens in case we have to undo feedback if the time + // step is too large. + Real* d_prev_dens; + + // only apply feedback if we have clusters + if (G.Particles.n_local > 0) { + CHECK(cudaMalloc(&d_dti, sizeof(Real))); + CHECK(cudaMemcpy(d_dti, &h_dti, sizeof(Real), cudaMemcpyHostToDevice)); + CHECK(cudaMalloc(&d_prev_dens, G.Particles.n_local * sizeof(Real))); + CHECK(cudaMemset(d_prev_dens, 0, G.Particles.n_local * sizeof(Real))); + + ngrid = std::ceil((1. * G.Particles.n_local) / TPB_FEEDBACK); + CHECK(cudaMalloc((void**)&d_info, FEED_INFO_N * sizeof(Real))); + + // before applying feedback, set gas density around clusters to the + // average value from the 27 neighboring cells. We don't want to + // do this during application of feedback since "undoing it" in the + // event that the time step is too large becomes difficult. + hipLaunchKernelGGL( + Set_Ave_Density_Kernel, ngrid, TPB_FEEDBACK, 0, 0, G.Particles.n_local, + G.Particles.pos_x_dev, G.Particles.pos_y_dev, G.Particles.pos_z_dev, + G.Particles.mass_dev, G.Particles.age_dev, G.Particles.partIDs_dev, + G.H.xblocal, G.H.yblocal, G.H.zblocal, G.H.xblocal_max, G.H.yblocal_max, + G.H.zblocal_max, G.H.dx, G.H.dy, G.H.dz, G.H.nx, G.H.ny, G.H.nz, + G.H.n_ghost, G.H.t, G.H.dt, G.C.d_density, dev_snr, snr_dt, + time_sn_start, time_sn_end, time_sw_start, time_sw_end, G.H.n_step); + } + + int loop_counter = 0; + + do { + time_direction = 1; + loop_counter++; + + if (G.Particles.n_local > 0) { + // always reset d_info to 0 since otherwise do/while looping could add + // values that should have been reverted. + cudaMemset(d_info, 0, FEED_INFO_N * sizeof(Real)); + cudaMemset(d_dti, 0, sizeof(Real)); + hipLaunchKernelGGL( + Cluster_Feedback_Kernel, ngrid, TPB_FEEDBACK, 0, 0, + G.Particles.n_local, G.Particles.partIDs_dev, G.Particles.pos_x_dev, + G.Particles.pos_y_dev, G.Particles.pos_z_dev, G.Particles.mass_dev, + G.Particles.age_dev, G.H.xblocal, G.H.yblocal, G.H.zblocal, + G.H.xblocal_max, G.H.yblocal_max, G.H.zblocal_max, G.H.dx, G.H.dy, + G.H.dz, G.H.nx, G.H.ny, G.H.nz, G.H.n_ghost, G.H.t, G.H.dt, d_dti, + d_info, G.C.d_density, gama, d_prev_dens, time_direction, dev_snr, + snr_dt, time_sn_start, time_sn_end, dev_sw_p, dev_sw_e, sw_dt, + time_sw_start, time_sw_end, G.H.n_step, loop_counter); + + CHECK(cudaMemcpy(&h_dti, d_dti, sizeof(Real), cudaMemcpyDeviceToHost)); + } + + #ifdef MPI_CHOLLA + h_dti = ReduceRealMax(h_dti); + MPI_Barrier(world); + #endif // MPI_CHOLLA + if (h_dti != 0) { + chprintf("+++++++ feed dt = %.12e, H.dt = %.12e\n", C_cfl / h_dti, G.H.dt); + } + + if (h_dti != 0 && (C_cfl / h_dti < G.H.dt)) + { + // timestep too big: need to undo the last operation + time_direction = -1; + if (G.Particles.n_local > 0) { + hipLaunchKernelGGL( + Cluster_Feedback_Kernel, ngrid, TPB_FEEDBACK, 0, 0, + G.Particles.n_local, G.Particles.partIDs_dev, G.Particles.pos_x_dev, + G.Particles.pos_y_dev, G.Particles.pos_z_dev, G.Particles.mass_dev, + G.Particles.age_dev, G.H.xblocal, G.H.yblocal, G.H.zblocal, + G.H.xblocal_max, G.H.yblocal_max, G.H.zblocal_max, G.H.dx, G.H.dy, + G.H.dz, G.H.nx, G.H.ny, G.H.nz, G.H.n_ghost, G.H.t, G.H.dt, d_dti, + d_info, G.C.d_density, gama, d_prev_dens, time_direction, dev_snr, + snr_dt, time_sn_start, time_sn_end, dev_sw_p, dev_sw_e, sw_dt, + time_sw_start, time_sw_end, G.H.n_step, loop_counter); + + CHECK(cudaDeviceSynchronize()); + } + + G.H.dt = C_cfl / h_dti; + if (loop_counter > 2) { // avoid excessive looping + G.H.dt = 0.9 * C_cfl / h_dti; + } + } + } while (time_direction == -1); + + // TODO reduce cluster mass + if (G.Particles.n_local > 0) { + hipLaunchKernelGGL( + Adjust_Cluster_Mass_Kernel, ngrid, TPB_FEEDBACK, 0, 0, + G.Particles.n_local, G.Particles.pos_x_dev, G.Particles.pos_y_dev, + G.Particles.pos_z_dev, G.Particles.age_dev, G.Particles.mass_dev, + G.Particles.partIDs_dev, + G.H.xblocal, G.H.yblocal, G.H.zblocal, G.H.xblocal_max, G.H.yblocal_max, + G.H.zblocal_max, G.H.dx, G.H.dy, G.H.dz, G.H.nx, G.H.ny, G.H.nz, + G.H.n_ghost, G.H.n_step, G.H.t, G.H.dt, dev_snr, snr_dt, time_sn_start, + time_sn_end, dev_sw_p, dev_sw_e, sw_dt, time_sw_start, + time_sw_end); + } + /* + part_int_t n_local, Real* pos_x_dev, + Real* pos_y_dev, Real* pos_z_dev, Real* age_dev, Real* mass_dev, + part_int_t* id_dev, Real xMin, Real yMin, Real zMin, Real xMax, Real yMax, + Real zMax, Real dx, Real dy, Real dz, int nx_g, int ny_g, int nz_g, + int n_ghost, int n_step, Real t, Real dt, Real* dev_snr, + Real snr_dt, Real time_sn_start, Real time_sn_end, + Real* dev_sw_p, Real* dev_sw_e, Real sw_dt, Real time_sw_start, + Real time_sw_end*/ + + chprintf("******* looped %d time(s)\n", loop_counter); + + if (G.Particles.n_local > 0) { + CHECK(cudaMemcpy(&h_info, d_info, FEED_INFO_N * sizeof(Real), + cudaMemcpyDeviceToHost)); + CHECK(cudaFree(d_dti)); + CHECK(cudaFree(d_info)); + CHECK(cudaFree(d_prev_dens)); + } + + #ifdef MPI_CHOLLA + MPI_Reduce(&h_info, &info, FEED_INFO_N, MPI_CHREAL, MPI_SUM, root, world); + #else + info = h_info; + #endif + + #ifdef MPI_CHOLLA // only do stats gathering on root rank + if (procID == 0) { + #endif + + analysis.countSN += (int)info[feedback::SN]; + analysis.countResolved += (int)info[feedback::RESOLVED]; + analysis.countUnresolved += (int)info[feedback::NOT_RESOLVED]; + analysis.totalEnergy += info[feedback::ENERGY]; + analysis.totalMomentum += info[feedback::MOMENTUM]; + analysis.totalUnresEnergy += info[feedback::UNRES_ENERGY]; + analysis.totalWindMomentum += info[i_WIND_MOMENTUM]; + analysis.totalWindEnergy+= info[i_WIND_ENERGY]; + + Real global_resolved_ratio = 0.0; + if (analysis.countResolved > 0 || analysis.countUnresolved > 0) { + global_resolved_ratio = analysis.countResolved / + (analysis.countResolved + analysis.countUnresolved); + } + + chprintf("iteration %d, t %.4e, dt %.4e", G.H.n_step, G.H.t, G.H.dt); + + #ifndef NO_SN_FEEDBACK + chprintf(": number of SN: %d,(R: %d, UR: %d)\n", (int)info[feedback::SN], + (long)info[feedback::RESOLVED], (long)info[feedback::NOT_RESOLVED]); + chprintf( + " cummulative: #SN: %d, ratio of resolved (R: %d, UR: %d) = %.3e\n", + (long)analysis.countSN, (long)analysis.countResolved, + (long)analysis.countUnresolved, global_resolved_ratio); + chprintf(" sn r energy : %.5e erg, cumulative: %.5e erg\n", + info[feedback::ENERGY] * FORCE_UNIT * LENGTH_UNIT, + analysis.totalEnergy * FORCE_UNIT * LENGTH_UNIT); + chprintf(" sn ur energy : %.5e erg, cumulative: %.5e erg\n", + info[feedback::UNRES_ENERGY] * FORCE_UNIT * LENGTH_UNIT, + analysis.totalUnresEnergy * FORCE_UNIT * LENGTH_UNIT); + chprintf(" sn momentum : %.5e SM km/s, cumulative: %.5e SM km/s\n", + info[feedback::MOMENTUM] * VELOCITY_UNIT / 1e5, + analysis.totalMomentum * VELOCITY_UNIT / 1e5); + #endif //NO_SN_FEEDBACK + + #ifndef NO_WIND_FEEDBACK + chprintf( + " wind momentum: %.5e S.M. km/s, cumulative: %.5e S.M. km/s\n", + info[i_WIND_MOMENTUM] * VELOCITY_UNIT / 1e5, + analysis.totalWindMomentum * VELOCITY_UNIT / 1e5); + chprintf( + " wind energy : %.5e erg, cumulative: %.5e erg\n", + info[i_WIND_ENERGY] * FORCE_UNIT * LENGTH_UNIT, + analysis.totalWindEnergy * FORCE_UNIT * LENGTH_UNIT); + #endif //NO_WIND_FEEDBACK + + #ifdef MPI_CHOLLA + } // end if procID == 0 + #endif + + #ifdef CPU_TIME + G.Timer.Feedback.End(); + #endif + + return h_dti; +} + + +#endif // FEEDBACK & PARTICLES_GPU & PARTICLE_IDS & PARTICLE_AGE diff --git a/src/feedback/feedback.h b/src/feedback/feedback.h new file mode 100644 index 000000000..f8c5839dc --- /dev/null +++ b/src/feedback/feedback.h @@ -0,0 +1,55 @@ +#pragma once +#if defined(PARTICLES_GPU) && defined(FEEDBACK) + + #include "../analysis/feedback_analysis.h" + #include "../global/global.h" + #ifdef O_HIP + #include + #include + #else + #include + #include + #endif // O_HIP + + +typedef curandStateMRG32k3a_t feedback_prng_t; + +namespace feedback +{ + const int SN = 0, RESOLVED = 1, NOT_RESOLVED = 2, ENERGY = 3, MOMENTUM = 4, + UNRES_ENERGY = 5; + + // supernova rate: 1SN / 100 solar masses per 36 Myr + static const Real DEFAULT_SNR = 2.8e-7; + static const Real ENERGY_PER_SN = + 1e51 / MASS_UNIT * TIME_UNIT * TIME_UNIT / LENGTH_UNIT / LENGTH_UNIT; + // 10 solarMasses per SN + static const Real MASS_PER_SN = 10.0; + // 2.8e5 M_s km/s * n_0^{-0.17} -> eq.(34) Kim & Ostriker (2015) + static const Real FINAL_MOMENTUM = 2.8e5 / LENGTH_UNIT * 1e5 * TIME_UNIT; + static const Real MU = 0.6; + // 30.2 pc * n_0^{-0.46} -> eq.(31) Kim & Ostriker (2015) + static const Real R_SH = 0.0302; + // default value for when SNe stop (40 Myr) + static const Real DEFAULT_SN_END = 40000; + // default value for when SNe start (4 Myr) + static const Real DEFAULT_SN_START = 4000; + + extern feedback_prng_t* randStates; + extern part_int_t n_states; + extern Real *dev_snr, snr_dt, time_sn_end, time_sn_start; + extern Real *dev_sw_p, *dev_sw_e, sw_dt, time_sw_start, time_sw_end; + + + #ifndef NO_SN_FEEDBACK + void initState(struct parameters* P, part_int_t n_local, + Real allocation_factor = 1); + #endif + #ifndef NO_WIND_FEEDBACK + void initWindState(struct parameters* P); + #endif + Real Cluster_Feedback(Grid3D& G, FeedbackAnalysis& sn_analysis); +} // namespace supernova + + +#endif // PARTICLES_GPU && FEEDBACK diff --git a/src/particles/starburst99_snr.txt b/src/feedback/starburst99_snr.txt similarity index 100% rename from src/particles/starburst99_snr.txt rename to src/feedback/starburst99_snr.txt diff --git a/src/feedback/starburst99_sw.txt b/src/feedback/starburst99_sw.txt new file mode 100755 index 000000000..7434894df --- /dev/null +++ b/src/feedback/starburst99_sw.txt @@ -0,0 +1,1007 @@ + MODEL DESIGNATION: MW_center + MODEL GENERATED: Mon Nov 28 15:05:08 2022 + + RESULTS FOR THE **STELLAR** WIND POWER AND ENERGY + + POWER [ERG/SEC] ENERGY [ERG] MOMENTUM FLUX [DYN] + TIME ALL OB RSG LBV WR ALL ALL OB RSG LBV WR + .10000E+05 39.937 39.937 13.593 -30.000 -30.000 51.436 31.689 31.689 7.419 -30.000 -30.000 + .11000E+06 39.943 39.943 13.593 -30.000 -30.000 52.483 31.699 31.699 7.419 -30.000 -30.000 + .21000E+06 39.950 39.950 13.593 -30.000 -30.000 52.767 31.708 31.708 7.419 -30.000 -30.000 + .31000E+06 39.956 39.956 13.593 -30.000 -30.000 52.940 31.717 31.717 7.419 -30.000 -30.000 + .41000E+06 39.962 39.962 13.593 -30.000 -30.000 53.064 31.726 31.726 7.419 -30.000 -30.000 + .51000E+06 39.968 39.968 13.593 -30.000 -30.000 53.162 31.735 31.735 7.419 -30.000 -30.000 + .61000E+06 39.974 39.974 13.593 -30.000 -30.000 53.243 31.744 31.744 7.419 -30.000 -30.000 + .71000E+06 39.979 39.979 13.593 -30.000 -30.000 53.312 31.753 31.753 7.419 -30.000 -30.000 + .81000E+06 39.985 39.985 13.593 -30.000 -30.000 53.372 31.762 31.762 7.419 -30.000 -30.000 + .91000E+06 39.990 39.990 13.593 -30.000 -30.000 53.426 31.772 31.772 7.419 -30.000 -30.000 + .10100E+07 39.996 39.996 13.593 -30.000 -30.000 53.474 31.781 31.781 7.419 -30.000 -30.000 + .11100E+07 40.001 40.001 13.593 -30.000 -30.000 53.518 31.790 31.790 7.419 -30.000 -30.000 + .12100E+07 40.005 40.005 13.593 -30.000 -30.000 53.558 31.799 31.799 7.419 -30.000 -30.000 + .13100E+07 40.010 40.010 13.593 -30.000 -30.000 53.595 31.808 31.808 7.419 -30.000 -30.000 + .14100E+07 40.013 40.013 13.593 -30.000 -30.000 53.629 31.817 31.817 7.419 -30.000 -30.000 + .15100E+07 40.015 40.015 13.593 -30.000 -30.000 53.662 31.825 31.825 7.419 -30.000 -30.000 + .16100E+07 40.016 40.016 13.593 -30.000 -30.000 53.691 31.832 31.832 7.419 -30.000 -30.000 + .17100E+07 40.016 40.016 13.593 -30.000 -30.000 53.719 31.838 31.838 7.419 -30.000 -30.000 + .18100E+07 40.013 40.013 13.593 -30.000 -30.000 53.746 31.842 31.842 7.419 -30.000 -30.000 + .19100E+07 40.007 40.007 13.593 -30.000 -30.000 53.770 31.844 31.844 7.419 -30.000 -30.000 + .20100E+07 39.997 39.997 13.593 -30.000 -30.000 53.792 31.842 31.842 7.419 -30.000 -30.000 + .21100E+07 39.983 39.983 13.593 -30.000 -30.000 53.813 31.837 31.837 7.419 -30.000 -30.000 + .22100E+07 39.962 39.962 13.593 -30.000 -30.000 53.832 31.824 31.824 7.419 -30.000 -30.000 + .23100E+07 39.932 39.932 13.593 -30.000 -30.000 53.849 31.804 31.804 7.419 -30.000 -30.000 + .24100E+07 39.946 39.946 13.593 -30.000 -30.000 53.866 31.837 31.837 7.419 -30.000 -30.000 + .25100E+07 39.970 39.970 13.593 -30.000 -30.000 53.883 31.887 31.887 7.419 -30.000 -30.000 + .26100E+07 40.016 40.016 13.593 -30.000 -30.000 53.901 31.970 31.970 7.419 -30.000 -30.000 + .27100E+07 40.026 40.026 13.593 -30.000 -30.000 53.919 32.012 32.012 7.419 -30.000 -30.000 + .28100E+07 40.009 40.009 13.593 -30.000 -30.000 53.936 32.022 32.022 7.419 -30.000 -30.000 + .29100E+07 40.094 39.988 13.593 37.803 39.415 53.955 32.144 32.021 7.419 30.803 31.448 + .30100E+07 40.202 39.955 13.593 37.917 39.833 53.978 32.257 31.991 7.419 30.917 31.873 + .31100E+07 40.265 39.923 13.593 37.983 39.997 54.004 32.322 31.963 7.419 30.983 32.036 + .32100E+07 40.306 39.911 13.593 37.950 40.079 54.031 32.343 31.961 7.419 30.950 32.079 + .33100E+07 40.290 39.908 13.593 37.890 40.054 54.055 32.333 31.973 7.419 30.890 32.054 + .34100E+07 40.277 39.908 33.178 37.853 40.031 54.077 32.331 31.996 27.002 30.853 32.033 + .35100E+07 40.221 39.787 33.598 37.809 40.019 54.096 32.265 31.863 27.422 30.809 32.019 + .36100E+07 40.161 39.646 33.541 37.707 40.000 54.111 32.192 31.696 27.364 30.707 32.003 + .37100E+07 40.120 39.536 13.593 37.584 39.987 54.125 32.147 31.580 7.419 30.584 31.992 + .38100E+07 40.068 39.470 34.570 37.504 39.940 54.137 32.099 31.534 28.394 30.504 31.945 + .39100E+07 39.913 39.537 35.331 37.307 39.675 54.145 31.989 31.691 29.155 30.307 31.665 + .40100E+07 39.806 39.491 35.654 37.087 39.516 54.151 31.917 31.665 29.478 30.087 31.544 + .41100E+07 39.721 39.449 35.887 36.287 39.389 54.156 31.867 31.643 29.711 29.287 31.461 + .42100E+07 39.647 39.405 36.022 36.003 39.278 54.161 31.806 31.611 29.845 29.003 31.349 + .43100E+07 39.621 39.351 36.188 35.579 39.286 54.165 31.776 31.554 30.012 28.579 31.358 + .44100E+07 39.578 39.308 36.299 -30.000 39.244 54.168 31.736 31.512 30.123 -30.000 31.314 + .45100E+07 39.542 39.268 36.382 -30.000 39.211 54.171 31.703 31.474 30.206 -30.000 31.279 + .46100E+07 39.492 39.223 36.367 -30.000 39.157 54.174 31.658 31.436 30.191 -30.000 31.221 + .47100E+07 39.434 39.209 36.324 -30.000 39.039 54.177 31.616 31.438 30.148 -30.000 31.095 + .48100E+07 39.395 39.134 36.345 -30.000 39.049 54.179 31.554 31.346 30.169 -30.000 31.084 + .49100E+07 39.383 39.110 36.318 -30.000 39.051 54.181 31.545 31.334 30.142 -30.000 31.082 + .50100E+07 39.332 39.101 36.285 -30.000 38.946 54.183 31.517 31.343 30.109 -30.000 30.981 + .51100E+07 39.278 39.093 36.256 -30.000 38.817 54.185 31.491 31.354 30.080 -30.000 30.858 + .52100E+07 39.199 39.054 36.310 -30.000 38.650 54.186 31.410 31.281 30.134 -30.000 30.720 + .53100E+07 39.155 39.045 36.314 -30.000 38.500 54.188 31.381 31.276 30.138 -30.000 30.574 + .54100E+07 39.141 39.050 36.281 -30.000 38.413 54.189 31.378 31.291 30.105 -30.000 30.488 + .55100E+07 39.110 39.030 36.259 -30.000 38.330 54.190 31.355 31.277 30.083 -30.000 30.401 + .56100E+07 39.073 39.002 36.228 -30.000 38.250 54.191 31.323 31.251 30.051 -30.000 30.316 + .57100E+07 39.017 38.960 36.204 -30.000 38.099 54.192 31.272 31.210 30.028 -30.000 30.155 + .58100E+07 38.961 38.911 36.190 -30.000 37.997 54.193 31.222 31.162 30.014 -30.000 30.042 + .59100E+07 38.903 38.867 36.188 -30.000 37.782 54.193 31.172 31.120 30.011 -30.000 29.805 + .60100E+07 38.827 38.789 36.207 -30.000 37.736 54.194 31.097 31.036 30.031 -30.000 29.758 + .61100E+07 38.816 38.781 36.190 -30.000 37.698 54.195 31.102 31.045 30.014 -30.000 29.720 + .62100E+07 38.836 38.807 36.158 -30.000 37.640 54.195 31.142 31.095 29.982 -30.000 29.662 + .63100E+07 38.847 38.823 36.135 -30.000 37.569 54.196 31.169 31.128 29.959 -30.000 29.591 + .64100E+07 38.750 38.725 36.230 -30.000 37.463 54.196 31.031 30.968 30.054 -30.000 29.485 + .65100E+07 38.712 38.701 36.250 -30.000 37.048 54.197 30.999 30.939 30.074 -30.000 29.070 + .66100E+07 38.682 38.680 36.281 -30.000 -30.000 54.197 30.973 30.910 30.105 -30.000 -30.000 + .67100E+07 38.670 38.669 36.282 -30.000 -30.000 54.197 30.961 30.896 30.106 -30.000 -30.000 + .68100E+07 38.659 38.657 36.289 -30.000 -30.000 54.198 30.950 30.882 30.113 -30.000 -30.000 + .69100E+07 38.626 38.624 36.301 35.558 -30.000 54.198 30.892 30.808 30.124 28.558 -30.000 + .70100E+07 38.628 38.625 36.281 35.910 -30.000 54.199 30.895 30.812 30.105 28.910 -30.000 + .71100E+07 38.630 38.626 36.265 36.090 -30.000 54.199 30.897 30.816 30.089 29.090 -30.000 + .72100E+07 38.633 38.630 36.242 35.868 -30.000 54.199 30.898 30.823 30.066 28.868 -30.000 + .73100E+07 38.636 38.634 36.228 35.311 -30.000 54.200 30.898 30.830 30.052 28.311 -30.000 + .74100E+07 38.635 38.634 36.214 -30.000 -30.000 54.200 30.895 30.830 30.038 -30.000 -30.000 + .75100E+07 38.641 38.640 36.229 -30.000 -30.000 54.200 30.903 30.837 30.053 -30.000 -30.000 + .76100E+07 38.652 38.650 36.199 -30.000 -30.000 54.201 30.911 30.850 30.023 -30.000 -30.000 + .77100E+07 38.654 38.652 36.186 -30.000 -30.000 54.201 30.910 30.851 30.009 -30.000 -30.000 + .78100E+07 38.660 38.659 36.175 -30.000 -30.000 54.202 30.914 30.858 29.999 -30.000 -30.000 + .79100E+07 38.653 38.652 36.163 -30.000 -30.000 54.202 30.906 30.850 29.987 -30.000 -30.000 + .80100E+07 38.627 38.625 36.150 -30.000 -30.000 54.202 30.882 30.824 29.974 -30.000 -30.000 + .81100E+07 38.588 38.586 36.142 -30.000 -30.000 54.203 30.847 30.785 29.966 -30.000 -30.000 + .82100E+07 38.541 38.539 36.139 -30.000 -30.000 54.203 30.802 30.734 29.963 -30.000 -30.000 + .83100E+07 38.498 38.496 36.129 -30.000 -30.000 54.203 30.762 30.689 29.953 -30.000 -30.000 + .84100E+07 38.457 38.455 36.114 -30.000 -30.000 54.204 30.723 30.646 29.938 -30.000 -30.000 + .85100E+07 38.420 38.418 36.098 -30.000 -30.000 54.204 30.688 30.606 29.922 -30.000 -30.000 + .86100E+07 38.379 38.376 36.077 -30.000 -30.000 54.204 30.647 30.561 29.901 -30.000 -30.000 + .87100E+07 38.344 38.342 36.045 -30.000 -30.000 54.204 30.611 30.524 29.869 -30.000 -30.000 + .88100E+07 38.309 38.307 36.006 -30.000 -30.000 54.204 30.574 30.487 29.830 -30.000 -30.000 + .89100E+07 38.275 38.272 35.977 -30.000 -30.000 54.205 30.537 30.449 29.800 -30.000 -30.000 + .90100E+07 38.243 38.240 35.941 -30.000 -30.000 54.205 30.503 30.415 29.765 -30.000 -30.000 + .91100E+07 38.210 38.208 35.909 -30.000 -30.000 54.205 30.468 30.380 29.733 -30.000 -30.000 + .92100E+07 38.176 38.174 35.881 -30.000 -30.000 54.205 30.432 30.342 29.705 -30.000 -30.000 + .93100E+07 38.146 38.144 35.852 -30.000 -30.000 54.205 30.400 30.309 29.676 -30.000 -30.000 + .94100E+07 38.120 38.117 35.822 -30.000 -30.000 54.205 30.371 30.281 29.646 -30.000 -30.000 + .95100E+07 38.090 38.088 35.796 -30.000 -30.000 54.205 30.340 30.248 29.620 -30.000 -30.000 + .96100E+07 38.062 38.059 35.762 -30.000 -30.000 54.205 30.308 30.216 29.586 -30.000 -30.000 + .97100E+07 38.037 38.035 35.734 -30.000 -30.000 54.205 30.281 30.190 29.558 -30.000 -30.000 + .98100E+07 38.007 38.005 35.710 -30.000 -30.000 54.206 30.249 30.156 29.534 -30.000 -30.000 + .99100E+07 37.985 37.983 35.680 -30.000 -30.000 54.206 30.224 30.133 29.504 -30.000 -30.000 + .10010E+08 37.955 37.953 35.660 -30.000 -30.000 54.206 30.193 30.098 29.484 -30.000 -30.000 + .10110E+08 37.928 37.926 35.630 -30.000 -30.000 54.206 30.163 30.068 29.454 -30.000 -30.000 + .10210E+08 37.905 37.903 35.605 -30.000 -30.000 54.206 30.138 30.043 29.429 -30.000 -30.000 + .10310E+08 37.886 37.884 35.580 -30.000 -30.000 54.206 30.117 30.024 29.404 -30.000 -30.000 + .10410E+08 37.857 37.855 35.557 -30.000 -30.000 54.206 30.085 29.989 29.381 -30.000 -30.000 + .10510E+08 37.838 37.836 35.533 -30.000 -30.000 54.206 30.066 29.971 29.357 -30.000 -30.000 + .10610E+08 37.811 37.809 35.510 -30.000 -30.000 54.206 30.036 29.939 29.334 -30.000 -30.000 + .10710E+08 37.792 37.790 35.490 -30.000 -30.000 54.206 30.016 29.920 29.314 -30.000 -30.000 + .10810E+08 37.763 37.761 35.466 -30.000 -30.000 54.206 29.985 29.887 29.290 -30.000 -30.000 + .10910E+08 37.745 37.743 35.443 -30.000 -30.000 54.206 29.966 29.869 29.267 -30.000 -30.000 + .11010E+08 37.725 37.723 35.422 -30.000 -30.000 54.206 29.946 29.849 29.246 -30.000 -30.000 + .11110E+08 37.698 37.695 35.401 -30.000 -30.000 54.206 29.916 29.817 29.225 -30.000 -30.000 + .11210E+08 37.676 37.674 35.380 -30.000 -30.000 54.206 29.894 29.795 29.204 -30.000 -30.000 + .11310E+08 37.654 37.652 35.361 -30.000 -30.000 54.206 29.872 29.772 29.185 -30.000 -30.000 + .11410E+08 37.638 37.635 35.339 -30.000 -30.000 54.206 29.856 29.758 29.163 -30.000 -30.000 + .11510E+08 37.622 37.620 35.320 -30.000 -30.000 54.206 29.842 29.745 29.144 -30.000 -30.000 + .11610E+08 37.597 37.595 35.303 -30.000 -30.000 54.207 29.817 29.718 29.127 -30.000 -30.000 + .11710E+08 37.568 37.565 35.288 -30.000 -30.000 54.207 29.788 29.685 29.112 -30.000 -30.000 + .11810E+08 37.542 37.540 35.270 -30.000 -30.000 54.207 29.763 29.658 29.094 -30.000 -30.000 + .11910E+08 37.508 37.505 35.256 -30.000 -30.000 54.207 29.727 29.616 29.080 -30.000 -30.000 + .12010E+08 37.468 37.465 35.244 -30.000 -30.000 54.207 29.685 29.565 29.068 -30.000 -30.000 + .12110E+08 37.431 37.428 35.232 -30.000 -30.000 54.207 29.646 29.517 29.056 -30.000 -30.000 + .12210E+08 37.376 37.373 35.222 -30.000 -30.000 54.207 29.586 29.438 29.046 -30.000 -30.000 + .12310E+08 37.351 37.348 35.207 -30.000 -30.000 54.207 29.561 29.408 29.031 -30.000 -30.000 + .12410E+08 37.325 37.322 35.194 -30.000 -30.000 54.207 29.535 29.378 29.018 -30.000 -30.000 + .12510E+08 37.301 37.298 35.182 -30.000 -30.000 54.207 29.512 29.349 29.006 -30.000 -30.000 + .12610E+08 37.283 37.279 35.167 -30.000 -30.000 54.207 29.494 29.330 28.991 -30.000 -30.000 + .12710E+08 37.263 37.260 35.154 -30.000 -30.000 54.207 29.476 29.309 28.978 -30.000 -30.000 + .12810E+08 37.245 37.242 35.142 -30.000 -30.000 54.207 29.459 29.290 28.966 -30.000 -30.000 + .12910E+08 37.226 37.222 35.129 -30.000 -30.000 54.207 29.440 29.269 28.953 -30.000 -30.000 + .13010E+08 37.208 37.205 35.117 -30.000 -30.000 54.207 29.424 29.251 28.941 -30.000 -30.000 + .13110E+08 37.193 37.189 35.104 -30.000 -30.000 54.207 29.410 29.236 28.928 -30.000 -30.000 + .13210E+08 37.177 37.174 35.092 -30.000 -30.000 54.207 29.396 29.221 28.916 -30.000 -30.000 + .13310E+08 37.162 37.158 35.081 -30.000 -30.000 54.207 29.382 29.207 28.904 -30.000 -30.000 + .13410E+08 37.147 37.143 35.068 -30.000 -30.000 54.207 29.369 29.192 28.892 -30.000 -30.000 + .13510E+08 37.131 37.128 35.056 -30.000 -30.000 54.207 29.355 29.178 28.880 -30.000 -30.000 + .13610E+08 37.117 37.113 35.045 -30.000 -30.000 54.207 29.342 29.163 28.869 -30.000 -30.000 + .13710E+08 37.101 37.098 35.034 -30.000 -30.000 54.207 29.328 29.149 28.858 -30.000 -30.000 + .13810E+08 37.086 37.082 35.024 -30.000 -30.000 54.207 29.315 29.134 28.847 -30.000 -30.000 + .13910E+08 37.071 37.068 35.012 -30.000 -30.000 54.207 29.302 29.120 28.836 -30.000 -30.000 + .14010E+08 37.056 37.052 35.002 -30.000 -30.000 54.207 29.289 29.106 28.826 -30.000 -30.000 + .14110E+08 37.041 37.038 34.992 -30.000 -30.000 54.207 29.276 29.091 28.815 -30.000 -30.000 + .14210E+08 37.027 37.023 34.981 -30.000 -30.000 54.207 29.263 29.077 28.805 -30.000 -30.000 + .14310E+08 37.012 37.008 34.971 -30.000 -30.000 54.207 29.251 29.063 28.795 -30.000 -30.000 + .14410E+08 36.997 36.993 34.961 -30.000 -30.000 54.207 29.238 29.049 28.785 -30.000 -30.000 + .14510E+08 36.982 36.978 34.952 -30.000 -30.000 54.207 29.225 29.035 28.776 -30.000 -30.000 + .14610E+08 36.967 36.963 34.943 -30.000 -30.000 54.207 29.213 29.020 28.767 -30.000 -30.000 + .14710E+08 36.952 36.948 34.934 -30.000 -30.000 54.207 29.201 29.006 28.758 -30.000 -30.000 + .14810E+08 36.938 36.934 34.924 -30.000 -30.000 54.207 29.188 28.992 28.748 -30.000 -30.000 + .14910E+08 36.923 36.919 34.915 -30.000 -30.000 54.207 29.176 28.978 28.739 -30.000 -30.000 + .15010E+08 36.908 36.904 34.907 -30.000 -30.000 54.207 29.164 28.964 28.731 -30.000 -30.000 + .15110E+08 36.894 36.889 34.898 -30.000 -30.000 54.207 29.152 28.950 28.722 -30.000 -30.000 + .15210E+08 36.879 36.874 34.889 -30.000 -30.000 54.207 29.139 28.935 28.713 -30.000 -30.000 + .15310E+08 36.864 36.860 34.881 -30.000 -30.000 54.207 29.127 28.921 28.705 -30.000 -30.000 + .15410E+08 36.849 36.844 34.872 -30.000 -30.000 54.207 29.115 28.907 28.696 -30.000 -30.000 + .15510E+08 36.834 36.829 34.864 -30.000 -30.000 54.207 29.103 28.892 28.688 -30.000 -30.000 + .15610E+08 36.819 36.815 34.856 -30.000 -30.000 54.207 29.091 28.878 28.680 -30.000 -30.000 + .15710E+08 36.805 36.800 34.848 -30.000 -30.000 54.207 29.080 28.864 28.671 -30.000 -30.000 + .15810E+08 36.791 36.786 34.839 -30.000 -30.000 54.207 29.068 28.851 28.663 -30.000 -30.000 + .15910E+08 36.777 36.773 34.831 -30.000 -30.000 54.207 29.057 28.838 28.655 -30.000 -30.000 + .16010E+08 36.764 36.759 34.821 -30.000 -30.000 54.207 29.045 28.825 28.645 -30.000 -30.000 + .16110E+08 36.750 36.745 34.812 -30.000 -30.000 54.207 29.034 28.811 28.636 -30.000 -30.000 + .16210E+08 36.737 36.732 34.804 -30.000 -30.000 54.207 29.023 28.799 28.627 -30.000 -30.000 + .16310E+08 36.723 36.718 34.795 -30.000 -30.000 54.207 29.011 28.786 28.619 -30.000 -30.000 + .16410E+08 36.710 36.705 34.786 -30.000 -30.000 54.207 29.000 28.773 28.610 -30.000 -30.000 + .16510E+08 36.697 36.691 34.778 -30.000 -30.000 54.207 28.989 28.760 28.601 -30.000 -30.000 + .16610E+08 36.684 36.678 34.769 -30.000 -30.000 54.207 28.978 28.748 28.593 -30.000 -30.000 + .16710E+08 36.671 36.666 34.762 -30.000 -30.000 54.207 28.968 28.736 28.586 -30.000 -30.000 + .16810E+08 36.658 36.653 34.754 -30.000 -30.000 54.207 28.957 28.723 28.577 -30.000 -30.000 + .16910E+08 36.645 36.640 34.746 -30.000 -30.000 54.207 28.947 28.711 28.570 -30.000 -30.000 + .17010E+08 36.632 36.627 34.739 -30.000 -30.000 54.207 28.937 28.698 28.563 -30.000 -30.000 + .17110E+08 36.620 36.615 34.731 -30.000 -30.000 54.207 28.927 28.687 28.555 -30.000 -30.000 + .17210E+08 36.608 36.602 34.723 -30.000 -30.000 54.207 28.917 28.675 28.547 -30.000 -30.000 + .17310E+08 36.595 36.589 34.717 -30.000 -30.000 54.207 28.907 28.663 28.541 -30.000 -30.000 + .17410E+08 36.582 36.576 34.708 -30.000 -30.000 54.207 28.896 28.650 28.532 -30.000 -30.000 + .17510E+08 36.571 36.565 34.698 -30.000 -30.000 54.207 28.886 28.640 28.522 -30.000 -30.000 + .17610E+08 36.559 36.553 34.689 -30.000 -30.000 54.207 28.875 28.628 28.513 -30.000 -30.000 + .17710E+08 36.547 36.541 34.680 -30.000 -30.000 54.207 28.865 28.616 28.504 -30.000 -30.000 + .17810E+08 36.534 36.528 34.672 -30.000 -30.000 54.207 28.854 28.604 28.495 -30.000 -30.000 + .17910E+08 36.523 36.517 34.662 -30.000 -30.000 54.207 28.844 28.593 28.486 -30.000 -30.000 + .18010E+08 36.511 36.505 34.654 -30.000 -30.000 54.207 28.834 28.581 28.478 -30.000 -30.000 + .18110E+08 36.499 36.493 34.645 -30.000 -30.000 54.207 28.824 28.570 28.469 -30.000 -30.000 + .18210E+08 36.487 36.481 34.636 -30.000 -30.000 54.207 28.813 28.559 28.460 -30.000 -30.000 + .18310E+08 36.476 36.469 34.629 -30.000 -30.000 54.207 28.804 28.547 28.453 -30.000 -30.000 + .18410E+08 36.465 36.458 34.620 -30.000 -30.000 54.207 28.794 28.537 28.444 -30.000 -30.000 + .18510E+08 36.453 36.447 34.612 -30.000 -30.000 54.207 28.784 28.525 28.436 -30.000 -30.000 + .18610E+08 36.442 36.435 34.604 -30.000 -30.000 54.207 28.774 28.514 28.428 -30.000 -30.000 + .18710E+08 36.430 36.423 34.595 -30.000 -30.000 54.207 28.764 28.503 28.419 -30.000 -30.000 + .18810E+08 36.418 36.412 34.588 -30.000 -30.000 54.207 28.755 28.492 28.412 -30.000 -30.000 + .18910E+08 36.407 36.401 34.579 -30.000 -30.000 54.207 28.745 28.481 28.403 -30.000 -30.000 + .19010E+08 36.397 36.391 34.571 -30.000 -30.000 54.207 28.736 28.471 28.395 -30.000 -30.000 + .19110E+08 36.385 36.379 34.566 -30.000 -30.000 54.207 28.727 28.460 28.389 -30.000 -30.000 + .19210E+08 36.375 36.368 34.556 -30.000 -30.000 54.207 28.717 28.450 28.380 -30.000 -30.000 + .19310E+08 36.364 36.358 34.550 -30.000 -30.000 54.207 28.709 28.440 28.374 -30.000 -30.000 + .19410E+08 36.353 36.346 34.544 -30.000 -30.000 54.207 28.700 28.428 28.367 -30.000 -30.000 + .19510E+08 36.342 36.335 34.535 -30.000 -30.000 54.207 28.690 28.418 28.358 -30.000 -30.000 + .19610E+08 36.331 36.324 34.528 -30.000 -30.000 54.207 28.682 28.408 28.352 -30.000 -30.000 + .19710E+08 36.321 36.314 34.521 -30.000 -30.000 54.207 28.673 28.397 28.345 -30.000 -30.000 + .19810E+08 36.310 36.303 34.514 -30.000 -30.000 54.207 28.664 28.387 28.338 -30.000 -30.000 + .19910E+08 36.300 36.293 34.507 -30.000 -30.000 54.207 28.656 28.378 28.331 -30.000 -30.000 + .20010E+08 36.290 36.283 34.501 -30.000 -30.000 54.207 28.648 28.367 28.325 -30.000 -30.000 + .20110E+08 36.280 36.272 34.495 -30.000 -30.000 54.207 28.640 28.358 28.318 -30.000 -30.000 + .20210E+08 36.270 36.262 34.487 -30.000 -30.000 54.207 28.631 28.348 28.311 -30.000 -30.000 + .20310E+08 36.260 36.252 34.484 -30.000 -30.000 54.207 28.624 28.338 28.308 -30.000 -30.000 + .20410E+08 36.250 36.243 34.475 -30.000 -30.000 54.207 28.615 28.329 28.299 -30.000 -30.000 + .20510E+08 36.239 36.231 34.469 -30.000 -30.000 54.207 28.607 28.318 28.293 -30.000 -30.000 + .20610E+08 36.229 36.222 34.463 -30.000 -30.000 54.207 28.599 28.309 28.287 -30.000 -30.000 + .20710E+08 36.219 36.212 34.457 -30.000 -30.000 54.207 28.591 28.300 28.281 -30.000 -30.000 + .20810E+08 36.211 36.203 34.450 -30.000 -30.000 54.207 28.584 28.291 28.274 -30.000 -30.000 + .20910E+08 36.200 36.193 34.444 -30.000 -30.000 54.207 28.576 28.281 28.268 -30.000 -30.000 + .21010E+08 36.191 36.183 34.439 -30.000 -30.000 54.207 28.568 28.272 28.263 -30.000 -30.000 + .21110E+08 36.181 36.173 34.433 -30.000 -30.000 54.207 28.561 28.263 28.257 -30.000 -30.000 + .21210E+08 36.172 36.164 34.427 -30.000 -30.000 54.207 28.553 28.254 28.251 -30.000 -30.000 + .21310E+08 36.162 36.154 34.422 -30.000 -30.000 54.207 28.546 28.244 28.245 -30.000 -30.000 + .21410E+08 36.152 36.144 34.416 -30.000 -30.000 54.207 28.538 28.235 28.240 -30.000 -30.000 + .21510E+08 36.142 36.134 34.411 -30.000 -30.000 54.207 28.531 28.225 28.235 -30.000 -30.000 + .21610E+08 36.134 36.125 34.406 -30.000 -30.000 54.207 28.524 28.216 28.230 -30.000 -30.000 + .21710E+08 36.123 36.115 34.401 -30.000 -30.000 54.207 28.517 28.207 28.225 -30.000 -30.000 + .21810E+08 36.115 36.106 34.395 -30.000 -30.000 54.207 28.510 28.198 28.219 -30.000 -30.000 + .21910E+08 36.106 36.097 34.391 -30.000 -30.000 54.207 28.503 28.190 28.215 -30.000 -30.000 + .22010E+08 36.097 36.089 34.386 -30.000 -30.000 54.207 28.497 28.181 28.210 -30.000 -30.000 + .22110E+08 36.087 36.078 34.381 -30.000 -30.000 54.207 28.489 28.171 28.205 -30.000 -30.000 + .22210E+08 36.077 36.069 34.376 -30.000 -30.000 54.207 28.482 28.162 28.200 -30.000 -30.000 + .22310E+08 36.069 36.060 34.370 -30.000 -30.000 54.207 28.476 28.154 28.194 -30.000 -30.000 + .22410E+08 36.060 36.051 34.367 -30.000 -30.000 54.207 28.470 28.146 28.191 -30.000 -30.000 + .22510E+08 36.050 36.041 34.361 -30.000 -30.000 54.207 28.462 28.136 28.185 -30.000 -30.000 + .22610E+08 36.041 36.032 34.356 -30.000 -30.000 54.207 28.456 28.127 28.180 -30.000 -30.000 + .22710E+08 36.032 36.023 34.353 -30.000 -30.000 54.207 28.449 28.118 28.176 -30.000 -30.000 + .22810E+08 36.023 36.014 34.348 -30.000 -30.000 54.207 28.443 28.110 28.172 -30.000 -30.000 + .22910E+08 36.015 36.005 34.344 -30.000 -30.000 54.207 28.437 28.101 28.167 -30.000 -30.000 + .23010E+08 36.006 35.996 34.340 -30.000 -30.000 54.207 28.431 28.093 28.164 -30.000 -30.000 + .23110E+08 35.997 35.988 34.334 -30.000 -30.000 54.207 28.424 28.085 28.158 -30.000 -30.000 + .23210E+08 35.988 35.979 34.323 -30.000 -30.000 54.207 28.414 28.076 28.146 -30.000 -30.000 + .23310E+08 35.980 35.970 34.324 -30.000 -30.000 54.207 28.411 28.068 28.148 -30.000 -30.000 + .23410E+08 35.971 35.961 34.320 -30.000 -30.000 54.207 28.405 28.059 28.144 -30.000 -30.000 + .23510E+08 35.962 35.952 34.316 -30.000 -30.000 54.207 28.399 28.050 28.140 -30.000 -30.000 + .23610E+08 35.954 35.944 34.315 -30.000 -30.000 54.207 28.394 28.042 28.139 -30.000 -30.000 + .23710E+08 35.945 35.935 34.303 -30.000 -30.000 54.207 28.384 28.034 28.127 -30.000 -30.000 + .23810E+08 35.937 35.926 34.304 -30.000 -30.000 54.207 28.381 28.026 28.127 -30.000 -30.000 + .23910E+08 35.927 35.917 34.295 -30.000 -30.000 54.207 28.372 28.017 28.119 -30.000 -30.000 + .24010E+08 35.919 35.909 34.297 -30.000 -30.000 54.207 28.370 28.009 28.121 -30.000 -30.000 + .24110E+08 35.910 35.899 34.288 -30.000 -30.000 54.207 28.360 28.000 28.112 -30.000 -30.000 + .24210E+08 35.902 35.892 34.285 -30.000 -30.000 54.207 28.355 27.992 28.108 -30.000 -30.000 + .24310E+08 35.894 35.884 34.282 -30.000 -30.000 54.207 28.351 27.985 28.106 -30.000 -30.000 + .24410E+08 35.885 35.874 34.279 -30.000 -30.000 54.207 28.345 27.975 28.103 -30.000 -30.000 + .24510E+08 35.876 35.865 34.277 -30.000 -30.000 54.207 28.340 27.967 28.101 -30.000 -30.000 + .24610E+08 35.869 35.857 34.278 -30.000 -30.000 54.207 28.337 27.960 28.102 -30.000 -30.000 + .24710E+08 35.860 35.849 34.270 -30.000 -30.000 54.207 28.329 27.952 28.093 -30.000 -30.000 + .24810E+08 35.853 35.841 34.266 -30.000 -30.000 54.207 28.324 27.944 28.090 -30.000 -30.000 + .24910E+08 35.843 35.832 34.263 -30.000 -30.000 54.207 28.319 27.935 28.087 -30.000 -30.000 + .25010E+08 35.834 35.822 34.259 -30.000 -30.000 54.207 28.312 27.926 28.083 -30.000 -30.000 + .25110E+08 35.827 35.815 34.257 -30.000 -30.000 54.207 28.308 27.919 28.081 -30.000 -30.000 + .25210E+08 35.819 35.807 34.252 -30.000 -30.000 54.207 28.302 27.911 28.076 -30.000 -30.000 + .25310E+08 35.939 35.931 34.244 -30.000 -30.000 54.207 28.383 28.096 28.068 -30.000 -30.000 + .25410E+08 36.054 36.048 34.234 -30.000 -30.000 54.207 28.465 28.250 28.058 -30.000 -30.000 + .25510E+08 36.044 36.037 34.233 -30.000 -30.000 54.207 28.457 28.237 28.057 -30.000 -30.000 + .25610E+08 36.088 36.082 34.225 -30.000 -30.000 54.207 28.490 28.294 28.049 -30.000 -30.000 + .25710E+08 36.087 36.081 34.223 -30.000 -30.000 54.207 28.489 28.295 28.047 -30.000 -30.000 + .25810E+08 36.090 36.084 34.219 -30.000 -30.000 54.207 28.491 28.300 28.043 -30.000 -30.000 + .25910E+08 36.095 36.089 34.215 -30.000 -30.000 54.207 28.494 28.307 28.039 -30.000 -30.000 + .26010E+08 36.089 36.083 34.212 -30.000 -30.000 54.207 28.488 28.300 28.036 -30.000 -30.000 + .26110E+08 36.079 36.073 34.209 -30.000 -30.000 54.207 28.480 28.288 28.033 -30.000 -30.000 + .26210E+08 36.080 36.074 34.208 -30.000 -30.000 54.207 28.482 28.292 28.032 -30.000 -30.000 + .26310E+08 36.085 36.079 34.203 -30.000 -30.000 54.207 28.484 28.298 28.027 -30.000 -30.000 + .26410E+08 36.078 36.072 34.200 -30.000 -30.000 54.207 28.479 28.291 28.024 -30.000 -30.000 + .26510E+08 36.078 36.072 34.197 -30.000 -30.000 54.207 28.478 28.291 28.021 -30.000 -30.000 + .26610E+08 36.069 36.063 34.194 -30.000 -30.000 54.207 28.471 28.281 28.018 -30.000 -30.000 + .26710E+08 36.070 36.064 34.194 -30.000 -30.000 54.207 28.472 28.284 28.018 -30.000 -30.000 + .26810E+08 36.063 36.057 34.193 -30.000 -30.000 54.207 28.467 28.277 28.017 -30.000 -30.000 + .26910E+08 36.065 36.059 34.191 -30.000 -30.000 54.207 28.469 28.280 28.015 -30.000 -30.000 + .27010E+08 36.058 36.052 34.190 -30.000 -30.000 54.207 28.464 28.273 28.014 -30.000 -30.000 + .27110E+08 36.053 36.047 34.189 -30.000 -30.000 54.207 28.460 28.268 28.013 -30.000 -30.000 + .27210E+08 36.046 36.040 34.189 -30.000 -30.000 54.207 28.455 28.260 28.013 -30.000 -30.000 + .27310E+08 36.042 36.036 34.185 -30.000 -30.000 54.207 28.451 28.257 28.009 -30.000 -30.000 + .27410E+08 36.039 36.033 34.185 -30.000 -30.000 54.207 28.450 28.255 28.008 -30.000 -30.000 + .27510E+08 36.032 36.026 34.183 -30.000 -30.000 54.207 28.444 28.247 28.007 -30.000 -30.000 + .27610E+08 36.033 36.026 34.179 -30.000 -30.000 54.207 28.444 28.248 28.003 -30.000 -30.000 + .27710E+08 36.027 36.021 34.178 -30.000 -30.000 54.207 28.440 28.243 28.002 -30.000 -30.000 + .27810E+08 36.025 36.019 34.173 -30.000 -30.000 54.207 28.437 28.241 27.997 -30.000 -30.000 + .27910E+08 36.020 36.014 34.172 -30.000 -30.000 54.207 28.434 28.237 27.996 -30.000 -30.000 + .28010E+08 36.014 36.008 34.170 -30.000 -30.000 54.207 28.429 28.231 27.994 -30.000 -30.000 + .28110E+08 36.013 36.006 34.167 -30.000 -30.000 54.207 28.428 28.230 27.991 -30.000 -30.000 + .28210E+08 36.005 35.999 34.165 -30.000 -30.000 54.207 28.422 28.222 27.989 -30.000 -30.000 + .28310E+08 35.998 35.991 34.163 -30.000 -30.000 54.207 28.416 28.214 27.987 -30.000 -30.000 + .28410E+08 35.994 35.988 34.160 -30.000 -30.000 54.207 28.413 28.211 27.984 -30.000 -30.000 + .28510E+08 35.987 35.980 34.157 -30.000 -30.000 54.207 28.407 28.203 27.981 -30.000 -30.000 + .28610E+08 35.983 35.976 34.155 -30.000 -30.000 54.207 28.404 28.199 27.979 -30.000 -30.000 + .28710E+08 35.980 35.973 34.152 -30.000 -30.000 54.207 28.401 28.197 27.976 -30.000 -30.000 + .28810E+08 35.973 35.967 34.150 -30.000 -30.000 54.207 28.396 28.190 27.974 -30.000 -30.000 + .28910E+08 35.967 35.960 34.147 -30.000 -30.000 54.207 28.391 28.183 27.971 -30.000 -30.000 + .29010E+08 35.959 35.952 34.144 -30.000 -30.000 54.207 28.384 28.174 27.968 -30.000 -30.000 + .29110E+08 35.954 35.947 34.143 -30.000 -30.000 54.207 28.381 28.170 27.967 -30.000 -30.000 + .29210E+08 35.947 35.941 34.139 -30.000 -30.000 54.207 28.375 28.163 27.963 -30.000 -30.000 + .29310E+08 35.943 35.936 34.136 -30.000 -30.000 54.207 28.372 28.159 27.960 -30.000 -30.000 + .29410E+08 35.941 35.934 34.133 -30.000 -30.000 54.207 28.369 28.157 27.957 -30.000 -30.000 + .29510E+08 35.935 35.928 34.129 -30.000 -30.000 54.207 28.364 28.151 27.953 -30.000 -30.000 + .29610E+08 35.931 35.924 34.125 -30.000 -30.000 54.207 28.360 28.147 27.949 -30.000 -30.000 + .29710E+08 35.928 35.921 34.123 -30.000 -30.000 54.207 28.357 28.144 27.947 -30.000 -30.000 + .29810E+08 35.924 35.917 34.120 -30.000 -30.000 54.207 28.354 28.141 27.944 -30.000 -30.000 + .29910E+08 35.925 35.919 34.116 -30.000 -30.000 54.207 28.355 28.144 27.940 -30.000 -30.000 + .30010E+08 35.921 35.915 34.113 -30.000 -30.000 54.207 28.351 28.140 27.937 -30.000 -30.000 + .30110E+08 35.920 35.913 34.110 -30.000 -30.000 54.207 28.349 28.139 27.934 -30.000 -30.000 + .30210E+08 35.918 35.911 34.107 -30.000 -30.000 54.207 28.348 28.138 27.931 -30.000 -30.000 + .30310E+08 35.914 35.908 34.107 -30.000 -30.000 54.207 28.346 28.135 27.930 -30.000 -30.000 + .30410E+08 35.911 35.905 34.106 -30.000 -30.000 54.207 28.344 28.132 27.930 -30.000 -30.000 + .30510E+08 35.910 35.903 34.103 -30.000 -30.000 54.207 28.342 28.132 27.927 -30.000 -30.000 + .30610E+08 35.905 35.898 34.101 -30.000 -30.000 54.207 28.339 28.127 27.925 -30.000 -30.000 + .30710E+08 35.905 35.898 34.099 -30.000 -30.000 54.207 28.338 28.128 27.923 -30.000 -30.000 + .30810E+08 35.900 35.893 34.095 -30.000 -30.000 54.207 28.334 28.124 27.919 -30.000 -30.000 + .30910E+08 35.898 35.891 34.094 -30.000 -30.000 54.207 28.334 28.123 27.918 -30.000 -30.000 + .31010E+08 35.893 35.886 34.091 -30.000 -30.000 54.207 28.329 28.118 27.915 -30.000 -30.000 + .31110E+08 35.892 35.885 34.089 -30.000 -30.000 54.207 28.328 28.117 27.913 -30.000 -30.000 + .31210E+08 35.889 35.882 34.086 -30.000 -30.000 54.207 28.326 28.115 27.910 -30.000 -30.000 + .31310E+08 35.887 35.881 34.083 -30.000 -30.000 54.207 28.324 28.114 27.907 -30.000 -30.000 + .31410E+08 35.883 35.876 34.081 -30.000 -30.000 54.207 28.321 28.110 27.905 -30.000 -30.000 + .31510E+08 35.879 35.872 34.078 -30.000 -30.000 54.207 28.318 28.107 27.902 -30.000 -30.000 + .31610E+08 35.878 35.871 34.076 -30.000 -30.000 54.207 28.317 28.107 27.900 -30.000 -30.000 + .31710E+08 35.874 35.867 34.072 -30.000 -30.000 54.207 28.313 28.103 27.896 -30.000 -30.000 + .31810E+08 35.872 35.865 34.070 -30.000 -30.000 54.207 28.311 28.102 27.894 -30.000 -30.000 + .31910E+08 35.868 35.861 34.067 -30.000 -30.000 54.207 28.308 28.098 27.891 -30.000 -30.000 + .32010E+08 35.867 35.860 34.066 -30.000 -30.000 54.207 28.307 28.097 27.890 -30.000 -30.000 + .32110E+08 35.866 35.859 34.062 -30.000 -30.000 54.207 28.306 28.098 27.886 -30.000 -30.000 + .32210E+08 35.862 35.855 34.061 -30.000 -30.000 54.207 28.303 28.094 27.885 -30.000 -30.000 + .32310E+08 35.858 35.851 34.057 -30.000 -30.000 54.207 28.299 28.091 27.881 -30.000 -30.000 + .32410E+08 35.857 35.850 34.054 -30.000 -30.000 54.207 28.298 28.090 27.878 -30.000 -30.000 + .32510E+08 35.855 35.848 34.051 -30.000 -30.000 54.207 28.296 28.089 27.875 -30.000 -30.000 + .32610E+08 35.854 35.847 34.049 -30.000 -30.000 54.207 28.295 28.089 27.873 -30.000 -30.000 + .32710E+08 35.852 35.845 34.047 -30.000 -30.000 54.207 28.293 28.087 27.871 -30.000 -30.000 + .32810E+08 35.849 35.843 34.044 -30.000 -30.000 54.207 28.291 28.085 27.868 -30.000 -30.000 + .32910E+08 35.847 35.840 34.041 -30.000 -30.000 54.207 28.289 28.083 27.865 -30.000 -30.000 + .33010E+08 35.846 35.839 34.039 -30.000 -30.000 54.207 28.288 28.083 27.863 -30.000 -30.000 + .33110E+08 35.842 35.835 34.036 -30.000 -30.000 54.207 28.284 28.079 27.860 -30.000 -30.000 + .33210E+08 35.841 35.834 34.034 -30.000 -30.000 54.207 28.283 28.079 27.858 -30.000 -30.000 + .33310E+08 35.836 35.829 34.032 -30.000 -30.000 54.207 28.279 28.074 27.856 -30.000 -30.000 + .33410E+08 35.837 35.830 34.029 -30.000 -30.000 54.207 28.279 28.076 27.853 -30.000 -30.000 + .33510E+08 35.833 35.826 34.027 -30.000 -30.000 54.207 28.277 28.073 27.851 -30.000 -30.000 + .33610E+08 35.831 35.824 34.026 -30.000 -30.000 54.207 28.275 28.071 27.850 -30.000 -30.000 + .33710E+08 35.831 35.824 34.023 -30.000 -30.000 54.207 28.275 28.072 27.847 -30.000 -30.000 + .33810E+08 35.829 35.822 34.020 -30.000 -30.000 54.207 28.273 28.070 27.843 -30.000 -30.000 + .33910E+08 35.826 35.820 34.018 -30.000 -30.000 54.207 28.270 28.068 27.842 -30.000 -30.000 + .34010E+08 35.824 35.817 34.015 -30.000 -30.000 54.207 28.268 28.066 27.839 -30.000 -30.000 + .34110E+08 35.824 35.817 34.014 -30.000 -30.000 54.207 28.268 28.066 27.838 -30.000 -30.000 + .34210E+08 35.820 35.813 34.010 -30.000 -30.000 54.207 28.264 28.063 27.834 -30.000 -30.000 + .34310E+08 35.818 35.812 34.009 -30.000 -30.000 54.207 28.263 28.062 27.833 -30.000 -30.000 + .34410E+08 35.817 35.811 34.007 -30.000 -30.000 54.207 28.262 28.062 27.831 -30.000 -30.000 + .34510E+08 35.814 35.807 34.003 -30.000 -30.000 54.207 28.259 28.058 27.827 -30.000 -30.000 + .34610E+08 35.810 35.804 34.002 -30.000 -30.000 54.207 28.256 28.055 27.826 -30.000 -30.000 + .34710E+08 35.808 35.801 34.000 -30.000 -30.000 54.207 28.254 28.053 27.824 -30.000 -30.000 + .34810E+08 35.808 35.802 33.996 -30.000 -30.000 54.207 28.254 28.054 27.820 -30.000 -30.000 + .34910E+08 35.804 35.798 33.995 -30.000 -30.000 54.207 28.251 28.050 27.818 -30.000 -30.000 + .35010E+08 35.803 35.796 33.993 -30.000 -30.000 54.207 28.250 28.049 27.817 -30.000 -30.000 + .35110E+08 35.803 35.797 33.989 -30.000 -30.000 54.207 28.249 28.051 27.813 -30.000 -30.000 + .35210E+08 35.799 35.792 33.987 -30.000 -30.000 54.207 28.245 28.046 27.811 -30.000 -30.000 + .35310E+08 35.798 35.791 33.985 -30.000 -30.000 54.207 28.245 28.046 27.809 -30.000 -30.000 + .35410E+08 35.796 35.789 33.983 -30.000 -30.000 54.207 28.243 28.044 27.807 -30.000 -30.000 + .35510E+08 35.794 35.787 33.982 -30.000 -30.000 54.207 28.241 28.043 27.806 -30.000 -30.000 + .35610E+08 35.793 35.787 33.980 -30.000 -30.000 54.207 28.240 28.043 27.804 -30.000 -30.000 + .35710E+08 35.790 35.783 33.977 -30.000 -30.000 54.207 28.237 28.039 27.801 -30.000 -30.000 + .35810E+08 35.786 35.779 33.975 -30.000 -30.000 54.207 28.234 28.035 27.799 -30.000 -30.000 + .35910E+08 35.787 35.780 33.972 -30.000 -30.000 54.207 28.234 28.037 27.796 -30.000 -30.000 + .36010E+08 35.788 35.781 33.970 -30.000 -30.000 54.207 28.234 28.039 27.794 -30.000 -30.000 + .36110E+08 35.779 35.772 33.968 -30.000 -30.000 54.207 28.228 28.029 27.792 -30.000 -30.000 + .36210E+08 35.777 35.770 33.965 -30.000 -30.000 54.207 28.226 28.028 27.789 -30.000 -30.000 + .36310E+08 35.776 35.770 33.963 -30.000 -30.000 54.207 28.225 28.028 27.787 -30.000 -30.000 + .36410E+08 35.777 35.770 33.962 -30.000 -30.000 54.207 28.225 28.029 27.786 -30.000 -30.000 + .36510E+08 35.775 35.768 33.959 -30.000 -30.000 54.207 28.223 28.027 27.783 -30.000 -30.000 + .36610E+08 35.774 35.768 33.957 -30.000 -30.000 54.207 28.222 28.027 27.781 -30.000 -30.000 + .36710E+08 35.775 35.768 33.955 -30.000 -30.000 54.207 28.223 28.029 27.779 -30.000 -30.000 + .36810E+08 35.767 35.760 33.953 -30.000 -30.000 54.207 28.217 28.021 27.777 -30.000 -30.000 + .36910E+08 35.769 35.762 33.950 -30.000 -30.000 54.207 28.217 28.023 27.774 -30.000 -30.000 + .37010E+08 35.769 35.762 33.948 -30.000 -30.000 54.207 28.217 28.023 27.772 -30.000 -30.000 + .37110E+08 35.763 35.756 33.946 -30.000 -30.000 54.207 28.212 28.018 27.770 -30.000 -30.000 + .37210E+08 35.762 35.756 33.944 -30.000 -30.000 54.207 28.211 28.018 27.768 -30.000 -30.000 + .37310E+08 35.766 35.760 33.942 -30.000 -30.000 54.207 28.214 28.022 27.766 -30.000 -30.000 + .37410E+08 35.766 35.759 33.939 -30.000 -30.000 54.207 28.213 28.022 27.763 -30.000 -30.000 + .37510E+08 35.754 35.747 33.939 -30.000 -30.000 54.207 28.205 28.010 27.763 -30.000 -30.000 + .37610E+08 35.759 35.753 33.935 -30.000 -30.000 54.207 28.208 28.016 27.759 -30.000 -30.000 + .37710E+08 35.753 35.746 33.934 -30.000 -30.000 54.207 28.203 28.010 27.757 -30.000 -30.000 + .37810E+08 35.753 35.746 33.931 -30.000 -30.000 54.207 28.202 28.010 27.755 -30.000 -30.000 + .37910E+08 35.752 35.745 33.930 -30.000 -30.000 54.207 28.201 28.010 27.754 -30.000 -30.000 + .38010E+08 35.748 35.741 33.928 -30.000 -30.000 54.207 28.198 28.005 27.751 -30.000 -30.000 + .38110E+08 35.751 35.745 33.926 -30.000 -30.000 54.207 28.200 28.010 27.750 -30.000 -30.000 + .38210E+08 35.742 35.736 33.923 -30.000 -30.000 54.207 28.193 28.000 27.747 -30.000 -30.000 + .38310E+08 35.745 35.738 33.921 -30.000 -30.000 54.207 28.195 28.004 27.745 -30.000 -30.000 + .38410E+08 35.746 35.740 33.919 -30.000 -30.000 54.207 28.195 28.006 27.743 -30.000 -30.000 + .38510E+08 35.744 35.738 33.917 -30.000 -30.000 54.207 28.193 28.004 27.741 -30.000 -30.000 + .38610E+08 35.736 35.729 33.916 -30.000 -30.000 54.207 28.187 27.995 27.740 -30.000 -30.000 + .38710E+08 35.738 35.731 33.913 -30.000 -30.000 54.207 28.188 27.998 27.737 -30.000 -30.000 + .38810E+08 35.741 35.734 33.911 -30.000 -30.000 54.207 28.190 28.002 27.735 -30.000 -30.000 + .38910E+08 35.739 35.733 33.909 -30.000 -30.000 54.207 28.188 28.000 27.733 -30.000 -30.000 + .39010E+08 35.740 35.734 33.908 -30.000 -30.000 54.207 28.189 28.002 27.731 -30.000 -30.000 + .39110E+08 35.739 35.732 33.905 -30.000 -30.000 54.207 28.187 28.001 27.729 -30.000 -30.000 + .39210E+08 35.731 35.724 33.904 -30.000 -30.000 54.207 28.181 27.992 27.728 -30.000 -30.000 + .39310E+08 35.727 35.720 33.901 -30.000 -30.000 54.207 28.178 27.989 27.725 -30.000 -30.000 + .39410E+08 35.731 35.725 33.899 -30.000 -30.000 54.207 28.180 27.994 27.723 -30.000 -30.000 + .39510E+08 35.728 35.722 33.897 -30.000 -30.000 54.207 28.178 27.991 27.721 -30.000 -30.000 + .39610E+08 35.731 35.724 33.895 -30.000 -30.000 54.207 28.179 27.994 27.719 -30.000 -30.000 + .39710E+08 35.723 35.716 33.893 -30.000 -30.000 54.207 28.173 27.986 27.717 -30.000 -30.000 + .39810E+08 35.724 35.718 33.891 -30.000 -30.000 54.207 28.174 27.988 27.715 -30.000 -30.000 + .39910E+08 35.724 35.718 33.889 -30.000 -30.000 54.207 28.173 27.988 27.713 -30.000 -30.000 + .40010E+08 35.719 35.713 33.887 -30.000 -30.000 54.207 28.169 27.983 27.711 -30.000 -30.000 + .40110E+08 35.719 35.713 33.885 -30.000 -30.000 54.207 28.169 27.984 27.709 -30.000 -30.000 + .40210E+08 35.716 35.710 33.883 -30.000 -30.000 54.207 28.166 27.981 27.707 -30.000 -30.000 + .40310E+08 35.717 35.711 33.882 -30.000 -30.000 54.207 28.167 27.982 27.706 -30.000 -30.000 + .40410E+08 35.716 35.709 33.881 -30.000 -30.000 54.207 28.166 27.981 27.704 -30.000 -30.000 + .40510E+08 35.717 35.710 33.878 -30.000 -30.000 54.207 28.165 27.983 27.702 -30.000 -30.000 + .40610E+08 35.716 35.710 33.876 -30.000 -30.000 54.207 28.165 27.982 27.700 -30.000 -30.000 + .40710E+08 35.705 35.699 33.874 -30.000 -30.000 54.207 28.157 27.971 27.698 -30.000 -30.000 + .40810E+08 35.703 35.696 33.873 -30.000 -30.000 54.207 28.155 27.969 27.697 -30.000 -30.000 + .40910E+08 35.712 35.705 33.870 -30.000 -30.000 54.207 28.160 27.979 27.694 -30.000 -30.000 + .41010E+08 35.710 35.704 33.869 -30.000 -30.000 54.207 28.159 27.978 27.693 -30.000 -30.000 + .41110E+08 35.702 35.695 33.868 -30.000 -30.000 54.207 28.153 27.969 27.692 -30.000 -30.000 + .41210E+08 35.702 35.696 33.865 -30.000 -30.000 54.207 28.153 27.970 27.688 -30.000 -30.000 + .41310E+08 35.697 35.691 33.863 -30.000 -30.000 54.207 28.148 27.964 27.687 -30.000 -30.000 + .41410E+08 35.697 35.691 33.861 -30.000 -30.000 54.207 28.148 27.965 27.685 -30.000 -30.000 + .41510E+08 35.701 35.694 33.859 -30.000 -30.000 54.207 28.150 27.969 27.683 -30.000 -30.000 + .41610E+08 35.696 35.690 33.857 -30.000 -30.000 54.207 28.147 27.965 27.681 -30.000 -30.000 + .41710E+08 35.696 35.690 33.855 -30.000 -30.000 54.207 28.146 27.965 27.679 -30.000 -30.000 + .41810E+08 35.693 35.686 33.853 -30.000 -30.000 54.207 28.143 27.962 27.677 -30.000 -30.000 + .41910E+08 35.693 35.686 33.851 -30.000 -30.000 54.207 28.143 27.962 27.675 -30.000 -30.000 + .42010E+08 35.692 35.686 33.850 -30.000 -30.000 54.207 28.142 27.962 27.673 -30.000 -30.000 + .42110E+08 35.687 35.680 33.848 -30.000 -30.000 54.207 28.138 27.956 27.672 -30.000 -30.000 + .42210E+08 35.684 35.678 33.847 -30.000 -30.000 54.207 28.136 27.954 27.671 -30.000 -30.000 + .42310E+08 35.685 35.679 33.845 -30.000 -30.000 54.207 28.137 27.956 27.669 -30.000 -30.000 + .42410E+08 35.683 35.677 33.843 -30.000 -30.000 54.207 28.135 27.954 27.667 -30.000 -30.000 + .42510E+08 35.685 35.679 33.840 -30.000 -30.000 54.207 28.135 27.956 27.664 -30.000 -30.000 + .42610E+08 35.682 35.676 33.838 -30.000 -30.000 54.207 28.133 27.954 27.662 -30.000 -30.000 + .42710E+08 35.682 35.676 33.837 -30.000 -30.000 54.207 28.133 27.954 27.661 -30.000 -30.000 + .42810E+08 35.679 35.673 33.833 -30.000 -30.000 54.207 28.130 27.951 27.657 -30.000 -30.000 + .42910E+08 35.679 35.673 33.831 -30.000 -30.000 54.207 28.129 27.951 27.655 -30.000 -30.000 + .43010E+08 35.679 35.673 33.828 -30.000 -30.000 54.207 28.128 27.952 27.652 -30.000 -30.000 + .43110E+08 35.680 35.673 33.826 -30.000 -30.000 54.207 28.128 27.952 27.650 -30.000 -30.000 + .43210E+08 35.675 35.668 33.824 -30.000 -30.000 54.207 28.124 27.947 27.648 -30.000 -30.000 + .43310E+08 35.675 35.669 33.821 -30.000 -30.000 54.207 28.123 27.947 27.645 -30.000 -30.000 + .43410E+08 35.669 35.663 33.819 -30.000 -30.000 54.207 28.119 27.942 27.643 -30.000 -30.000 + .43510E+08 35.673 35.667 33.817 -30.000 -30.000 54.207 28.121 27.946 27.641 -30.000 -30.000 + .43610E+08 35.669 35.663 33.815 -30.000 -30.000 54.207 28.117 27.942 27.638 -30.000 -30.000 + .43710E+08 35.661 35.655 33.813 -30.000 -30.000 54.207 28.111 27.934 27.637 -30.000 -30.000 + .43810E+08 35.666 35.660 33.810 -30.000 -30.000 54.207 28.114 27.939 27.634 -30.000 -30.000 + .43910E+08 35.670 35.664 33.807 -30.000 -30.000 54.207 28.116 27.944 27.631 -30.000 -30.000 + .44010E+08 35.668 35.662 33.805 -30.000 -30.000 54.207 28.114 27.942 27.629 -30.000 -30.000 + .44110E+08 35.661 35.655 33.803 -30.000 -30.000 54.207 28.109 27.935 27.627 -30.000 -30.000 + .44210E+08 35.663 35.657 33.801 -30.000 -30.000 54.207 28.109 27.937 27.625 -30.000 -30.000 + .44310E+08 35.668 35.662 33.799 -30.000 -30.000 54.207 28.112 27.942 27.623 -30.000 -30.000 + .44410E+08 35.667 35.661 33.797 -30.000 -30.000 54.207 28.111 27.941 27.621 -30.000 -30.000 + .44510E+08 35.659 35.653 33.795 -30.000 -30.000 54.207 28.105 27.933 27.619 -30.000 -30.000 + .44610E+08 35.657 35.651 33.793 -30.000 -30.000 54.207 28.103 27.931 27.617 -30.000 -30.000 + .44710E+08 35.651 35.645 33.791 -30.000 -30.000 54.207 28.098 27.925 27.615 -30.000 -30.000 + .44810E+08 35.668 35.662 33.788 -30.000 -30.000 54.207 28.109 27.943 27.612 -30.000 -30.000 + .44910E+08 35.666 35.660 33.786 -30.000 -30.000 54.207 28.107 27.941 27.610 -30.000 -30.000 + .45010E+08 35.663 35.657 33.785 -30.000 -30.000 54.207 28.105 27.938 27.609 -30.000 -30.000 + .45110E+08 35.656 35.650 33.783 -30.000 -30.000 54.207 28.099 27.931 27.607 -30.000 -30.000 + .45210E+08 35.646 35.640 33.782 -30.000 -30.000 54.207 28.092 27.920 27.606 -30.000 -30.000 + .45310E+08 35.650 35.644 33.779 -30.000 -30.000 54.207 28.094 27.924 27.603 -30.000 -30.000 + .45410E+08 35.647 35.641 33.778 -30.000 -30.000 54.207 28.092 27.922 27.602 -30.000 -30.000 + .45510E+08 35.658 35.652 33.775 -30.000 -30.000 54.207 28.098 27.933 27.599 -30.000 -30.000 + .45610E+08 35.658 35.652 33.773 -30.000 -30.000 54.207 28.098 27.934 27.597 -30.000 -30.000 + .45710E+08 35.645 35.639 33.773 -30.000 -30.000 54.207 28.089 27.920 27.597 -30.000 -30.000 + .45810E+08 35.645 35.639 33.770 -30.000 -30.000 54.207 28.088 27.920 27.594 -30.000 -30.000 + .45910E+08 35.648 35.642 33.768 -30.000 -30.000 54.207 28.090 27.924 27.592 -30.000 -30.000 + .46010E+08 35.652 35.646 33.766 -30.000 -30.000 54.207 28.092 27.928 27.590 -30.000 -30.000 + .46110E+08 35.652 35.647 33.764 -30.000 -30.000 54.207 28.092 27.929 27.588 -30.000 -30.000 + .46210E+08 35.646 35.640 33.763 -30.000 -30.000 54.207 28.087 27.922 27.587 -30.000 -30.000 + .46310E+08 35.653 35.648 33.761 -30.000 -30.000 54.207 28.092 27.930 27.585 -30.000 -30.000 + .46410E+08 35.643 35.637 33.760 -30.000 -30.000 54.207 28.084 27.919 27.584 -30.000 -30.000 + .46510E+08 35.649 35.643 33.757 -30.000 -30.000 54.207 28.088 27.926 27.581 -30.000 -30.000 + .46610E+08 35.649 35.644 33.756 -30.000 -30.000 54.207 28.088 27.926 27.580 -30.000 -30.000 + .46710E+08 35.650 35.644 33.754 -30.000 -30.000 54.207 28.087 27.927 27.577 -30.000 -30.000 + .46810E+08 35.650 35.645 33.752 -30.000 -30.000 54.207 28.087 27.928 27.576 -30.000 -30.000 + .46910E+08 35.639 35.634 33.751 -30.000 -30.000 54.207 28.079 27.916 27.575 -30.000 -30.000 + .47010E+08 35.643 35.637 33.749 -30.000 -30.000 54.207 28.081 27.920 27.573 -30.000 -30.000 + .47110E+08 35.645 35.640 33.747 -30.000 -30.000 54.207 28.083 27.923 27.571 -30.000 -30.000 + .47210E+08 35.644 35.639 33.746 -30.000 -30.000 54.207 28.081 27.921 27.570 -30.000 -30.000 + .47310E+08 35.643 35.637 33.744 -30.000 -30.000 54.207 28.080 27.920 27.568 -30.000 -30.000 + .47410E+08 35.631 35.626 33.743 -30.000 -30.000 54.207 28.071 27.908 27.567 -30.000 -30.000 + .47510E+08 35.625 35.619 33.742 -30.000 -30.000 54.207 28.067 27.902 27.566 -30.000 -30.000 + .47610E+08 35.644 35.639 33.739 -30.000 -30.000 54.207 28.080 27.922 27.563 -30.000 -30.000 + .47710E+08 35.639 35.633 33.737 -30.000 -30.000 54.207 28.075 27.916 27.561 -30.000 -30.000 + .47810E+08 35.626 35.620 33.737 -30.000 -30.000 54.207 28.066 27.903 27.561 -30.000 -30.000 + .47910E+08 35.633 35.628 33.734 -30.000 -30.000 54.207 28.071 27.911 27.558 -30.000 -30.000 + .48010E+08 35.635 35.630 33.733 -30.000 -30.000 54.207 28.071 27.913 27.557 -30.000 -30.000 + .48110E+08 35.627 35.622 33.731 -30.000 -30.000 54.207 28.066 27.905 27.555 -30.000 -30.000 + .48210E+08 35.628 35.622 33.730 -30.000 -30.000 54.207 28.066 27.906 27.554 -30.000 -30.000 + .48310E+08 35.614 35.608 33.729 -30.000 -30.000 54.207 28.056 27.892 27.553 -30.000 -30.000 + .48410E+08 35.618 35.613 33.727 -30.000 -30.000 54.207 28.058 27.896 27.551 -30.000 -30.000 + .48510E+08 35.606 35.600 33.726 -30.000 -30.000 54.207 28.049 27.884 27.550 -30.000 -30.000 + .48610E+08 35.606 35.600 33.724 -30.000 -30.000 54.207 28.049 27.884 27.548 -30.000 -30.000 + .48710E+08 35.606 35.600 33.723 -30.000 -30.000 54.207 28.048 27.884 27.547 -30.000 -30.000 + .48810E+08 35.605 35.600 33.721 -30.000 -30.000 54.207 28.048 27.884 27.545 -30.000 -30.000 + .48910E+08 35.596 35.590 33.720 -30.000 -30.000 54.207 28.040 27.874 27.544 -30.000 -30.000 + .49010E+08 35.597 35.591 33.718 -30.000 -30.000 54.207 28.041 27.876 27.542 -30.000 -30.000 + .49110E+08 35.598 35.592 33.716 -30.000 -30.000 54.207 28.041 27.877 27.540 -30.000 -30.000 + .49210E+08 35.578 35.572 33.716 -30.000 -30.000 54.207 28.027 27.856 27.540 -30.000 -30.000 + .49310E+08 35.575 35.569 33.714 -30.000 -30.000 54.207 28.025 27.853 27.538 -30.000 -30.000 + .49410E+08 35.576 35.570 33.712 -30.000 -30.000 54.207 28.025 27.854 27.536 -30.000 -30.000 + .49510E+08 35.587 35.581 33.709 -30.000 -30.000 54.207 28.032 27.866 27.533 -30.000 -30.000 + .49610E+08 35.574 35.568 33.708 -30.000 -30.000 54.207 28.022 27.853 27.532 -30.000 -30.000 + .49710E+08 35.573 35.567 33.706 -30.000 -30.000 54.207 28.021 27.852 27.529 -30.000 -30.000 + .49810E+08 35.572 35.566 33.702 -30.000 -30.000 54.207 28.019 27.851 27.526 -30.000 -30.000 + .49910E+08 35.571 35.565 33.696 -30.000 -30.000 54.207 28.017 27.850 27.520 -30.000 -30.000 + .50010E+08 35.562 35.556 33.695 -30.000 -30.000 54.207 28.010 27.841 27.519 -30.000 -30.000 + .50110E+08 35.561 35.555 33.693 -30.000 -30.000 54.207 28.009 27.840 27.517 -30.000 -30.000 + .50210E+08 35.555 35.549 33.692 -30.000 -30.000 54.207 28.005 27.834 27.516 -30.000 -30.000 + .50310E+08 35.554 35.548 33.691 -30.000 -30.000 54.207 28.004 27.833 27.515 -30.000 -30.000 + .50410E+08 35.562 35.556 33.688 -30.000 -30.000 54.207 28.008 27.841 27.512 -30.000 -30.000 + .50510E+08 35.549 35.542 33.687 -30.000 -30.000 54.207 27.999 27.828 27.511 -30.000 -30.000 + .50610E+08 35.547 35.541 33.685 -30.000 -30.000 54.207 27.997 27.827 27.509 -30.000 -30.000 + .50710E+08 35.547 35.541 33.684 -30.000 -30.000 54.207 27.997 27.827 27.508 -30.000 -30.000 + .50810E+08 35.546 35.540 33.681 -30.000 -30.000 54.207 27.995 27.826 27.505 -30.000 -30.000 + .50910E+08 35.534 35.528 33.680 -30.000 -30.000 54.207 27.987 27.814 27.504 -30.000 -30.000 + .51010E+08 35.528 35.522 33.679 -30.000 -30.000 54.207 27.982 27.807 27.503 -30.000 -30.000 + .51110E+08 35.543 35.538 33.676 -30.000 -30.000 54.207 27.993 27.824 27.500 -30.000 -30.000 + .51210E+08 35.526 35.520 33.675 -30.000 -30.000 54.207 27.980 27.806 27.499 -30.000 -30.000 + .51310E+08 35.528 35.522 33.674 -30.000 -30.000 54.207 27.981 27.808 27.498 -30.000 -30.000 + .51410E+08 35.527 35.521 33.672 -30.000 -30.000 54.207 27.980 27.807 27.495 -30.000 -30.000 + .51510E+08 35.526 35.520 33.670 -30.000 -30.000 54.207 27.979 27.807 27.494 -30.000 -30.000 + .51610E+08 35.522 35.515 33.669 -30.000 -30.000 54.207 27.975 27.802 27.493 -30.000 -30.000 + .51710E+08 35.509 35.503 33.668 -30.000 -30.000 54.207 27.966 27.789 27.492 -30.000 -30.000 + .51810E+08 35.507 35.501 33.666 -30.000 -30.000 54.207 27.965 27.787 27.490 -30.000 -30.000 + .51910E+08 35.506 35.500 33.664 -30.000 -30.000 54.207 27.963 27.786 27.488 -30.000 -30.000 + .52010E+08 35.501 35.495 33.662 -30.000 -30.000 54.207 27.959 27.781 27.486 -30.000 -30.000 + .52110E+08 35.500 35.494 33.661 -30.000 -30.000 54.207 27.959 27.781 27.485 -30.000 -30.000 + .52210E+08 35.494 35.487 33.660 -30.000 -30.000 54.207 27.954 27.774 27.483 -30.000 -30.000 + .52310E+08 35.484 35.478 33.659 -30.000 -30.000 54.207 27.947 27.764 27.483 -30.000 -30.000 + .52410E+08 35.492 35.486 33.656 -30.000 -30.000 54.207 27.951 27.772 27.479 -30.000 -30.000 + .52510E+08 35.496 35.490 33.654 -30.000 -30.000 54.207 27.954 27.777 27.478 -30.000 -30.000 + .52610E+08 35.481 35.475 33.653 -30.000 -30.000 54.207 27.943 27.762 27.477 -30.000 -30.000 + .52710E+08 35.466 35.459 33.652 -30.000 -30.000 54.207 27.932 27.746 27.476 -30.000 -30.000 + .52810E+08 35.480 35.473 33.650 -30.000 -30.000 54.207 27.941 27.760 27.474 -30.000 -30.000 + .52910E+08 35.478 35.471 33.648 -30.000 -30.000 54.207 27.939 27.758 27.472 -30.000 -30.000 + .53010E+08 35.478 35.471 33.646 -30.000 -30.000 54.207 27.939 27.759 27.469 -30.000 -30.000 + .53110E+08 35.459 35.452 33.646 -30.000 -30.000 54.207 27.926 27.739 27.470 -30.000 -30.000 + .53210E+08 35.462 35.455 33.644 -30.000 -30.000 54.207 27.928 27.743 27.468 -30.000 -30.000 + .53310E+08 35.475 35.469 33.641 -30.000 -30.000 54.207 27.936 27.757 27.465 -30.000 -30.000 + .53410E+08 35.453 35.446 33.640 -30.000 -30.000 54.207 27.920 27.733 27.464 -30.000 -30.000 + .53510E+08 35.456 35.449 33.639 -30.000 -30.000 54.207 27.922 27.737 27.463 -30.000 -30.000 + .53610E+08 35.451 35.444 33.637 -30.000 -30.000 54.207 27.918 27.732 27.461 -30.000 -30.000 + .53710E+08 35.453 35.446 33.635 -30.000 -30.000 54.207 27.919 27.734 27.459 -30.000 -30.000 + .53810E+08 35.453 35.446 33.634 -30.000 -30.000 54.207 27.919 27.734 27.458 -30.000 -30.000 + .53910E+08 35.452 35.445 33.631 -30.000 -30.000 54.207 27.917 27.734 27.455 -30.000 -30.000 + .54010E+08 35.444 35.437 33.631 -30.000 -30.000 54.207 27.912 27.725 27.454 -30.000 -30.000 + .54110E+08 35.438 35.431 33.629 -30.000 -30.000 54.207 27.907 27.719 27.453 -30.000 -30.000 + .54210E+08 35.434 35.427 33.628 -30.000 -30.000 54.207 27.904 27.715 27.451 -30.000 -30.000 + .54310E+08 35.433 35.426 33.626 -30.000 -30.000 54.207 27.903 27.715 27.449 -30.000 -30.000 + .54410E+08 35.432 35.425 33.625 -30.000 -30.000 54.207 27.902 27.714 27.448 -30.000 -30.000 + .54510E+08 35.431 35.424 33.623 -30.000 -30.000 54.207 27.901 27.713 27.446 -30.000 -30.000 + .54610E+08 35.430 35.423 33.621 -30.000 -30.000 54.207 27.900 27.712 27.445 -30.000 -30.000 + .54710E+08 35.429 35.422 33.619 -30.000 -30.000 54.207 27.899 27.711 27.443 -30.000 -30.000 + .54810E+08 35.409 35.402 33.619 -30.000 -30.000 54.207 27.885 27.690 27.442 -30.000 -30.000 + .54910E+08 35.408 35.401 33.617 -30.000 -30.000 54.207 27.884 27.690 27.441 -30.000 -30.000 + .55010E+08 35.417 35.411 33.615 -30.000 -30.000 54.207 27.889 27.700 27.439 -30.000 -30.000 + .55110E+08 35.406 35.399 33.613 -30.000 -30.000 54.207 27.881 27.688 27.437 -30.000 -30.000 + .55210E+08 35.405 35.398 33.612 -30.000 -30.000 54.207 27.881 27.688 27.436 -30.000 -30.000 + .55310E+08 35.405 35.398 33.610 -30.000 -30.000 54.207 27.880 27.687 27.434 -30.000 -30.000 + .55410E+08 35.404 35.397 33.608 -30.000 -30.000 54.207 27.879 27.686 27.432 -30.000 -30.000 + .55510E+08 35.403 35.396 33.606 -30.000 -30.000 54.207 27.878 27.686 27.430 -30.000 -30.000 + .55610E+08 35.398 35.391 33.605 -30.000 -30.000 54.207 27.874 27.681 27.429 -30.000 -30.000 + .55710E+08 35.401 35.394 33.604 -30.000 -30.000 54.207 27.875 27.684 27.428 -30.000 -30.000 + .55810E+08 35.396 35.389 33.602 -30.000 -30.000 54.207 27.872 27.679 27.426 -30.000 -30.000 + .55910E+08 35.400 35.393 33.601 -30.000 -30.000 54.207 27.874 27.683 27.425 -30.000 -30.000 + .56010E+08 35.388 35.381 33.599 -30.000 -30.000 54.207 27.865 27.671 27.423 -30.000 -30.000 + .56110E+08 35.381 35.374 33.598 -30.000 -30.000 54.207 27.861 27.664 27.422 -30.000 -30.000 + .56210E+08 35.376 35.368 33.597 -30.000 -30.000 54.207 27.857 27.658 27.421 -30.000 -30.000 + .56310E+08 35.375 35.367 33.595 -30.000 -30.000 54.207 27.855 27.657 27.419 -30.000 -30.000 + .56410E+08 35.371 35.364 33.593 -30.000 -30.000 54.207 27.853 27.654 27.417 -30.000 -30.000 + .56510E+08 35.357 35.349 33.593 -30.000 -30.000 54.207 27.843 27.639 27.417 -30.000 -30.000 + .56610E+08 35.356 35.348 33.591 -30.000 -30.000 54.207 27.842 27.638 27.415 -30.000 -30.000 + .56710E+08 35.352 35.344 33.590 -30.000 -30.000 54.207 27.839 27.634 27.414 -30.000 -30.000 + .56810E+08 35.354 35.347 33.588 -30.000 -30.000 54.207 27.840 27.637 27.412 -30.000 -30.000 + .56910E+08 35.354 35.346 33.586 -30.000 -30.000 54.207 27.839 27.637 27.410 -30.000 -30.000 + .57010E+08 35.345 35.337 33.585 -30.000 -30.000 54.207 27.833 27.628 27.409 -30.000 -30.000 + .57110E+08 35.344 35.336 33.584 -30.000 -30.000 54.207 27.832 27.627 27.407 -30.000 -30.000 + .57210E+08 35.343 35.335 33.582 -30.000 -30.000 54.207 27.831 27.626 27.406 -30.000 -30.000 + .57310E+08 35.350 35.343 33.580 -30.000 -30.000 54.207 27.835 27.634 27.404 -30.000 -30.000 + .57410E+08 35.326 35.318 33.579 -30.000 -30.000 54.207 27.819 27.609 27.403 -30.000 -30.000 + .57510E+08 35.345 35.337 33.577 -30.000 -30.000 54.207 27.830 27.629 27.401 -30.000 -30.000 + .57610E+08 35.347 35.340 33.575 -30.000 -30.000 54.207 27.832 27.632 27.399 -30.000 -30.000 + .57710E+08 35.347 35.339 33.573 -30.000 -30.000 54.207 27.831 27.631 27.397 -30.000 -30.000 + .57810E+08 35.328 35.320 33.573 -30.000 -30.000 54.207 27.819 27.612 27.397 -30.000 -30.000 + .57910E+08 35.327 35.320 33.571 -30.000 -30.000 54.207 27.817 27.611 27.395 -30.000 -30.000 + .58010E+08 35.321 35.313 33.569 -30.000 -30.000 54.207 27.813 27.605 27.393 -30.000 -30.000 + .58110E+08 35.326 35.318 33.568 -30.000 -30.000 54.207 27.815 27.610 27.391 -30.000 -30.000 + .58210E+08 35.328 35.321 33.567 -30.000 -30.000 54.207 27.817 27.613 27.391 -30.000 -30.000 + .58310E+08 35.324 35.316 33.565 -30.000 -30.000 54.207 27.814 27.608 27.389 -30.000 -30.000 + .58410E+08 35.315 35.307 33.564 -30.000 -30.000 54.207 27.807 27.599 27.388 -30.000 -30.000 + .58510E+08 35.322 35.315 33.562 -30.000 -30.000 54.207 27.812 27.607 27.386 -30.000 -30.000 + .58610E+08 35.304 35.296 33.561 -30.000 -30.000 54.207 27.800 27.588 27.385 -30.000 -30.000 + .58710E+08 35.303 35.295 33.559 -30.000 -30.000 54.207 27.798 27.588 27.383 -30.000 -30.000 + .58810E+08 35.288 35.280 33.559 -30.000 -30.000 54.207 27.789 27.572 27.383 -30.000 -30.000 + .58910E+08 35.280 35.272 33.558 -30.000 -30.000 54.207 27.784 27.564 27.382 -30.000 -30.000 + .59010E+08 35.276 35.267 33.556 -30.000 -30.000 54.207 27.780 27.559 27.380 -30.000 -30.000 + .59110E+08 35.275 35.266 33.555 -30.000 -30.000 54.207 27.779 27.559 27.379 -30.000 -30.000 + .59210E+08 35.274 35.266 33.553 -30.000 -30.000 54.207 27.778 27.558 27.377 -30.000 -30.000 + .59310E+08 35.273 35.265 33.551 -30.000 -30.000 54.207 27.777 27.557 27.375 -30.000 -30.000 + .59410E+08 35.272 35.264 33.550 -30.000 -30.000 54.207 27.776 27.557 27.374 -30.000 -30.000 + .59510E+08 35.273 35.265 33.549 -30.000 -30.000 54.207 27.776 27.558 27.372 -30.000 -30.000 + .59610E+08 35.272 35.264 33.547 -30.000 -30.000 54.207 27.775 27.557 27.371 -30.000 -30.000 + .59710E+08 35.272 35.263 33.545 -30.000 -30.000 54.207 27.774 27.557 27.369 -30.000 -30.000 + .59810E+08 35.250 35.241 33.545 -30.000 -30.000 54.207 27.760 27.534 27.369 -30.000 -30.000 + .59910E+08 35.267 35.259 33.542 -30.000 -30.000 54.207 27.770 27.553 27.366 -30.000 -30.000 + .60010E+08 35.248 35.240 33.542 -30.000 -30.000 54.207 27.758 27.533 27.366 -30.000 -30.000 + .60110E+08 35.248 35.239 33.540 -30.000 -30.000 54.207 27.757 27.532 27.364 -30.000 -30.000 + .60210E+08 35.247 35.238 33.538 -30.000 -30.000 54.207 27.756 27.532 27.362 -30.000 -30.000 + .60310E+08 35.246 35.237 33.537 -30.000 -30.000 54.207 27.755 27.531 27.361 -30.000 -30.000 + .60410E+08 35.245 35.237 33.536 -30.000 -30.000 54.207 27.754 27.531 27.360 -30.000 -30.000 + .60510E+08 35.244 35.236 33.534 -30.000 -30.000 54.207 27.754 27.530 27.358 -30.000 -30.000 + .60610E+08 35.236 35.227 33.534 -30.000 -30.000 54.207 27.748 27.521 27.357 -30.000 -30.000 + .60710E+08 35.215 35.206 33.532 -30.000 -30.000 54.207 27.735 27.500 27.356 -30.000 -30.000 + .60810E+08 35.198 35.189 33.532 -30.000 -30.000 54.207 27.725 27.482 27.356 -30.000 -30.000 + .60910E+08 35.198 35.188 33.530 -30.000 -30.000 54.207 27.724 27.482 27.354 -30.000 -30.000 + .61010E+08 35.213 35.204 33.528 -30.000 -30.000 54.207 27.732 27.498 27.352 -30.000 -30.000 + .61110E+08 35.212 35.203 33.526 -30.000 -30.000 54.207 27.731 27.497 27.350 -30.000 -30.000 + .61210E+08 35.195 35.186 33.526 -30.000 -30.000 54.207 27.721 27.480 27.350 -30.000 -30.000 + .61310E+08 35.194 35.185 33.524 -30.000 -30.000 54.207 27.720 27.479 27.348 -30.000 -30.000 + .61410E+08 35.188 35.179 33.523 -30.000 -30.000 54.207 27.716 27.473 27.347 -30.000 -30.000 + .61510E+08 35.187 35.177 33.522 -30.000 -30.000 54.207 27.714 27.472 27.346 -30.000 -30.000 + .61610E+08 35.183 35.173 33.520 -30.000 -30.000 54.207 27.711 27.468 27.344 -30.000 -30.000 + .61710E+08 35.182 35.173 33.519 -30.000 -30.000 54.207 27.711 27.467 27.343 -30.000 -30.000 + .61810E+08 35.173 35.163 33.518 -30.000 -30.000 54.207 27.704 27.457 27.342 -30.000 -30.000 + .61910E+08 35.172 35.162 33.517 -30.000 -30.000 54.207 27.704 27.457 27.341 -30.000 -30.000 + .62010E+08 35.166 35.156 33.515 -30.000 -30.000 54.207 27.699 27.450 27.339 -30.000 -30.000 + .62110E+08 35.168 35.158 33.514 -30.000 -30.000 54.207 27.700 27.453 27.338 -30.000 -30.000 + .62210E+08 35.169 35.160 33.513 -30.000 -30.000 54.207 27.701 27.455 27.337 -30.000 -30.000 + .62310E+08 35.167 35.157 33.510 -30.000 -30.000 54.207 27.698 27.452 27.334 -30.000 -30.000 + .62410E+08 35.125 35.114 33.511 -30.000 -30.000 54.207 27.675 27.409 27.335 -30.000 -30.000 + .62510E+08 35.162 35.152 33.508 -30.000 -30.000 54.207 27.695 27.448 27.332 -30.000 -30.000 + .62610E+08 35.151 35.141 33.507 -30.000 -30.000 54.207 27.688 27.437 27.331 -30.000 -30.000 + .62710E+08 35.126 35.115 33.507 -30.000 -30.000 54.207 27.673 27.411 27.330 -30.000 -30.000 + .62810E+08 35.118 35.108 33.505 -30.000 -30.000 54.207 27.669 27.403 27.329 -30.000 -30.000 + .62910E+08 35.080 35.068 33.506 -30.000 -30.000 54.207 27.648 27.363 27.329 -30.000 -30.000 + .63010E+08 35.079 35.068 33.504 -30.000 -30.000 54.207 27.646 27.363 27.328 -30.000 -30.000 + .63110E+08 35.078 35.067 33.503 -30.000 -30.000 54.207 27.646 27.362 27.327 -30.000 -30.000 + .63210E+08 35.078 35.066 33.501 -30.000 -30.000 54.207 27.645 27.361 27.325 -30.000 -30.000 + .63310E+08 35.077 35.065 33.500 -30.000 -30.000 54.207 27.644 27.361 27.324 -30.000 -30.000 + .63410E+08 35.076 35.065 33.499 -30.000 -30.000 54.207 27.643 27.360 27.322 -30.000 -30.000 + .63510E+08 35.101 35.090 33.497 -30.000 -30.000 54.207 27.655 27.386 27.320 -30.000 -30.000 + .63610E+08 35.075 35.063 33.496 -30.000 -30.000 54.207 27.641 27.359 27.320 -30.000 -30.000 + .63710E+08 35.074 35.062 33.494 -30.000 -30.000 54.207 27.640 27.358 27.318 -30.000 -30.000 + .63810E+08 35.074 35.062 33.493 -30.000 -30.000 54.207 27.639 27.358 27.317 -30.000 -30.000 + .63910E+08 35.094 35.083 33.491 -30.000 -30.000 54.207 27.649 27.380 27.315 -30.000 -30.000 + .64010E+08 35.040 35.027 33.491 -30.000 -30.000 54.207 27.620 27.323 27.315 -30.000 -30.000 + .64110E+08 35.039 35.026 33.490 -30.000 -30.000 54.207 27.619 27.322 27.314 -30.000 -30.000 + .64210E+08 35.064 35.052 33.487 -30.000 -30.000 54.207 27.631 27.348 27.311 -30.000 -30.000 + .64310E+08 35.037 35.025 33.487 -30.000 -30.000 54.207 27.617 27.321 27.311 -30.000 -30.000 + .64410E+08 35.062 35.051 33.485 -30.000 -30.000 54.207 27.629 27.347 27.309 -30.000 -30.000 + .64510E+08 35.024 35.011 33.484 -30.000 -30.000 54.207 27.609 27.307 27.308 -30.000 -30.000 + .64610E+08 35.023 35.010 33.484 -30.000 -30.000 54.207 27.608 27.307 27.307 -30.000 -30.000 + .64710E+08 35.022 35.010 33.482 -30.000 -30.000 54.207 27.607 27.306 27.306 -30.000 -30.000 + .64810E+08 35.034 35.021 33.480 -30.000 -30.000 54.207 27.612 27.318 27.304 -30.000 -30.000 + .64910E+08 34.988 34.975 33.480 -30.000 -30.000 54.207 27.589 27.271 27.304 -30.000 -30.000 + .65010E+08 35.036 35.023 33.477 -30.000 -30.000 54.207 27.612 27.321 27.301 -30.000 -30.000 + .65110E+08 34.987 34.973 33.478 -30.000 -30.000 54.207 27.587 27.270 27.302 -30.000 -30.000 + .65210E+08 34.986 34.972 33.476 -30.000 -30.000 54.207 27.586 27.269 27.300 -30.000 -30.000 + .65310E+08 35.003 34.990 33.474 -30.000 -30.000 54.207 27.594 27.287 27.298 -30.000 -30.000 + .65410E+08 34.985 34.971 33.473 -30.000 -30.000 54.207 27.584 27.268 27.297 -30.000 -30.000 + .65510E+08 34.984 34.970 33.472 -30.000 -30.000 54.207 27.583 27.267 27.296 -30.000 -30.000 + .65610E+08 34.983 34.970 33.470 -30.000 -30.000 54.207 27.582 27.267 27.294 -30.000 -30.000 + .65710E+08 34.982 34.969 33.469 -30.000 -30.000 54.207 27.581 27.266 27.293 -30.000 -30.000 + .65810E+08 34.982 34.968 33.468 -30.000 -30.000 54.207 27.580 27.266 27.292 -30.000 -30.000 + .65910E+08 34.981 34.967 33.467 -30.000 -30.000 54.207 27.579 27.265 27.291 -30.000 -30.000 + .66010E+08 34.997 34.984 33.465 -30.000 -30.000 54.207 27.587 27.282 27.289 -30.000 -30.000 + .66110E+08 34.951 34.937 33.465 -30.000 -30.000 54.207 27.564 27.235 27.289 -30.000 -30.000 + .66210E+08 34.895 34.878 33.465 -30.000 -30.000 54.207 27.537 27.176 27.289 -30.000 -30.000 + .66310E+08 34.894 34.878 33.464 -30.000 -30.000 54.207 27.536 27.175 27.288 -30.000 -30.000 + .66410E+08 34.962 34.948 33.461 -30.000 -30.000 54.207 27.567 27.246 27.284 -30.000 -30.000 + .66510E+08 34.849 34.831 33.462 -30.000 -30.000 54.207 27.515 27.129 27.286 -30.000 -30.000 + .66610E+08 34.930 34.915 33.459 -30.000 -30.000 54.207 27.550 27.213 27.283 -30.000 -30.000 + .66710E+08 34.848 34.830 33.459 -30.000 -30.000 54.207 27.513 27.127 27.283 -30.000 -30.000 + .66810E+08 34.890 34.874 33.457 -30.000 -30.000 54.207 27.531 27.172 27.281 -30.000 -30.000 + .66910E+08 34.886 34.870 33.456 -30.000 -30.000 54.207 27.529 27.168 27.280 -30.000 -30.000 + .67010E+08 34.889 34.873 33.455 -30.000 -30.000 54.207 27.529 27.171 27.278 -30.000 -30.000 + .67110E+08 34.888 34.872 33.453 -30.000 -30.000 54.207 27.528 27.170 27.277 -30.000 -30.000 + .67210E+08 34.888 34.871 33.452 -30.000 -30.000 54.207 27.527 27.170 27.276 -30.000 -30.000 + .67310E+08 34.844 34.826 33.452 -30.000 -30.000 54.207 27.507 27.124 27.276 -30.000 -30.000 + .67410E+08 34.796 34.776 33.451 -30.000 -30.000 54.207 27.487 27.074 27.275 -30.000 -30.000 + .67510E+08 34.842 34.824 33.449 -30.000 -30.000 54.207 27.505 27.123 27.273 -30.000 -30.000 + .67610E+08 34.842 34.824 33.448 -30.000 -30.000 54.207 27.505 27.122 27.272 -30.000 -30.000 + .67710E+08 34.791 34.771 33.448 -30.000 -30.000 54.207 27.483 27.069 27.271 -30.000 -30.000 + .67810E+08 34.793 34.774 33.447 -30.000 -30.000 54.207 27.484 27.072 27.270 -30.000 -30.000 + .67910E+08 34.789 34.769 33.445 -30.000 -30.000 54.207 27.481 27.068 27.269 -30.000 -30.000 + .68010E+08 34.839 34.821 33.443 -30.000 -30.000 54.207 27.501 27.120 27.267 -30.000 -30.000 + .68110E+08 34.791 34.772 33.443 -30.000 -30.000 54.207 27.480 27.071 27.266 -30.000 -30.000 + .68210E+08 34.726 34.702 33.442 -30.000 -30.000 54.207 27.455 27.001 27.266 -30.000 -30.000 + .68310E+08 34.725 34.702 33.441 -30.000 -30.000 54.207 27.454 27.001 27.265 -30.000 -30.000 + .68410E+08 34.786 34.766 33.439 -30.000 -30.000 54.207 27.476 27.065 27.263 -30.000 -30.000 + .68510E+08 34.724 34.701 33.439 -30.000 -30.000 54.207 27.452 27.000 27.263 -30.000 -30.000 + .68610E+08 34.723 34.700 33.438 -30.000 -30.000 54.207 27.451 26.999 27.261 -30.000 -30.000 + .68710E+08 34.722 34.699 33.436 -30.000 -30.000 54.207 27.450 26.998 27.260 -30.000 -30.000 + .68810E+08 34.722 34.699 33.435 -30.000 -30.000 54.207 27.449 26.998 27.259 -30.000 -30.000 + .68910E+08 34.640 34.612 33.435 -30.000 -30.000 54.207 27.420 26.911 27.259 -30.000 -30.000 + .69010E+08 34.639 34.611 33.434 -30.000 -30.000 54.207 27.419 26.911 27.258 -30.000 -30.000 + .69110E+08 34.639 34.611 33.432 -30.000 -30.000 54.207 27.418 26.910 27.256 -30.000 -30.000 + .69210E+08 34.642 34.614 33.431 -30.000 -30.000 54.207 27.418 26.913 27.255 -30.000 -30.000 + .69310E+08 34.637 34.610 33.430 -30.000 -30.000 54.207 27.416 26.909 27.254 -30.000 -30.000 + .69410E+08 34.534 34.499 33.430 -30.000 -30.000 54.207 27.384 26.798 27.254 -30.000 -30.000 + .69510E+08 34.614 34.584 33.428 -30.000 -30.000 54.207 27.407 26.884 27.252 -30.000 -30.000 + .69610E+08 34.613 34.583 33.427 -30.000 -30.000 54.207 27.406 26.883 27.251 -30.000 -30.000 + .69710E+08 34.638 34.611 33.425 -30.000 -30.000 54.207 27.413 26.911 27.249 -30.000 -30.000 + .69810E+08 34.634 34.606 33.424 -30.000 -30.000 54.207 27.411 26.907 27.248 -30.000 -30.000 + .69910E+08 34.633 34.606 33.423 -30.000 -30.000 54.207 27.410 26.906 27.247 -30.000 -30.000 + .70010E+08 34.633 34.605 33.421 -30.000 -30.000 54.207 27.409 26.906 27.245 -30.000 -30.000 + .70110E+08 34.533 34.498 33.421 -30.000 -30.000 54.207 27.378 26.798 27.245 -30.000 -30.000 + .70210E+08 34.631 34.604 33.419 -30.000 -30.000 54.207 27.407 26.905 27.243 -30.000 -30.000 + .70310E+08 34.627 34.599 33.418 -30.000 -30.000 54.207 27.405 26.900 27.242 -30.000 -30.000 + .70410E+08 34.527 34.492 33.418 -30.000 -30.000 54.207 27.374 26.793 27.242 -30.000 -30.000 + .70510E+08 34.362 34.309 33.418 -30.000 -30.000 54.207 27.333 26.610 27.242 -30.000 -30.000 + .70610E+08 34.523 34.487 33.415 -30.000 -30.000 54.207 27.371 26.788 27.239 -30.000 -30.000 + .70710E+08 34.427 34.382 33.415 -30.000 -30.000 54.207 27.346 26.683 27.239 -30.000 -30.000 + .70810E+08 34.360 34.307 33.414 -30.000 -30.000 54.207 27.330 26.608 27.238 -30.000 -30.000 + .70910E+08 34.425 34.381 33.413 -30.000 -30.000 54.207 27.344 26.682 27.237 -30.000 -30.000 + .71010E+08 34.263 34.197 33.413 -30.000 -30.000 54.207 27.310 26.498 27.237 -30.000 -30.000 + .71110E+08 34.150 34.063 33.412 -30.000 -30.000 54.207 27.291 26.364 27.236 -30.000 -30.000 + .71210E+08 34.059 33.948 33.412 -30.000 -30.000 54.207 27.278 26.249 27.236 -30.000 -30.000 + .71310E+08 33.412 19.728 33.412 -30.000 -30.000 54.207 27.236 11.933 27.236 -30.000 -30.000 + .71410E+08 33.411 19.694 33.411 -30.000 -30.000 54.207 27.235 11.898 27.235 -30.000 -30.000 + .71510E+08 33.410 19.660 33.410 -30.000 -30.000 54.207 27.234 11.865 27.234 -30.000 -30.000 + .71610E+08 33.408 19.627 33.408 -30.000 -30.000 54.207 27.232 11.832 27.232 -30.000 -30.000 + .71710E+08 33.407 19.600 33.407 -30.000 -30.000 54.207 27.231 11.805 27.231 -30.000 -30.000 + .71810E+08 33.406 19.548 33.406 -30.000 -30.000 54.207 27.230 11.753 27.230 -30.000 -30.000 + .71910E+08 33.405 19.513 33.405 -30.000 -30.000 54.207 27.229 11.718 27.229 -30.000 -30.000 + .72010E+08 33.404 19.478 33.404 -30.000 -30.000 54.207 27.228 11.684 27.228 -30.000 -30.000 + .72110E+08 33.402 19.443 33.402 -30.000 -30.000 54.207 27.226 11.649 27.226 -30.000 -30.000 + .72210E+08 33.401 19.408 33.401 -30.000 -30.000 54.207 27.225 11.614 27.225 -30.000 -30.000 + .72310E+08 33.401 19.376 33.401 -30.000 -30.000 54.207 27.224 11.581 27.224 -30.000 -30.000 + .72410E+08 33.399 19.341 33.399 -30.000 -30.000 54.207 27.223 11.547 27.223 -30.000 -30.000 + .72510E+08 33.398 19.306 33.398 -30.000 -30.000 54.207 27.222 11.512 27.222 -30.000 -30.000 + .72610E+08 33.397 19.272 33.397 -30.000 -30.000 54.207 27.221 11.479 27.221 -30.000 -30.000 + .72710E+08 33.395 19.238 33.395 -30.000 -30.000 54.207 27.219 11.444 27.219 -30.000 -30.000 + .72810E+08 33.394 19.203 33.394 -30.000 -30.000 54.207 27.218 11.410 27.218 -30.000 -30.000 + .72910E+08 33.393 19.170 33.393 -30.000 -30.000 54.207 27.217 11.376 27.217 -30.000 -30.000 + .73010E+08 33.392 19.136 33.392 -30.000 -30.000 54.207 27.216 11.342 27.216 -30.000 -30.000 + .73110E+08 33.391 19.102 33.391 -30.000 -30.000 54.207 27.215 11.309 27.215 -30.000 -30.000 + .73210E+08 33.390 19.071 33.390 -30.000 -30.000 54.207 27.214 11.278 27.214 -30.000 -30.000 + .73310E+08 33.389 19.037 33.389 -30.000 -30.000 54.207 27.212 11.244 27.212 -30.000 -30.000 + .73410E+08 33.388 19.004 33.388 -30.000 -30.000 54.207 27.212 11.211 27.212 -30.000 -30.000 + .73510E+08 33.387 18.971 33.387 -30.000 -30.000 54.207 27.210 11.178 27.210 -30.000 -30.000 + .73610E+08 33.386 18.926 33.386 -30.000 -30.000 54.207 27.210 11.134 27.210 -30.000 -30.000 + .73710E+08 33.384 18.891 33.384 -30.000 -30.000 54.207 27.208 11.099 27.208 -30.000 -30.000 + .73810E+08 33.383 18.857 33.383 -30.000 -30.000 54.207 27.207 11.064 27.207 -30.000 -30.000 + .73910E+08 33.382 18.824 33.382 -30.000 -30.000 54.207 27.206 11.031 27.206 -30.000 -30.000 + .74010E+08 33.381 18.791 33.381 -30.000 -30.000 54.207 27.205 10.998 27.205 -30.000 -30.000 + .74110E+08 33.379 18.756 33.379 -30.000 -30.000 54.207 27.203 10.964 27.203 -30.000 -30.000 + .74210E+08 33.379 18.723 33.379 -30.000 -30.000 54.207 27.202 10.932 27.202 -30.000 -30.000 + .74310E+08 33.377 18.689 33.377 -30.000 -30.000 54.207 27.201 10.898 27.201 -30.000 -30.000 + .74410E+08 33.377 18.656 33.377 -30.000 -30.000 54.207 27.200 10.864 27.200 -30.000 -30.000 + .74510E+08 33.375 18.622 33.375 -30.000 -30.000 54.207 27.199 10.831 27.199 -30.000 -30.000 + .74610E+08 33.375 18.589 33.375 -30.000 -30.000 54.207 27.198 10.797 27.198 -30.000 -30.000 + .74710E+08 33.373 18.557 33.373 -30.000 -30.000 54.207 27.197 10.766 27.197 -30.000 -30.000 + .74810E+08 33.372 18.538 33.372 -30.000 -30.000 54.207 27.196 10.747 27.196 -30.000 -30.000 + .74910E+08 33.370 18.505 33.370 -30.000 -30.000 54.207 27.194 10.714 27.194 -30.000 -30.000 + .75010E+08 33.370 18.458 33.370 -30.000 -30.000 54.207 27.194 10.668 27.194 -30.000 -30.000 + .75110E+08 33.369 18.425 33.369 -30.000 -30.000 54.207 27.193 10.635 27.193 -30.000 -30.000 + .75210E+08 33.368 18.392 33.368 -30.000 -30.000 54.207 27.192 10.602 27.192 -30.000 -30.000 + .75310E+08 33.367 18.360 33.367 -30.000 -30.000 54.207 27.190 10.570 27.190 -30.000 -30.000 + .75410E+08 33.365 18.327 33.365 -30.000 -30.000 54.207 27.189 10.537 27.189 -30.000 -30.000 + .75510E+08 33.364 18.295 33.364 -30.000 -30.000 54.207 27.188 10.505 27.188 -30.000 -30.000 + .75610E+08 33.363 18.262 33.363 -30.000 -30.000 54.207 27.187 10.472 27.187 -30.000 -30.000 + .75710E+08 33.362 18.230 33.362 -30.000 -30.000 54.207 27.186 10.441 27.186 -30.000 -30.000 + .75810E+08 33.361 18.198 33.361 -30.000 -30.000 54.207 27.185 10.409 27.185 -30.000 -30.000 + .75910E+08 33.360 18.165 33.360 -30.000 -30.000 54.207 27.184 10.376 27.184 -30.000 -30.000 + .76010E+08 33.359 18.135 33.359 -30.000 -30.000 54.207 27.183 10.346 27.183 -30.000 -30.000 + .76110E+08 33.358 18.102 33.358 -30.000 -30.000 54.207 27.181 10.313 27.181 -30.000 -30.000 + .76210E+08 33.357 18.070 33.357 -30.000 -30.000 54.207 27.181 10.282 27.181 -30.000 -30.000 + .76310E+08 33.356 18.039 33.356 -30.000 -30.000 54.207 27.180 10.251 27.180 -30.000 -30.000 + .76410E+08 33.354 18.006 33.354 -30.000 -30.000 54.207 27.178 10.218 27.178 -30.000 -30.000 + .76510E+08 33.354 17.971 33.354 -30.000 -30.000 54.207 27.177 10.184 27.177 -30.000 -30.000 + .76610E+08 33.352 17.939 33.352 -30.000 -30.000 54.207 27.176 10.152 27.176 -30.000 -30.000 + .76710E+08 33.351 17.907 33.351 -30.000 -30.000 54.207 27.175 10.120 27.175 -30.000 -30.000 + .76810E+08 33.350 17.876 33.350 -30.000 -30.000 54.207 27.174 10.090 27.174 -30.000 -30.000 + .76910E+08 33.349 17.845 33.349 -30.000 -30.000 54.207 27.173 10.059 27.173 -30.000 -30.000 + .77010E+08 33.348 17.816 33.348 -30.000 -30.000 54.207 27.172 10.030 27.172 -30.000 -30.000 + .77110E+08 33.347 17.785 33.347 -30.000 -30.000 54.207 27.171 10.000 27.171 -30.000 -30.000 + .77210E+08 33.346 17.754 33.346 -30.000 -30.000 54.207 27.170 9.969 27.170 -30.000 -30.000 + .77310E+08 33.345 17.724 33.345 -30.000 -30.000 54.207 27.169 9.939 27.169 -30.000 -30.000 + .77410E+08 33.344 17.695 33.344 -30.000 -30.000 54.207 27.168 9.911 27.168 -30.000 -30.000 + .77510E+08 33.343 17.662 33.343 -30.000 -30.000 54.207 27.167 9.879 27.167 -30.000 -30.000 + .77610E+08 33.342 17.633 33.342 -30.000 -30.000 54.207 27.166 9.850 27.166 -30.000 -30.000 + .77710E+08 33.341 17.603 33.341 -30.000 -30.000 54.207 27.165 9.820 27.165 -30.000 -30.000 + .77810E+08 33.340 17.572 33.340 -30.000 -30.000 54.207 27.164 9.791 27.164 -30.000 -30.000 + .77910E+08 33.339 17.543 33.339 -30.000 -30.000 54.207 27.162 9.762 27.162 -30.000 -30.000 + .78010E+08 33.337 17.513 33.337 -30.000 -30.000 54.207 27.161 9.733 27.161 -30.000 -30.000 + .78110E+08 33.337 17.484 33.337 -30.000 -30.000 54.207 27.160 9.704 27.160 -30.000 -30.000 + .78210E+08 33.336 17.454 33.336 -30.000 -30.000 54.207 27.160 9.675 27.160 -30.000 -30.000 + .78310E+08 33.335 17.423 33.335 -30.000 -30.000 54.207 27.158 9.645 27.158 -30.000 -30.000 + .78410E+08 33.334 17.393 33.334 -30.000 -30.000 54.207 27.158 9.616 27.158 -30.000 -30.000 + .78510E+08 33.333 17.366 33.333 -30.000 -30.000 54.207 27.157 9.589 27.157 -30.000 -30.000 + .78610E+08 33.331 17.337 33.331 -30.000 -30.000 54.207 27.155 9.561 27.155 -30.000 -30.000 + .78710E+08 33.330 17.298 33.330 -30.000 -30.000 54.207 27.154 9.524 27.154 -30.000 -30.000 + .78810E+08 33.329 17.270 33.329 -30.000 -30.000 54.207 27.153 9.496 27.153 -30.000 -30.000 + .78910E+08 33.329 17.240 33.329 -30.000 -30.000 54.207 27.152 9.468 27.152 -30.000 -30.000 + .79010E+08 33.327 17.211 33.327 -30.000 -30.000 54.207 27.151 9.440 27.151 -30.000 -30.000 + .79110E+08 33.326 17.184 33.326 -30.000 -30.000 54.207 27.150 9.414 27.150 -30.000 -30.000 + .79210E+08 33.325 17.157 33.325 -30.000 -30.000 54.207 27.149 9.388 27.149 -30.000 -30.000 + .79310E+08 33.324 17.129 33.324 -30.000 -30.000 54.207 27.148 9.362 27.148 -30.000 -30.000 + .79410E+08 33.323 17.101 33.323 -30.000 -30.000 54.207 27.147 9.335 27.147 -30.000 -30.000 + .79510E+08 33.322 17.075 33.322 -30.000 -30.000 54.207 27.146 9.310 27.146 -30.000 -30.000 + .79610E+08 33.321 17.048 33.321 -30.000 -30.000 54.207 27.145 9.285 27.145 -30.000 -30.000 + .79710E+08 33.320 17.020 33.320 -30.000 -30.000 54.207 27.144 9.259 27.144 -30.000 -30.000 + .79810E+08 33.319 17.002 33.319 -30.000 -30.000 54.207 27.143 9.242 27.143 -30.000 -30.000 + .79910E+08 33.318 16.968 33.318 -30.000 -30.000 54.207 27.142 9.210 27.142 -30.000 -30.000 + .80010E+08 33.317 16.942 33.317 -30.000 -30.000 54.207 27.141 9.185 27.141 -30.000 -30.000 + .80110E+08 33.316 16.917 33.316 -30.000 -30.000 54.207 27.140 9.162 27.140 -30.000 -30.000 + .80210E+08 33.315 16.893 33.315 -30.000 -30.000 54.207 27.139 9.140 27.139 -30.000 -30.000 + .80310E+08 33.314 16.869 33.314 -30.000 -30.000 54.207 27.138 9.118 27.138 -30.000 -30.000 + .80410E+08 33.313 16.844 33.313 -30.000 -30.000 54.207 27.137 9.096 27.137 -30.000 -30.000 + .80510E+08 33.312 16.821 33.312 -30.000 -30.000 54.207 27.136 9.074 27.136 -30.000 -30.000 + .80610E+08 33.311 16.797 33.311 -30.000 -30.000 54.207 27.135 9.053 27.135 -30.000 -30.000 + .80710E+08 33.310 16.775 33.310 -30.000 -30.000 54.207 27.134 9.033 27.134 -30.000 -30.000 + .80810E+08 33.309 16.753 33.309 -30.000 -30.000 54.207 27.133 9.013 27.133 -30.000 -30.000 + .80910E+08 33.308 16.731 33.308 -30.000 -30.000 54.207 27.132 8.993 27.132 -30.000 -30.000 + .81010E+08 33.307 16.709 33.307 -30.000 -30.000 54.207 27.131 8.974 27.131 -30.000 -30.000 + .81110E+08 33.306 16.688 33.306 -30.000 -30.000 54.207 27.130 8.956 27.130 -30.000 -30.000 + .81210E+08 33.306 16.668 33.306 -30.000 -30.000 54.207 27.130 8.938 27.130 -30.000 -30.000 + .81310E+08 33.304 16.648 33.304 -30.000 -30.000 54.207 27.128 8.920 27.128 -30.000 -30.000 + .81410E+08 33.303 16.629 33.303 -30.000 -30.000 54.207 27.127 8.904 27.127 -30.000 -30.000 + .81510E+08 33.302 16.611 33.302 -30.000 -30.000 54.207 27.126 8.888 27.126 -30.000 -30.000 + .81610E+08 33.301 16.592 33.301 -30.000 -30.000 54.207 27.125 8.872 27.125 -30.000 -30.000 + .81710E+08 33.300 16.574 33.300 -30.000 -30.000 54.207 27.124 8.857 27.124 -30.000 -30.000 + .81810E+08 33.300 16.557 33.300 -30.000 -30.000 54.207 27.124 8.843 27.124 -30.000 -30.000 + .81910E+08 33.299 16.541 33.299 -30.000 -30.000 54.207 27.122 8.829 27.122 -30.000 -30.000 + .82010E+08 33.297 16.520 33.297 -30.000 -30.000 54.207 27.121 8.811 27.121 -30.000 -30.000 + .82110E+08 33.296 16.505 33.296 -30.000 -30.000 54.207 27.120 8.798 27.120 -30.000 -30.000 + .82210E+08 33.296 16.489 33.296 -30.000 -30.000 54.207 27.119 8.786 27.119 -30.000 -30.000 + .82310E+08 33.294 16.475 33.294 -30.000 -30.000 54.207 27.118 8.774 27.118 -30.000 -30.000 + .82410E+08 33.294 16.460 33.294 -30.000 -30.000 54.207 27.117 8.762 27.117 -30.000 -30.000 + .82510E+08 33.292 16.447 33.292 -30.000 -30.000 54.207 27.116 8.750 27.116 -30.000 -30.000 + .82610E+08 33.292 16.434 33.292 -30.000 -30.000 54.207 27.115 8.740 27.115 -30.000 -30.000 + .82710E+08 33.291 16.420 33.291 -30.000 -30.000 54.207 27.115 8.729 27.115 -30.000 -30.000 + .82810E+08 33.290 16.409 33.290 -30.000 -30.000 54.207 27.114 8.720 27.114 -30.000 -30.000 + .82910E+08 33.289 16.397 33.289 -30.000 -30.000 54.207 27.113 8.711 27.113 -30.000 -30.000 + .83010E+08 33.288 16.385 33.288 -30.000 -30.000 54.207 27.111 8.702 27.111 -30.000 -30.000 + .83110E+08 33.287 16.375 33.287 -30.000 -30.000 54.207 27.111 8.693 27.111 -30.000 -30.000 + .83210E+08 33.286 16.365 33.286 -30.000 -30.000 54.207 27.110 8.685 27.110 -30.000 -30.000 + .83310E+08 33.285 16.354 33.285 -30.000 -30.000 54.207 27.109 8.677 27.109 -30.000 -30.000 + .83410E+08 33.284 16.345 33.284 -30.000 -30.000 54.207 27.108 8.670 27.108 -30.000 -30.000 + .83510E+08 33.283 16.336 33.283 -30.000 -30.000 54.207 27.107 8.663 27.107 -30.000 -30.000 + .83610E+08 33.282 16.327 33.282 -30.000 -30.000 54.207 27.106 8.656 27.106 -30.000 -30.000 + .83710E+08 33.282 16.319 33.282 -30.000 -30.000 54.207 27.105 8.650 27.105 -30.000 -30.000 + .83810E+08 33.280 16.312 33.280 -30.000 -30.000 54.207 27.104 8.644 27.104 -30.000 -30.000 + .83910E+08 33.279 16.304 33.279 -30.000 -30.000 54.207 27.103 8.638 27.103 -30.000 -30.000 + .84010E+08 33.278 16.297 33.278 -30.000 -30.000 54.207 27.102 8.633 27.102 -30.000 -30.000 + .84110E+08 33.277 16.290 33.277 -30.000 -30.000 54.207 27.101 8.628 27.101 -30.000 -30.000 + .84210E+08 33.276 16.284 33.276 -30.000 -30.000 54.207 27.100 8.623 27.100 -30.000 -30.000 + .84310E+08 33.275 16.278 33.275 -30.000 -30.000 54.207 27.099 8.619 27.099 -30.000 -30.000 + .84410E+08 33.274 16.273 33.274 -30.000 -30.000 54.207 27.098 8.615 27.098 -30.000 -30.000 + .84510E+08 33.274 16.267 33.274 -30.000 -30.000 54.207 27.098 8.611 27.098 -30.000 -30.000 + .84610E+08 33.273 16.262 33.273 -30.000 -30.000 54.207 27.097 8.607 27.097 -30.000 -30.000 + .84710E+08 33.272 16.257 33.272 -30.000 -30.000 54.207 27.096 8.603 27.096 -30.000 -30.000 + .84810E+08 33.270 16.253 33.270 -30.000 -30.000 54.207 27.094 8.600 27.094 -30.000 -30.000 + .84910E+08 33.270 16.249 33.270 -30.000 -30.000 54.207 27.094 8.597 27.094 -30.000 -30.000 + .85010E+08 33.269 16.245 33.269 -30.000 -30.000 54.207 27.093 8.594 27.093 -30.000 -30.000 + .85110E+08 33.268 16.241 33.268 -30.000 -30.000 54.207 27.092 8.591 27.092 -30.000 -30.000 + .85210E+08 33.267 16.237 33.267 -30.000 -30.000 54.207 27.091 8.589 27.091 -30.000 -30.000 + .85310E+08 33.266 16.234 33.266 -30.000 -30.000 54.207 27.090 8.586 27.090 -30.000 -30.000 + .85410E+08 33.266 16.231 33.266 -30.000 -30.000 54.207 27.089 8.584 27.089 -30.000 -30.000 + .85510E+08 33.264 16.228 33.264 -30.000 -30.000 54.207 27.088 8.582 27.088 -30.000 -30.000 + .85610E+08 33.264 16.225 33.264 -30.000 -30.000 54.207 27.087 8.580 27.087 -30.000 -30.000 + .85710E+08 33.263 16.222 33.263 -30.000 -30.000 54.207 27.087 8.578 27.087 -30.000 -30.000 + .85810E+08 33.262 16.220 33.262 -30.000 -30.000 54.207 27.086 8.576 27.086 -30.000 -30.000 + .85910E+08 33.260 16.218 33.260 -30.000 -30.000 54.207 27.084 8.574 27.084 -30.000 -30.000 + .86010E+08 33.260 16.215 33.260 -30.000 -30.000 54.207 27.084 8.573 27.084 -30.000 -30.000 + .86110E+08 33.259 16.213 33.259 -30.000 -30.000 54.207 27.083 8.571 27.083 -30.000 -30.000 + .86210E+08 33.258 16.211 33.258 -30.000 -30.000 54.207 27.082 8.570 27.082 -30.000 -30.000 + .86310E+08 33.257 16.210 33.257 -30.000 -30.000 54.207 27.081 8.568 27.081 -30.000 -30.000 + .86410E+08 33.256 16.208 33.256 -30.000 -30.000 54.207 27.080 8.567 27.080 -30.000 -30.000 + .86510E+08 33.255 16.206 33.255 -30.000 -30.000 54.207 27.079 8.566 27.079 -30.000 -30.000 + .86610E+08 33.254 16.205 33.254 -30.000 -30.000 54.207 27.078 8.565 27.078 -30.000 -30.000 + .86710E+08 33.253 16.203 33.253 -30.000 -30.000 54.207 27.077 8.564 27.077 -30.000 -30.000 + .86810E+08 33.253 16.202 33.253 -30.000 -30.000 54.207 27.077 8.563 27.077 -30.000 -30.000 + .86910E+08 33.251 16.201 33.251 -30.000 -30.000 54.207 27.075 8.562 27.075 -30.000 -30.000 + .87010E+08 33.251 16.200 33.251 -30.000 -30.000 54.207 27.075 8.561 27.075 -30.000 -30.000 + .87110E+08 33.250 16.198 33.250 -30.000 -30.000 54.207 27.074 8.560 27.074 -30.000 -30.000 + .87210E+08 33.249 16.197 33.249 -30.000 -30.000 54.207 27.073 8.560 27.073 -30.000 -30.000 + .87310E+08 33.248 16.196 33.248 -30.000 -30.000 54.207 27.072 8.559 27.072 -30.000 -30.000 + .87410E+08 33.247 16.196 33.247 -30.000 -30.000 54.207 27.071 8.558 27.071 -30.000 -30.000 + .87510E+08 33.246 16.195 33.246 -30.000 -30.000 54.207 27.070 8.558 27.070 -30.000 -30.000 + .87610E+08 33.245 16.194 33.245 -30.000 -30.000 54.207 27.069 8.557 27.069 -30.000 -30.000 + .87710E+08 33.244 16.193 33.244 -30.000 -30.000 54.207 27.068 8.557 27.068 -30.000 -30.000 + .87810E+08 33.244 16.192 33.244 -30.000 -30.000 54.207 27.068 8.556 27.068 -30.000 -30.000 + .87910E+08 33.243 16.192 33.243 -30.000 -30.000 54.207 27.067 8.556 27.067 -30.000 -30.000 + .88010E+08 33.242 16.191 33.242 -30.000 -30.000 54.207 27.066 8.555 27.066 -30.000 -30.000 + .88110E+08 33.241 16.191 33.241 -30.000 -30.000 54.207 27.065 8.555 27.065 -30.000 -30.000 + .88210E+08 33.240 16.190 33.240 -30.000 -30.000 54.207 27.064 8.554 27.064 -30.000 -30.000 + .88310E+08 33.239 16.189 33.239 -30.000 -30.000 54.207 27.063 8.554 27.063 -30.000 -30.000 + .88410E+08 33.238 16.189 33.238 -30.000 -30.000 54.207 27.062 8.554 27.062 -30.000 -30.000 + .88510E+08 33.237 16.188 33.237 -30.000 -30.000 54.207 27.061 8.553 27.061 -30.000 -30.000 + .88610E+08 33.237 16.188 33.237 -30.000 -30.000 54.207 27.061 8.553 27.061 -30.000 -30.000 + .88710E+08 33.236 16.188 33.236 -30.000 -30.000 54.207 27.060 8.553 27.060 -30.000 -30.000 + .88810E+08 33.235 16.187 33.235 -30.000 -30.000 54.207 27.059 8.552 27.059 -30.000 -30.000 + .88910E+08 33.234 16.187 33.234 -30.000 -30.000 54.207 27.058 8.552 27.058 -30.000 -30.000 + .89010E+08 33.233 16.187 33.233 -30.000 -30.000 54.207 27.057 8.552 27.057 -30.000 -30.000 + .89110E+08 33.232 16.186 33.232 -30.000 -30.000 54.207 27.056 8.552 27.056 -30.000 -30.000 + .89210E+08 33.232 16.186 33.232 -30.000 -30.000 54.207 27.055 8.551 27.055 -30.000 -30.000 + .89310E+08 33.231 16.186 33.231 -30.000 -30.000 54.207 27.054 8.551 27.054 -30.000 -30.000 + .89410E+08 33.230 16.186 33.230 -30.000 -30.000 54.207 27.053 8.551 27.053 -30.000 -30.000 + .89510E+08 33.229 16.185 33.229 -30.000 -30.000 54.207 27.053 8.551 27.053 -30.000 -30.000 + .89610E+08 33.228 16.185 33.228 -30.000 -30.000 54.207 27.052 8.551 27.052 -30.000 -30.000 + .89710E+08 33.227 16.185 33.227 -30.000 -30.000 54.207 27.051 8.551 27.051 -30.000 -30.000 + .89810E+08 33.226 16.185 33.226 -30.000 -30.000 54.207 27.050 8.550 27.050 -30.000 -30.000 + .89910E+08 33.225 16.185 33.225 -30.000 -30.000 54.207 27.049 8.550 27.049 -30.000 -30.000 + .90010E+08 33.225 16.184 33.225 -30.000 -30.000 54.207 27.048 8.550 27.048 -30.000 -30.000 + .90110E+08 33.224 16.184 33.224 -30.000 -30.000 54.207 27.048 8.550 27.048 -30.000 -30.000 + .90210E+08 33.223 16.184 33.223 -30.000 -30.000 54.207 27.047 8.550 27.047 -30.000 -30.000 + .90310E+08 33.222 16.184 33.222 -30.000 -30.000 54.207 27.046 8.550 27.046 -30.000 -30.000 + .90410E+08 33.221 16.184 33.221 -30.000 -30.000 54.207 27.045 8.550 27.045 -30.000 -30.000 + .90510E+08 33.221 16.184 33.221 -30.000 -30.000 54.207 27.044 8.550 27.044 -30.000 -30.000 + .90610E+08 33.220 16.183 33.220 -30.000 -30.000 54.207 27.044 8.550 27.044 -30.000 -30.000 + .90710E+08 33.219 16.183 33.219 -30.000 -30.000 54.207 27.043 8.549 27.043 -30.000 -30.000 + .90810E+08 33.218 16.183 33.218 -30.000 -30.000 54.207 27.042 8.549 27.042 -30.000 -30.000 + .90910E+08 33.218 16.183 33.218 -30.000 -30.000 54.207 27.041 8.549 27.041 -30.000 -30.000 + .91010E+08 33.217 16.183 33.217 -30.000 -30.000 54.207 27.041 8.549 27.041 -30.000 -30.000 + .91110E+08 33.216 16.183 33.216 -30.000 -30.000 54.207 27.040 8.549 27.040 -30.000 -30.000 + .91210E+08 33.216 16.182 33.216 -30.000 -30.000 54.207 27.040 8.549 27.040 -30.000 -30.000 + .91310E+08 33.215 16.182 33.215 -30.000 -30.000 54.207 27.039 8.549 27.039 -30.000 -30.000 + .91410E+08 33.215 16.182 33.215 -30.000 -30.000 54.207 27.039 8.549 27.039 -30.000 -30.000 + .91510E+08 33.214 16.182 33.214 -30.000 -30.000 54.207 27.038 8.548 27.038 -30.000 -30.000 + .91610E+08 33.213 16.182 33.213 -30.000 -30.000 54.207 27.037 8.548 27.037 -30.000 -30.000 + .91710E+08 33.212 16.182 33.212 -30.000 -30.000 54.207 27.036 8.548 27.036 -30.000 -30.000 + .91810E+08 33.212 16.182 33.212 -30.000 -30.000 54.207 27.036 8.548 27.036 -30.000 -30.000 + .91910E+08 33.211 16.181 33.211 -30.000 -30.000 54.207 27.035 8.548 27.035 -30.000 -30.000 + .92010E+08 33.210 16.181 33.210 -30.000 -30.000 54.207 27.034 8.548 27.034 -30.000 -30.000 + .92110E+08 33.210 16.181 33.210 -30.000 -30.000 54.207 27.033 8.548 27.033 -30.000 -30.000 + .92210E+08 33.209 16.181 33.209 -30.000 -30.000 54.207 27.033 8.548 27.033 -30.000 -30.000 + .92310E+08 33.208 16.181 33.208 -30.000 -30.000 54.207 27.032 8.548 27.032 -30.000 -30.000 + .92410E+08 33.208 16.181 33.208 -30.000 -30.000 54.207 27.032 8.547 27.032 -30.000 -30.000 + .92510E+08 33.207 16.180 33.207 -30.000 -30.000 54.207 27.031 8.547 27.031 -30.000 -30.000 + .92610E+08 33.206 16.180 33.206 -30.000 -30.000 54.207 27.030 8.547 27.030 -30.000 -30.000 + .92710E+08 33.206 16.180 33.206 -30.000 -30.000 54.207 27.029 8.547 27.029 -30.000 -30.000 + .92810E+08 33.205 16.180 33.205 -30.000 -30.000 54.207 27.029 8.547 27.029 -30.000 -30.000 + .92910E+08 33.204 16.180 33.204 -30.000 -30.000 54.207 27.028 8.547 27.028 -30.000 -30.000 + .93010E+08 33.204 16.180 33.204 -30.000 -30.000 54.207 27.028 8.547 27.028 -30.000 -30.000 + .93110E+08 33.203 16.179 33.203 -30.000 -30.000 54.207 27.027 8.547 27.027 -30.000 -30.000 + .93210E+08 33.202 16.179 33.202 -30.000 -30.000 54.207 27.026 8.547 27.026 -30.000 -30.000 + .93310E+08 33.202 16.179 33.202 -30.000 -30.000 54.207 27.026 8.546 27.026 -30.000 -30.000 + .93410E+08 33.201 16.179 33.201 -30.000 -30.000 54.207 27.025 8.546 27.025 -30.000 -30.000 + .93510E+08 33.201 16.179 33.201 -30.000 -30.000 54.207 27.025 8.546 27.025 -30.000 -30.000 + .93610E+08 33.200 16.179 33.200 -30.000 -30.000 54.207 27.024 8.546 27.024 -30.000 -30.000 + .93710E+08 33.199 16.179 33.199 -30.000 -30.000 54.207 27.023 8.546 27.023 -30.000 -30.000 + .93810E+08 33.198 16.178 33.198 -30.000 -30.000 54.207 27.022 8.546 27.022 -30.000 -30.000 + .93910E+08 33.198 16.178 33.198 -30.000 -30.000 54.207 27.022 8.546 27.022 -30.000 -30.000 + .94010E+08 33.197 16.178 33.197 -30.000 -30.000 54.207 27.021 8.546 27.021 -30.000 -30.000 + .94110E+08 33.197 16.178 33.197 -30.000 -30.000 54.207 27.021 8.546 27.021 -30.000 -30.000 + .94210E+08 33.196 16.178 33.196 -30.000 -30.000 54.207 27.020 8.545 27.020 -30.000 -30.000 + .94310E+08 33.195 16.178 33.195 -30.000 -30.000 54.207 27.019 8.545 27.019 -30.000 -30.000 + .94410E+08 33.195 16.177 33.195 -30.000 -30.000 54.207 27.019 8.545 27.019 -30.000 -30.000 + .94510E+08 33.194 16.177 33.194 -30.000 -30.000 54.207 27.018 8.545 27.018 -30.000 -30.000 + .94610E+08 33.193 16.177 33.193 -30.000 -30.000 54.207 27.017 8.545 27.017 -30.000 -30.000 + .94710E+08 33.193 16.177 33.193 -30.000 -30.000 54.207 27.017 8.545 27.017 -30.000 -30.000 + .94810E+08 33.192 16.177 33.192 -30.000 -30.000 54.207 27.016 8.545 27.016 -30.000 -30.000 + .94910E+08 33.191 16.177 33.191 -30.000 -30.000 54.207 27.015 8.545 27.015 -30.000 -30.000 + .95010E+08 33.191 16.177 33.191 -30.000 -30.000 54.207 27.015 8.545 27.015 -30.000 -30.000 + .95110E+08 33.190 16.176 33.190 -30.000 -30.000 54.207 27.014 8.544 27.014 -30.000 -30.000 + .95210E+08 33.189 16.176 33.189 -30.000 -30.000 54.207 27.013 8.544 27.013 -30.000 -30.000 + .95310E+08 33.189 16.176 33.189 -30.000 -30.000 54.207 27.013 8.544 27.013 -30.000 -30.000 + .95410E+08 33.188 16.176 33.188 -30.000 -30.000 54.207 27.012 8.544 27.012 -30.000 -30.000 + .95510E+08 33.187 16.176 33.187 -30.000 -30.000 54.207 27.011 8.544 27.011 -30.000 -30.000 + .95610E+08 33.187 16.176 33.187 -30.000 -30.000 54.207 27.011 8.544 27.011 -30.000 -30.000 + .95710E+08 33.186 16.175 33.186 -30.000 -30.000 54.207 27.010 8.544 27.010 -30.000 -30.000 + .95810E+08 33.185 16.175 33.185 -30.000 -30.000 54.207 27.009 8.544 27.009 -30.000 -30.000 + .95910E+08 33.185 16.175 33.185 -30.000 -30.000 54.207 27.009 8.544 27.009 -30.000 -30.000 + .96010E+08 33.184 16.175 33.184 -30.000 -30.000 54.207 27.008 8.543 27.008 -30.000 -30.000 + .96110E+08 33.183 16.175 33.183 -30.000 -30.000 54.207 27.007 8.543 27.007 -30.000 -30.000 + .96210E+08 33.183 16.175 33.183 -30.000 -30.000 54.207 27.007 8.543 27.007 -30.000 -30.000 + .96310E+08 33.182 16.175 33.182 -30.000 -30.000 54.207 27.006 8.543 27.006 -30.000 -30.000 + .96410E+08 33.181 16.174 33.181 -30.000 -30.000 54.207 27.005 8.543 27.005 -30.000 -30.000 + .96510E+08 33.181 16.174 33.181 -30.000 -30.000 54.207 27.005 8.543 27.005 -30.000 -30.000 + .96610E+08 33.180 16.174 33.180 -30.000 -30.000 54.207 27.004 8.543 27.004 -30.000 -30.000 + .96710E+08 33.179 16.174 33.179 -30.000 -30.000 54.207 27.003 8.543 27.003 -30.000 -30.000 + .96810E+08 33.179 16.174 33.179 -30.000 -30.000 54.207 27.003 8.543 27.003 -30.000 -30.000 + .96910E+08 33.178 16.174 33.178 -30.000 -30.000 54.207 27.002 8.542 27.002 -30.000 -30.000 + .97010E+08 33.177 16.173 33.177 -30.000 -30.000 54.207 27.001 8.542 27.001 -30.000 -30.000 + .97110E+08 33.177 16.173 33.177 -30.000 -30.000 54.207 27.001 8.542 27.001 -30.000 -30.000 + .97210E+08 33.176 16.173 33.176 -30.000 -30.000 54.207 27.000 8.542 27.000 -30.000 -30.000 + .97310E+08 33.175 16.173 33.175 -30.000 -30.000 54.207 26.999 8.542 26.999 -30.000 -30.000 + .97410E+08 33.175 16.173 33.175 -30.000 -30.000 54.207 26.999 8.542 26.999 -30.000 -30.000 + .97510E+08 33.174 16.173 33.174 -30.000 -30.000 54.207 26.998 8.542 26.998 -30.000 -30.000 + .97610E+08 33.173 16.173 33.173 -30.000 -30.000 54.207 26.997 8.542 26.997 -30.000 -30.000 + .97710E+08 33.173 16.172 33.173 -30.000 -30.000 54.207 26.997 8.542 26.997 -30.000 -30.000 + .97810E+08 33.172 16.172 33.172 -30.000 -30.000 54.207 26.996 8.541 26.996 -30.000 -30.000 + .97910E+08 33.172 16.172 33.172 -30.000 -30.000 54.207 26.995 8.541 26.995 -30.000 -30.000 + .98010E+08 33.171 16.172 33.171 -30.000 -30.000 54.207 26.995 8.541 26.995 -30.000 -30.000 + .98110E+08 33.170 16.172 33.170 -30.000 -30.000 54.207 26.994 8.541 26.994 -30.000 -30.000 + .98210E+08 33.169 16.172 33.169 -30.000 -30.000 54.207 26.993 8.541 26.993 -30.000 -30.000 + .98310E+08 33.169 16.172 33.169 -30.000 -30.000 54.207 26.993 8.541 26.993 -30.000 -30.000 + .98410E+08 33.168 16.171 33.168 -30.000 -30.000 54.207 26.992 8.541 26.992 -30.000 -30.000 + .98510E+08 33.167 16.171 33.167 -30.000 -30.000 54.207 26.991 8.541 26.991 -30.000 -30.000 + .98610E+08 33.167 16.171 33.167 -30.000 -30.000 54.207 26.991 8.541 26.991 -30.000 -30.000 + .98710E+08 33.166 16.171 33.166 -30.000 -30.000 54.207 26.990 8.540 26.990 -30.000 -30.000 + .98810E+08 33.166 16.171 33.166 -30.000 -30.000 54.207 26.989 8.540 26.989 -30.000 -30.000 + .98910E+08 33.165 16.171 33.165 -30.000 -30.000 54.207 26.989 8.540 26.989 -30.000 -30.000 + .99010E+08 33.164 16.170 33.164 -30.000 -30.000 54.207 26.988 8.540 26.988 -30.000 -30.000 + .99110E+08 33.164 16.170 33.164 -30.000 -30.000 54.207 26.988 8.540 26.988 -30.000 -30.000 + .99210E+08 33.163 16.170 33.163 -30.000 -30.000 54.207 26.987 8.540 26.987 -30.000 -30.000 + .99310E+08 33.162 16.170 33.162 -30.000 -30.000 54.207 26.986 8.540 26.986 -30.000 -30.000 + .99410E+08 33.162 16.170 33.162 -30.000 -30.000 54.207 26.986 8.540 26.986 -30.000 -30.000 + .99510E+08 33.161 16.170 33.161 -30.000 -30.000 54.207 26.985 8.540 26.985 -30.000 -30.000 + .99610E+08 33.160 16.170 33.160 -30.000 -30.000 54.207 26.984 8.539 26.984 -30.000 -30.000 + .99710E+08 33.159 16.169 33.159 -30.000 -30.000 54.207 26.983 8.539 26.983 -30.000 -30.000 + .99810E+08 33.159 16.169 33.159 -30.000 -30.000 54.207 26.983 8.539 26.983 -30.000 -30.000 + .99910E+08 33.158 16.169 33.158 -30.000 -30.000 54.207 26.982 8.539 26.982 -30.000 -30.000 diff --git a/src/global/global.cpp b/src/global/global.cpp index 9447f2548..1450fe9e6 100644 --- a/src/global/global.cpp +++ b/src/global/global.cpp @@ -354,10 +354,14 @@ void parse_param(char *name, char *value, struct parameters *parms) else if (strcmp(name, "prng_seed") == 0) parms->prng_seed = atoi(value); #endif // PARTICLES -#ifdef SUPERNOVA +#if defined(FEEDBACK) && !defined(NO_SN_FEEDBACK) else if (strcmp(name, "snr_filename") == 0) strncpy(parms->snr_filename, value, MAXLEN); #endif +#if defined(FEEDBACK) && !defined(NO_WIND_FEEDBACK) + else if (strcmp(name, "sw_filename") == 0) + strncpy(parms->sw_filename, value, MAXLEN); +#endif #ifdef ROTATED_PROJECTION else if (strcmp(name, "nxr") == 0) parms->nxr = atoi(value); diff --git a/src/global/global.h b/src/global/global.h index cbcb4cf43..d7cbb9d5d 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -48,6 +48,7 @@ typedef double Real; #define LENGTH_UNIT 3.08567758e21 // 1 kpc in cm #define MASS_UNIT 1.98847e33 // 1 solar mass in grams #define DENSITY_UNIT (MASS_UNIT / (LENGTH_UNIT * LENGTH_UNIT * LENGTH_UNIT)) +#define FORCE_UNIT (MASS_UNIT * LENGTH_UNIT / TIME_UNIT / TIME_UNIT) #define VELOCITY_UNIT (LENGTH_UNIT / TIME_UNIT) #define ENERGY_UNIT (DENSITY_UNIT * VELOCITY_UNIT * VELOCITY_UNIT) #define PRESSURE_UNIT (DENSITY_UNIT * VELOCITY_UNIT * VELOCITY_UNIT) @@ -271,9 +272,12 @@ struct parameters { // machine dependent seed will be generated. std::uint_fast64_t prng_seed = 0; #endif // PARTICLES -#ifdef SUPERNOVA +#if defined(FEEDBACK) && !defined(NO_SN_FEEDBACK) char snr_filename[MAXLEN]; #endif +#if defined(FEEDBACK) && !defined(NO_WIND_FEEDBACK) + char sw_filename[MAXLEN]; +#endif #ifdef ROTATED_PROJECTION int nxr; int nzr; diff --git a/src/gravity/gravity_restart.cpp b/src/gravity/gravity_restart.cpp index e3626735a..5cbc6e49e 100644 --- a/src/gravity/gravity_restart.cpp +++ b/src/gravity/gravity_restart.cpp @@ -88,7 +88,7 @@ void Grav3D::Read_Restart_HDF5(struct parameters* P, int nfile) { } -void Grav3D::Write_Restart_HDF5(struct parameters P, int nfile) +void Grav3D::Write_Restart_HDF5(struct parameters* P, int nfile) { } #endif diff --git a/src/grid/grid3D.h b/src/grid/grid3D.h index 0b774bee7..a7373fbaf 100644 --- a/src/grid/grid3D.h +++ b/src/grid/grid3D.h @@ -315,14 +315,6 @@ class Grid3D Analysis_Module Analysis; #endif -#ifdef SUPERNOVA // TODO refactor this into Analysis module - Real countSN; - Real countResolved; - Real countUnresolved; - Real totalEnergy; - Real totalMomentum; - Real totalUnresEnergy; -#endif struct Conserved { /*! pointer to conserved variable array on the host */ Real *host; diff --git a/src/main.cpp b/src/main.cpp index b5913f7ae..b67a4cfc5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -15,12 +15,12 @@ #include "grid/grid3D.h" #include "io/io.h" #include "utils/error_handling.h" -#ifdef SUPERNOVA - #include "particles/supernova.h" +#ifdef FEEDBACK + #include "feedback/feedback.h" #ifdef ANALYSIS #include "analysis/feedback_analysis.h" #endif -#endif // SUPERNOVA +#endif // FEEDBACK #ifdef STAR_FORMATION #include "particles/star_formation.h" #endif @@ -148,14 +148,20 @@ int main(int argc, char *argv[]) if (G.Analysis.Output_Now) G.Compute_and_Output_Analysis(&P); #endif -#if defined(SUPERNOVA) && defined(PARTICLE_AGE) +#if defined(FEEDBACK) && defined(PARTICLE_AGE) FeedbackAnalysis sn_analysis(G); - #ifdef MPI_CHOLLA - supernova::initState(&P, G.Particles.n_total_initial); - #else - supernova::initState(&P, G.Particles.n_local); - #endif // MPI_CHOLLA -#endif // SUPERNOVA && PARTICLE_AGE + #ifndef NO_SN_FEEDBACK + #ifdef MPI_CHOLLA + feedback::initState(&P, G.Particles.n_total_initial); + #else + feedback::initState(&P, G.Particles.n_local); + #endif // MPI_CHOLLA + #endif // NO_SN_FEEDBACK +#endif // FEEDBACK && PARTICLE_AGE + +#ifndef NO_WIND_FEEDBACK + feedback::initWindState(&P); +#endif #ifdef STAR_FORMATION star_formation::Initialize(G); @@ -244,9 +250,9 @@ int main(int argc, char *argv[]) if (G.H.t + G.H.dt > outtime) G.H.dt = outtime - G.H.t; -#if defined(SUPERNOVA) && defined(PARTICLE_AGE) - supernova::Cluster_Feedback(G, sn_analysis); -#endif // SUPERNOVA && PARTICLE_AGE +#if defined(FEEDBACK) && defined(PARTICLE_AGE) + feedback::Cluster_Feedback(G, sn_analysis); +#endif // FEEDBACK && PARTICLE_AGE #ifdef PARTICLES // Advance the particles KDK( first step ): Velocities are updated by 0.5*dt @@ -313,7 +319,7 @@ int main(int argc, char *argv[]) #ifdef ANALYSIS if (G.Analysis.Output_Now) G.Compute_and_Output_Analysis(&P); - #if defined(SUPERNOVA) && defined(PARTICLE_AGE) + #if defined(FEEDBACK) && defined(PARTICLE_AGE) sn_analysis.Compute_Gas_Velocity_Dispersion(G); #endif #endif diff --git a/src/particles/feedback_CIC_gpu.cu b/src/particles/feedback_CIC_gpu.cu deleted file mode 100644 index 646914f8a..000000000 --- a/src/particles/feedback_CIC_gpu.cu +++ /dev/null @@ -1,1159 +0,0 @@ -#if defined(SUPERNOVA) && defined(PARTICLES_GPU) && defined(PARTICLE_AGE) && \ - defined(PARTICLE_IDS) - - #include - #include - #include - #include - - #include - #include - #include - #include - - #include "../global/global.h" - #include "../global/global_cuda.h" - #include "../grid/grid3D.h" - #include "../io/io.h" - #include "supernova.h" - - #define TPB_FEEDBACK 128 - #define FEED_INFO_N 6 - #define i_RES 1 - #define i_UNRES 2 - #define i_ENERGY 3 - #define i_MOMENTUM 4 - #define i_UNRES_ENERGY 5 - -typedef curandStateMRG32k3a_t feedback_prng_t; -//typedef curandStatePhilox4_32_10_t feedback_prng_t; - -namespace supernova -{ -feedback_prng_t* randStates; -part_int_t n_states; -Real *dev_snr, snr_dt, time_sn_start, time_sn_end; -int snr_n; -} // namespace supernova - - #ifndef O_HIP -__device__ double atomicMax(double* address, double val) -{ - unsigned long long int* address_as_ull = (unsigned long long int*)address; - unsigned long long int old = *address_as_ull, assumed; - do { - assumed = old; - old = atomicCAS( - address_as_ull, assumed, - __double_as_longlong(fmax(val, __longlong_as_double(assumed)))); - } while (assumed != old); - return __longlong_as_double(old); -} - #endif // O_HIP - -__global__ void initState_kernel(unsigned int seed, - feedback_prng_t* states) -{ - int id = blockIdx.x * blockDim.x + threadIdx.x; - curand_init(seed, id, 0, &states[id]); -} - -/** - * @brief Does 2 things: - * -# Read in SN rate data from Starburst 99. If no file exists, assume a - * constant rate. - * -# Initialize the cuRAND state, which is analogous to the concept of - * generators in CPU code. The state object maintains configuration and status - * the cuRAND context for each thread on the GPU. Initialize more than the - * number of local particles since the latter will change through MPI transfers. - * - * @param P pointer to parameters struct. Passes in starburst 99 filename and - * random number gen seed. - * @param n_local number of local particles on the GPU - * @param allocation_factor - */ -void supernova::initState(struct parameters* P, part_int_t n_local, - Real allocation_factor) -{ - chprintf("supernova::initState start\n"); - std::string snr_filename(P->snr_filename); - if (not snr_filename.empty()) { - chprintf("Specified a SNR filename %s.\n", snr_filename.data()); - - // read in array of supernova rate values. - std::ifstream snr_in(snr_filename); - if (!snr_in.is_open()) { - chprintf("ERROR: but couldn't read SNR file.\n"); - exit(-1); - } - - std::vector snr_time; - std::vector snr; - - const int N_HEADER = 7; // S'99 has 7 rows of header information - const char* s99_delim = " "; // S'99 data separator - std::string line; - int line_counter = 0; - - while (snr_in.good()) { - std::getline(snr_in, line); - if (line_counter++ < N_HEADER) continue; // skip header processing - - int i = 0; - char* data = strtok(line.data(), s99_delim); - while (data != nullptr) { - if (i == 0) { - // in the following divide by # years per kyr (1000) - snr_time.push_back(std::stof(std::string(data)) / 1000); - } else if (i == 1) { - snr.push_back(pow(10, std::stof(std::string(data))) / 1000); - } - if (i > 0) - break; // only care about the first 2 items. Once i = 1 can break - // here. - data = strtok(nullptr, s99_delim); - i++; - } - } - - time_sn_end = snr_time[snr_time.size() - 1]; - time_sn_start = snr_time[0]; - // the following is the time interval between data points - // (i.e. assumes regular temporal spacing) - snr_dt = (time_sn_end - time_sn_start) / (snr.size() - 1); - - CHECK(cudaMalloc((void**)&dev_snr, snr.size() * sizeof(Real))); - CHECK(cudaMemcpy(dev_snr, snr.data(), snr.size() * sizeof(Real), - cudaMemcpyHostToDevice)); - - } else { - chprintf("No SN rate file specified. Using constant rate\n"); - time_sn_start = DEFAULT_SN_START; - time_sn_end = DEFAULT_SN_END; - } - - // Now initialize the poisson random number generator state. - n_states = n_local * allocation_factor; - cudaMalloc((void**)&randStates, n_states * sizeof(feedback_prng_t)); - - int ngrid = (n_states + TPB_FEEDBACK - 1) / TPB_FEEDBACK; - dim3 grid(ngrid); - dim3 block(TPB_FEEDBACK); - - hipLaunchKernelGGL(initState_kernel, grid, block, 0, 0, P->prng_seed, - randStates); - CHECK(cudaDeviceSynchronize()); - chprintf("supernova::initState end: n_states=%ld, ngrid=%d, threads=%d\n", - n_states, ngrid, TPB_FEEDBACK); -} - -__device__ Real GetSNRate(Real t, Real* dev_snr, Real snr_dt, Real t_start, - Real t_end) -{ - if (t < t_start || t >= t_end) return 0; - if (dev_snr == nullptr) return supernova::DEFAULT_SNR; - - int index = (int)((t - t_start) / snr_dt); - return dev_snr[index] + - (t - index * snr_dt) * (dev_snr[index + 1] - dev_snr[index]) / snr_dt; -} - -__device__ Real Calc_Timestep(Real gamma, Real* density, Real* momentum_x, - Real* momentum_y, Real* momentum_z, Real* energy, - int index, Real dx, Real dy, Real dz) -{ - Real dens = fmax(density[index], DENS_FLOOR); - Real d_inv = 1.0 / dens; - Real vx = momentum_x[index] * d_inv; - Real vy = momentum_y[index] * d_inv; - Real vz = momentum_z[index] * d_inv; - Real P = fmax((energy[index] - 0.5 * dens * (vx * vx + vy * vy + vz * vz)) * - (gamma - 1.0), - TINY_NUMBER); - Real cs = sqrt(gamma * P * d_inv); - return fmax(fmax((fabs(vx) + cs) / dx, (fabs(vy) + cs) / dy), - (fabs(vz) + cs) / dz); -} - -/** the prescription for dividing a scalar quantity between 3x3x3 cells is done - by imagining a 2x2x2 cell volume around the SN. These fractions, then, - represent the linear extent of this volume into the cell in question. For i=0 - this should be 1*1/2. For i=-1 this should be (1-dx)*1/2. For i=+1 this - should be dx*1/2. In the above the 1/2 factor is normalize over 2 - cells/direction. - */ -__device__ Real frac(int i, Real dx) -{ - return (-0.5 * i * i - 0.5 * i + 1 + i * dx) * 0.5; -} - -__device__ Real d_fr(int i, Real dx) -{ - return (dx > 0.5) * i * (1 - 2 * dx) + ((i + 1) * dx + 0.5 * (i - 1)) - - 3 * (i - 1) * (i + 1) * (0.5 - dx); -} - -__device__ Real GetAverageDensity(Real* density, int xi, int yi, int zi, - int nx_grid, int ny_grid, int n_ghost) -{ - Real d_average = 0.0; - for (int i = -1; i < 2; i++) { - for (int j = -1; j < 2; j++) { - for (int k = -1; k < 2; k++) { - d_average += density[(xi + n_ghost + i) + (yi + n_ghost + j) * nx_grid + - (zi + n_ghost + k) * nx_grid * ny_grid]; - } - } - } - return d_average / 27; -} - -__device__ Real GetAverageNumberDensity_CGS(Real* density, int xi, int yi, - int zi, int nx_grid, int ny_grid, int n_ghost) -{ - return GetAverageDensity(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * DENSITY_UNIT / - (supernova::MU * MP); -} - -__device__ bool Particle_Is_Alone(Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, part_int_t n_local, int gtid, Real dx) -{ - Real x0 = pos_x_dev[gtid]; - Real y0 = pos_y_dev[gtid]; - Real z0 = pos_z_dev[gtid]; - // Brute force loop to see if particle is alone - for (int i=0;i dx) continue; - if (abs(y0 - pos_y_dev[i]) > dx) continue; - if (abs(z0 - pos_z_dev[i]) > dx) continue; - // If we made it here, something is too close. - return false; - } - return true; -} - -__device__ Real Cluster_Feedback_Resolved(Real pos_x, Real pos_y, Real pos_z, - Real xMin, Real yMin, Real zMin, - Real dx, Real dy, Real dz, - int nx_g, int ny_g, int n_ghost, int n_cells, - Real gamma, Real* conserved_device, - short direction, - Real feedback_density, Real feedback_energy) -{ - // For 2x2x2, a particle between 0-0.5 injects onto cell - 1 - int indx_x = (int)floor((pos_x - xMin - 0.5 * dx) / dx); - int indx_y = (int)floor((pos_y - yMin - 0.5 * dy) / dy); - int indx_z = (int)floor((pos_z - zMin - 0.5 * dz) / dz); - - Real cell_center_x = xMin + indx_x * dx + 0.5 * dx; - Real cell_center_y = yMin + indx_y * dy + 0.5 * dy; - Real cell_center_z = zMin + indx_z * dz + 0.5 * dz; - - Real delta_x = 1 - (pos_x - cell_center_x) / dx; - Real delta_y = 1 - (pos_y - cell_center_y) / dy; - Real delta_z = 1 - (pos_z - cell_center_z) / dz; - - Real* density = conserved_device; - Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; - Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; - Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; - Real* energy = &conserved_device[n_cells*grid_enum::Energy]; - Real* gasEnergy = &conserved_device[n_cells*grid_enum::GasEnergy]; - - Real local_dti = 0; - - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - for (int k = 0; k < 2; k++) { - int indx = (indx_x + i + n_ghost) + - (indx_y + j + n_ghost) * nx_g + - (indx_z + k + n_ghost) * nx_g * ny_g; - Real x_frac = i * (1 - delta_x) + (1 - i) * delta_x; - Real y_frac = j * (1 - delta_y) + (1 - j) * delta_y; - Real z_frac = k * (1 - delta_z) + (1 - k) * delta_z; - - atomicAdd(&density[indx], - x_frac * y_frac * z_frac * feedback_density); - atomicAdd(&gasEnergy[indx], - x_frac * y_frac * z_frac * feedback_energy); - atomicAdd(&energy[indx], - x_frac * y_frac * z_frac * feedback_energy); - - if (direction > 0) { - Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, - momentum_z, energy, indx, dx, dy, dz); - - local_dti = fmax(local_dti, cell_dti); - } - - - } // k loop - } // j loop - } // i loop - - return local_dti; -} - - -__device__ Real Cluster_Feedback_Unresolved(Real pos_x, Real pos_y, Real pos_z, - Real xMin, Real yMin, Real zMin, - Real dx, Real dy, Real dz, - int nx_g, int ny_g, int n_ghost, int n_cells, - Real gamma, Real* conserved_device, - short direction, - Real feedback_density, Real feedback_momentum, - Real n_0) -{ - // For 3x3x3 the index just centers on the cell containing the particle - int indx_x = (int)floor((pos_x - xMin) / dx); - int indx_y = (int)floor((pos_y - yMin) / dy); - int indx_z = (int)floor((pos_z - zMin) / dz); - - Real delta_x = (pos_x - xMin - indx_x * dx) / dx; - Real delta_y = (pos_y - yMin - indx_y * dy) / dy; - Real delta_z = (pos_z - zMin - indx_z * dz) / dz; - - Real local_dti = 0; - - Real* density = conserved_device; - Real* momentum_x = &conserved_device[n_cells*grid_enum::momentum_x]; - Real* momentum_y = &conserved_device[n_cells*grid_enum::momentum_y]; - Real* momentum_z = &conserved_device[n_cells*grid_enum::momentum_z]; - Real* energy = &conserved_device[n_cells*grid_enum::Energy]; - Real* gasEnergy = &conserved_device[n_cells*grid_enum::GasEnergy]; - - for (int i = -1; i < 2; i++) { - for (int j = -1; j < 2; j++) { - for (int k = -1; k < 2; k++) { - // index in array of conserved quantities - int indx = (indx_x + i + n_ghost) + - (indx_y + j + n_ghost) * nx_g + - (indx_z + k + n_ghost) * nx_g * ny_g; - - Real x_frac = - d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - Real y_frac = - frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - Real z_frac = - frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); - - Real px = x_frac * feedback_momentum; - Real py = y_frac * feedback_momentum; - Real pz = z_frac * feedback_momentum; - Real d = (abs(x_frac) + abs(y_frac) + abs(z_frac)) / 6 * - feedback_density + - n_0 * supernova::MU * MP / DENSITY_UNIT; - - atomicAdd(&momentum_x[indx], px); - atomicAdd(&momentum_y[indx], py); - atomicAdd(&momentum_z[indx], pz); - density[indx] = d; - energy[indx] = (momentum_x[indx] * momentum_x[indx] + - momentum_y[indx] * momentum_y[indx] + - momentum_z[indx] * momentum_z[indx]) / - 2 / density[indx] + - gasEnergy[indx]; - - if (direction > 0) { - Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, - momentum_z, energy, indx, dx, dy, dz); - - local_dti = fmax(local_dti, cell_dti); - } - - - - } // k loop - } // j loop - } // i loop - - return local_dti; - - - -} - - -__device__ void Cluster_Feedback_Helper(part_int_t n_local, - Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, - Real* age_dev, Real* mass_dev, part_int_t* id_dev, - Real xMin, Real yMin, Real zMin, - Real xMax, Real yMax, Real zMax, - Real dx, Real dy, Real dz, - int nx_g, int ny_g, int nz_g, - int n_ghost, - int n_step, - Real t, Real dt, Real* dti, - Real* dev_snr, Real snr_dt, Real time_sn_start, Real time_sn_end, - Real* prev_dens, int* prev_N, - short direction, - Real* s_info, Real* conserved_dev, Real gamma) -{ - int tid = threadIdx.x; - int gtid = blockIdx.x * blockDim.x + tid; - // Bounds check on particle arrays - if (gtid >= n_local) return; - - Real pos_x = pos_x_dev[gtid]; - Real pos_y = pos_y_dev[gtid]; - Real pos_z = pos_z_dev[gtid]; - bool in_local = (pos_x >= xMin && pos_x < xMax) && - (pos_y >= yMin && pos_y < yMax) && - (pos_z >= zMin && pos_z < zMax); - // Particle is outside bounds, exit - if (!in_local) return; - - int indx_x = (int)floor((pos_x - xMin) / dx); - int indx_y = (int)floor((pos_y - yMin) / dy); - int indx_z = (int)floor((pos_z - zMin) / dz); - bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || - indx_x >= nx_g - 2 * n_ghost || - indx_y >= ny_g - 2 * n_ghost || - indx_z >= nz_g - 2 * n_ghost; - // Ignore this particle, exit - if (ignore) return; - - bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); - if (!is_alone) return; - - - // note age_dev is actually the time of birth - Real age = t - age_dev[gtid]; - if (age > time_sn_end) return; - - int N = 0; - if (direction == -1) { - N = -prev_N[gtid]; - } else { - Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, - time_sn_start, time_sn_end) * mass_dev[gtid] * dt; - feedback_prng_t state; - curand_init(42,0,0,&state); - unsigned long long skip = n_step * 10000 + id_dev[gtid]; - skipahead(skip, &state); // provided by curand - N = (int) curand_poisson(&state, average_num_sn); - prev_N[gtid] = N; - } - - // No supernova, exit - if (N == 0) return; - - Real n_0; - if (direction == -1) { - n_0 = prev_dens[gtid]; - } else { - Real* density = conserved_dev; - n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, - nx_g, ny_g, n_ghost); - prev_dens[gtid] = n_0; - } - - - mass_dev[gtid] -= N * supernova::MASS_PER_SN; - Real dV = dx * dy * dz; - Real feedback_energy = N * supernova::ENERGY_PER_SN / dV; - Real feedback_density = N * supernova::MASS_PER_SN / dV; - Real feedback_momentum = direction * supernova::FINAL_MOMENTUM * - pow(n_0, -0.17) * pow(fabsf(N), 0.93) / dV; - - - Real shell_radius = supernova::R_SH * pow(n_0, -0.46) * pow(fabsf(N), 0.29); - bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; - - int n_cells = nx_g * ny_g * nz_g; - - Real local_dti = 0.0; - if (is_resolved) { - local_dti = Cluster_Feedback_Resolved( pos_x, pos_y, pos_z, - xMin, yMin, zMin, - dx, dy, dz, - nx_g, ny_g, n_ghost, n_cells, - gamma, conserved_dev, - direction, - feedback_density, feedback_energy); - // inject energy and density - } else { - // inject momentum and density - local_dti = Cluster_Feedback_Unresolved( pos_x, pos_y, pos_z, - xMin, yMin, zMin, - dx, dy, dz, - nx_g, ny_g, n_ghost, n_cells, - gamma, conserved_dev, - direction, - feedback_density, feedback_momentum, - n_0); - - } - - if (direction > 0) atomicMax(dti, local_dti); - - return; - -} - -__global__ void Cluster_Feedback_Kernel( - part_int_t n_local, part_int_t* id_dev, Real* pos_x_dev, Real* pos_y_dev, - Real* pos_z_dev, Real* mass_dev, Real* age_dev, Real xMin, Real yMin, - Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, - int nx_g, int ny_g, int nz_g, int n_ghost, Real t, Real dt, Real* dti, - Real* info, Real* density, Real* gasEnergy, Real* energy, Real* momentum_x, - Real* momentum_y, Real* momentum_z, Real gamma, - feedback_prng_t* states, Real* prev_dens, int* prev_N, - short direction, Real* dev_snr, Real snr_dt, Real time_sn_start, - Real time_sn_end, int n_step) -{ - - int tid = threadIdx.x; - //int gtid = blockIdx.x * blockDim.x + tid; - - // for collecting SN feedback information - __shared__ Real s_info[FEED_INFO_N * TPB_FEEDBACK]; - - Cluster_Feedback_Helper(n_local, - pos_x_dev, pos_y_dev, pos_z_dev, - age_dev, mass_dev, id_dev, - xMin, yMin, zMin, - xMax, yMax, zMax, - dx, dy, dz, - nx_g, ny_g, nz_g, - n_ghost, - n_step, - t, dt, dti, - dev_snr, snr_dt, time_sn_start, time_sn_end, - prev_dens, prev_N, - direction, - s_info, density, gamma); - - - __syncthreads(); - - // reduce the info from all the threads in the block - for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { - if (tid < s) { - s_info[FEED_INFO_N * tid] += s_info[FEED_INFO_N * (tid + s)]; - s_info[FEED_INFO_N * tid + 1] += s_info[FEED_INFO_N * (tid + s) + 1]; - s_info[FEED_INFO_N * tid + 2] += s_info[FEED_INFO_N * (tid + s) + 2]; - s_info[FEED_INFO_N * tid + 3] += s_info[FEED_INFO_N * (tid + s) + 3]; - s_info[FEED_INFO_N * tid + 4] += s_info[FEED_INFO_N * (tid + s) + 4]; - s_info[FEED_INFO_N * tid + 5] += s_info[FEED_INFO_N * (tid + s) + 5]; - } - __syncthreads(); - } - - if (tid == 0) { - info[FEED_INFO_N * blockIdx.x] = s_info[0]; - info[FEED_INFO_N * blockIdx.x + 1] = s_info[1]; - info[FEED_INFO_N * blockIdx.x + 2] = s_info[2]; - info[FEED_INFO_N * blockIdx.x + 3] = s_info[3]; - info[FEED_INFO_N * blockIdx.x + 4] = s_info[4]; - info[FEED_INFO_N * blockIdx.x + 5] = s_info[5]; - } - -} - -__global__ void Cluster_Feedback_Kernel2( - part_int_t n_local, part_int_t* id, Real* pos_x_dev, Real* pos_y_dev, - Real* pos_z_dev, Real* mass_dev, Real* age_dev, Real xMin, Real yMin, - Real zMin, Real xMax, Real yMax, Real zMax, Real dx, Real dy, Real dz, - int nx_g, int ny_g, int nz_g, int n_ghost, Real t, Real dt, Real* dti, - Real* info, Real* density, Real* gasEnergy, Real* energy, Real* momentum_x, - Real* momentum_y, Real* momentum_z, Real gamma, - feedback_prng_t* states, Real* prev_dens, int* prev_N, - short direction, Real* dev_snr, Real snr_dt, Real time_sn_start, - Real time_sn_end, int n_step) -{ - __shared__ Real - s_info[FEED_INFO_N * - TPB_FEEDBACK]; // for collecting SN feedback information, like # - // of SNe or # resolved. - int tid = threadIdx.x; - int gtid = blockIdx.x * blockDim.x + tid; - - s_info[FEED_INFO_N * tid] = 0; // number of supernovae - s_info[FEED_INFO_N * tid + 1] = 0; // number of resolved events - s_info[FEED_INFO_N * tid + 2] = 0; // number of unresolved events - s_info[FEED_INFO_N * tid + 3] = 0; // resolved energy - s_info[FEED_INFO_N * tid + 4] = 0; // unresolved momentum - s_info[FEED_INFO_N * tid + 5] = - 0; // unresolved KE added via momentum injection - - if (gtid < n_local) { - Real pos_x, pos_y, pos_z; - Real cell_center_x, cell_center_y, cell_center_z; - Real delta_x, delta_y, delta_z; - Real x_frac, y_frac, z_frac; - Real px, py, pz, d; - // Real t_b, t_a, v_1, v_2, d_b, d_a, p_b, p_a, e; - Real feedback_energy = 0, feedback_density = 0, feedback_momentum = 0, n_0, - shell_radius; - bool is_resolved = false; - Real dV = dx * dy * dz; - Real local_dti = 0.0; - - pos_x = pos_x_dev[gtid]; - pos_y = pos_y_dev[gtid]; - pos_z = pos_z_dev[gtid]; - // kernel_printf("(%d): pos:(%.4e, %.4e, %.4e)\n", gtid, pos_x, pos_y, pos_z); - // kernel_printf("(%d): MIN:(%.4e, %.4e, %.4e)\n", gtid, xMin, yMin, xMin); - - bool in_local = (pos_x >= xMin && pos_x < xMax) && - (pos_y >= yMin && pos_y < yMax) && - (pos_z >= zMin && pos_z < zMax); - if (!in_local) { - kernel_printf( - " Feedback GPU: Particle outside local domain [%f %f %f] [%f %f] " - "[%f %f] [%f %f]\n ", - pos_x, pos_y, pos_z, xMin, xMax, yMin, yMax, zMin, zMax); - } - - int indx_x = (int)floor((pos_x - xMin) / dx); - int indx_y = (int)floor((pos_y - yMin) / dy); - int indx_z = (int)floor((pos_z - zMin) / dz); - // kernel_printf("(%d): indx:(%d, %d, %d)\n", gtid, indx_x, indx_y, indx_z); - - bool ignore = indx_x < 0 || indx_y < 0 || indx_z < 0 || - indx_x >= nx_g - 2 * n_ghost || - indx_y >= ny_g - 2 * n_ghost || indx_z >= nz_g - 2 * n_ghost; - if (ignore) { - kernel_printf( - " Feedback GPU: Particle CIC index err [%f %f %f] [%d %d %d] [%d " - "%d %d] \n ", - pos_x, pos_y, pos_z, indx_x, indx_y, indx_z, nx_g, ny_g, nz_g); - } - - // Avoid overlap issues for now - bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx) ; - - if (!ignore && in_local && is_alone) { - int N = 0; - // only calculate this if there will be SN feedback - if ((t - age_dev[gtid]) <= time_sn_end) { - if (direction == -1) - N = -prev_N[gtid]; - else { - Real average_num_sn = GetSNRate(t - age_dev[gtid], dev_snr, snr_dt, - time_sn_start, time_sn_end) * mass_dev[gtid] * dt; - - //N = (int) (average_num_sn + 0.5); - - - feedback_prng_t state;// = states[0]; // load initial state - - curand_init(42,0,0,&state); - unsigned long long skip = n_step * 10000 + id[gtid]; - skipahead(skip, &state); // provided by curand - unsigned int debug_state = curand(&state); - - - //state = states[gtid]; - - - - - N = (int) curand_poisson(&state, average_num_sn); - printf("PRNG DEBUG: n_step: %d id: %d skip: %llu debug_state: %u N: %d \n", - n_step, (int) id[gtid], skip, debug_state, N); - //states[gtid] = state; // don't write back to state, keep it pristine - prev_N[gtid] = N; - } - if (N != 0) { - mass_dev[gtid] -= N * supernova::MASS_PER_SN; - feedback_energy = N * supernova::ENERGY_PER_SN / dV; - feedback_density = N * supernova::MASS_PER_SN / dV; - if (direction == -1) - n_0 = prev_dens[gtid]; - else { - n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, - nx_g, ny_g, n_ghost); - prev_dens[gtid] = n_0; - } - // int devcount; - // cudaGetDeviceCount(&devcount); - // int devId; - // cudaGetDevice(&devId); - // kernel_printf("[%d: %d] N: %d, time: %.4e, dt: %.4e, e: %.4e, n_0: - // %.4e\n", devId, gtid, N, t, dt, feedback_energy, n_0); - - feedback_momentum = direction * supernova::FINAL_MOMENTUM * - pow(n_0, -0.17) * pow(fabsf(N), 0.93) / dV; - shell_radius = - supernova::R_SH * pow(n_0, -0.46) * pow(fabsf(N), 0.29); - is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; - if (!is_resolved) - kernel_printf( - "UR[%f] at (%d, %d, %d) id=%d, N=%d, shell_rad=%0.4e, " - "n_0=%0.4e\n", - t, indx_x + n_ghost, indx_y + n_ghost, indx_z + n_ghost, - (int)id[gtid], N, shell_radius, n_0); - - s_info[FEED_INFO_N * tid] = 1. * N; - if (is_resolved) - s_info[FEED_INFO_N * tid + 1] = direction * 1.0; - else - s_info[FEED_INFO_N * tid + 2] = direction * 1.0; - - int indx; - - if (is_resolved) { // if resolved inject energy and density - s_info[FEED_INFO_N * tid + 3] = feedback_energy * dV; - - indx_x = (int)floor((pos_x - xMin - 0.5 * dx) / dx); - indx_y = (int)floor((pos_y - yMin - 0.5 * dy) / dy); - indx_z = (int)floor((pos_z - zMin - 0.5 * dz) / dz); - - cell_center_x = xMin + indx_x * dx + 0.5 * dx; - cell_center_y = yMin + indx_y * dy + 0.5 * dy; - cell_center_z = zMin + indx_z * dz + 0.5 * dz; - - delta_x = 1 - (pos_x - cell_center_x) / dx; - delta_y = 1 - (pos_y - cell_center_y) / dy; - delta_z = 1 - (pos_z - cell_center_z) / dz; - indx_x += n_ghost; - indx_y += n_ghost; - indx_z += n_ghost; - - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - for (int k = 0; k < 2; k++) { - indx = (indx_x + i) + (indx_y + j) * nx_g + - (indx_z + k) * nx_g * ny_g; - - if (abs(momentum_x[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Rb: (%d, %d, %d) vx = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_x[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_y[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Rb: (%d, %d, %d) vy = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_y[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_z[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Rb: (%d, %d, %d) vz = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_z[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - - // i_frac are the fractions of energy/density to be allocated - // to each of the 8 cells. - x_frac = i * (1 - delta_x) + (1 - i) * delta_x; - y_frac = j * (1 - delta_y) + (1 - j) * delta_y; - z_frac = k * (1 - delta_z) + (1 - k) * delta_z; - - atomicAdd(&density[indx], - x_frac * y_frac * z_frac * feedback_density); - atomicAdd(&gasEnergy[indx], - x_frac * y_frac * z_frac * feedback_energy); - atomicAdd(&energy[indx], - x_frac * y_frac * z_frac * feedback_energy); - - if (abs(momentum_x[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ra: (%d, %d, %d) vx = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_x[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_y[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ra: (%d, %d, %d) vy = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_y[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_z[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ra: (%d, %d, %d) vz = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_z[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - - if (direction > 0) - local_dti = fmax( - local_dti, - Calc_Timestep(gamma, density, momentum_x, momentum_y, - momentum_z, energy, indx, dx, dy, dz)); - } - } - } - } else { // if not resolved, inject momentum and density - s_info[FEED_INFO_N * tid + 4] = feedback_momentum * dV; - - delta_x = (pos_x - xMin - indx_x * dx) / dx; - delta_y = (pos_y - yMin - indx_y * dy) / dy; - delta_z = (pos_z - zMin - indx_z * dz) / dz; - // kernel_printf("(%d):indx:(%d, %d, %d)\n", gtid, indx_x, indx_y, indx_z); - // kernel_printf("(%d): pos:(%.4e, %.4e, %.4e), delta_x (%.2e, %.2e, - // %.2e)\n", gtid, pos_x, pos_y, pos_z, delta_x, delta_y, delta_z); - - indx_x += n_ghost; - indx_y += n_ghost; - indx_z += n_ghost; - - if (abs(feedback_momentum / feedback_density * VELOCITY_UNIT * - 1e-5) > - 40000) { // injected speeds are greater than 4e4 km/s - kernel_printf( - "**** (%d, %d, %d) injected speeds are %.3e km/s\n", indx_x, - indx_y, indx_z, - feedback_momentum / feedback_density * VELOCITY_UNIT * 1e-5); - } - feedback_momentum /= sqrt(3.0); - - for (int i = -1; i < 2; i++) { - for (int j = -1; j < 2; j++) { - for (int k = -1; k < 2; k++) { - // index in array of conserved quantities - indx = (indx_x + i) + (indx_y + j) * nx_g + - (indx_z + k) * nx_g * ny_g; - - x_frac = - d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - y_frac = - frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - z_frac = - frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); - - px = x_frac * feedback_momentum; - py = y_frac * feedback_momentum; - pz = z_frac * feedback_momentum; - d = (abs(x_frac) + abs(y_frac) + abs(z_frac)) / 6 * - feedback_density + - n_0 * supernova::MU * MP / DENSITY_UNIT; - - // d = frac(i, delta_x) * frac(j, delta_y) * frac(k, delta_z) - // * feedback_density; e = frac(i, delta_x) * frac(j, - // delta_y) * frac(k, delta_z) * feedback_energy; kernel_printf("(%d, - // %d, %d): delta:(%.4e, %.4e, %.4e), frac: %.4e\n", indx_x, - // indx_y, indx_z, delta_x, delta_y, delta_z, frac(i, - // delta_x)*frac(j, delta_y)*frac(k, delta_z)); kernel_printf("(%d, - // %d, %d):(%d SN) (i:%d, j:%d, k:%d) before: %.4e\n", indx_x, - // indx_y, indx_z, N, i, j, k, - // density[indx]*DENSITY_UNIT/0.6/MP); - - // v_1 = sqrt((momentum_x[indx]*momentum_x[indx] + - // momentum_y[indx]*momentum_y[indx] + - // momentum_z[indx]*momentum_z[indx])/density[indx]/density[indx])*VELOCITY_UNIT/1e5; - // t_b = gasEnergy[indx]*ENERGY_UNIT*(gamma - - // 1)/(density[indx]*DENSITY_UNIT/0.6/MP*KB); p_b = - // sqrt(momentum_x[indx]*momentum_x[indx] + - // momentum_y[indx]*momentum_y[indx] + - // momentum_z[indx]*momentum_z[indx])*VELOCITY_UNIT/1e5; d_b = - // density[indx]*DENSITY_UNIT/0.6/MP; - - if (abs(momentum_x[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ub: (%d, %d, %d) vx = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_x[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_y[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ub: (%d, %d, %d) vy = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_y[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_z[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ub: (%d, %d, %d) vz = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_z[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - - - //printf("INDX DEBUG: n_step: %d id: %d indx: %d \n", n_step, (int) id[gtid], indx); - - if (indx >= nx_g * ny_g * nz_g) { - printf("INDX DEBUG\n"); - } - - atomicAdd(&momentum_x[indx], px); - atomicAdd(&momentum_y[indx], py); - atomicAdd(&momentum_z[indx], pz); - - - density[indx] = d; - energy[indx] = (momentum_x[indx] * momentum_x[indx] + - momentum_y[indx] * momentum_y[indx] + - momentum_z[indx] * momentum_z[indx]) / - 2 / density[indx] + - gasEnergy[indx]; - - - - // atomicAdd( &energy[indx], e ); - // atomicAdd( &density[indx], d ); - - s_info[FEED_INFO_N * tid + i_UNRES_ENERGY] += - direction * (px * px + py * py + pz * pz) / 2 / - density[indx] * dV; - - if (abs(momentum_x[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ua: (%d, %d, %d) vx = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_x[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_y[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ua: (%d, %d, %d) vy = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_y[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - if (abs(momentum_z[indx] / density[indx]) >= C_L) { - kernel_printf( - "%d, Ua: (%d, %d, %d) vz = %.3e, d = %.3e, n_0 = " - "%.3e\n", - direction, indx_x + i, indx_y + j, indx_z + k, - momentum_z[indx] / density[indx] * VELOCITY_UNIT * 1e-5, - density[indx] * DENSITY_UNIT / 0.6 / MP, n_0); - } - // gasEnergy[indx] = energy[indx] - - // (momentum_x[indx]*momentum_x[indx] + - // momentum_y[indx]*momentum_y[indx] + - // momentum_z[indx]*momentum_z[indx])/2/density[indx]; v_2 = - // sqrt((momentum_x[indx]*momentum_x[indx] + - // momentum_y[indx]*momentum_y[indx] + - // momentum_z[indx]*momentum_z[indx])/density[indx]/density[indx]) - // * VELOCITY_UNIT/1e5; t_a = - // gasEnergy[indx]*ENERGY_UNIT*(gamma - - // 1)/(density[indx]*DENSITY_UNIT/0.6/MP*KB); d_a = - // density[indx]*DENSITY_UNIT/0.6/MP; p_a = - // sqrt(momentum_x[indx]*momentum_x[indx] + - // momentum_y[indx]*momentum_y[indx] + - // momentum_z[indx]*momentum_z[indx])*VELOCITY_UNIT/1e5; - - // kernel_printf("(%d, %d, %d):(CM: %.2e, SN: %d) (i:%d, j:%d, k:%d) - // v_1: %.5e v_2: %.5e V_DIFF-> %.4f %%\n", indx_x, indx_y, - // indx_z, mass_dev[gtid], N, i, j, k, v_1, v_2, - // (v_2-v_1)/v_1*100); kernel_printf(" (%d, %d, %d):(%d SN) (i:%d, - // j:%d, k:%d) T_b: %.5e T_a: %.5e T_DIFF-> %.4f %%\n", - // indx_x, indx_y, indx_z, N, i, j, k, t_b, t_a, - // (t_a-t_b)/t_b*100); kernel_printf(" (%d, %d, %d):(%d SN) - // (i:%d, j:%d, k:%d) d_b: %.5e d_a: %.5e D_DIFF-> %.1f - // %%\n", indx_x, indx_y, indx_z, N, i, j, k, d_b, d_a, - // (d_a-d_b)/d_b*100); kernel_printf(" (%d, %d, %d):(%d SN) - // (i:%d, j:%d, k:%d) p_b: %.5e p_a: %.5e P_DIFF-> %.4f - // %%\n", indx_x, indx_y, indx_z, N, i, j, k, p_b, p_a, - // (p_a-p_b)/p_b*100); - - if (direction > 0) { - // kernel_printf("urs time:%.3e id:%d N:%d d:%.5e\n", t, id[gtid], - // N, n_0); - local_dti = fmax( - local_dti, - Calc_Timestep(gamma, density, momentum_x, momentum_y, - momentum_z, energy, indx, dx, dy, dz)); - } - } - } - } - } - if (direction > 0) atomicMax(dti, local_dti); - } - } - } - } - - __syncthreads(); - - // reduce the info from all the threads in the block - for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { - if (tid < s) { - s_info[FEED_INFO_N * tid] += s_info[FEED_INFO_N * (tid + s)]; - s_info[FEED_INFO_N * tid + 1] += s_info[FEED_INFO_N * (tid + s) + 1]; - s_info[FEED_INFO_N * tid + 2] += s_info[FEED_INFO_N * (tid + s) + 2]; - s_info[FEED_INFO_N * tid + 3] += s_info[FEED_INFO_N * (tid + s) + 3]; - s_info[FEED_INFO_N * tid + 4] += s_info[FEED_INFO_N * (tid + s) + 4]; - s_info[FEED_INFO_N * tid + 5] += s_info[FEED_INFO_N * (tid + s) + 5]; - } - __syncthreads(); - } - - if (tid == 0) { - info[FEED_INFO_N * blockIdx.x] = s_info[0]; - info[FEED_INFO_N * blockIdx.x + 1] = s_info[1]; - info[FEED_INFO_N * blockIdx.x + 2] = s_info[2]; - info[FEED_INFO_N * blockIdx.x + 3] = s_info[3]; - info[FEED_INFO_N * blockIdx.x + 4] = s_info[4]; - info[FEED_INFO_N * blockIdx.x + 5] = s_info[5]; - } -} - -Real supernova::Cluster_Feedback(Grid3D& G, FeedbackAnalysis& analysis) -{ - #ifdef CPU_TIME - G.Timer.Feedback.Start(); - #endif - - if (G.H.dt == 0) return 0.0; - - /* - if (G.Particles.n_local > supernova::n_states) { - printf("ERROR: not enough cuRAND states (%ld) for %ld local particles\n", - supernova::n_states, G.Particles.n_local); - exit(-1); - } - */ - - Real h_dti = 0.0; - int direction, ngrid; - Real h_info[6] = {0, 0, 0, 0, 0, 0}; - Real info[6]; - Real *d_dti, *d_info; - // require d_prev_dens & d_prev_N in case we have to undo feedback if the time - // step is too large. - Real* d_prev_dens; - int* d_prev_N; - - if (G.Particles.n_local > 0) { - CHECK(cudaMalloc(&d_dti, sizeof(Real))); - CHECK(cudaMemcpy(d_dti, &h_dti, sizeof(Real), cudaMemcpyHostToDevice)); - CHECK(cudaMalloc(&d_prev_dens, G.Particles.n_local * sizeof(Real))); - CHECK(cudaMalloc(&d_prev_N, G.Particles.n_local * sizeof(int))); - CHECK(cudaMemset(d_prev_dens, 0, G.Particles.n_local * sizeof(Real))); - CHECK(cudaMemset(d_prev_N, 0, G.Particles.n_local * sizeof(int))); - - ngrid = std::ceil((1. * G.Particles.n_local) / TPB_FEEDBACK); - CHECK(cudaMalloc((void**)&d_info, FEED_INFO_N * ngrid * sizeof(Real))); - } - // TODO: info collection and max dti calculation - // assumes ngrid is 1. The reason being that reduction of - // d_info is currently done on each block. Only the first block reduction - // is used - - do { - direction = 1; - if (G.Particles.n_local > 0) { - hipLaunchKernelGGL( - Cluster_Feedback_Kernel, ngrid, TPB_FEEDBACK, 0, 0, - G.Particles.n_local, G.Particles.partIDs_dev, G.Particles.pos_x_dev, - G.Particles.pos_y_dev, G.Particles.pos_z_dev, G.Particles.mass_dev, - G.Particles.age_dev, G.H.xblocal, G.H.yblocal, G.H.zblocal, - G.H.xblocal_max, G.H.yblocal_max, G.H.zblocal_max, G.H.dx, G.H.dy, - G.H.dz, G.H.nx, G.H.ny, G.H.nz, G.H.n_ghost, G.H.t, G.H.dt, d_dti, - d_info, G.C.d_density, G.C.d_GasEnergy, G.C.d_Energy, - G.C.d_momentum_x, G.C.d_momentum_y, G.C.d_momentum_z, gama, - supernova::randStates, d_prev_dens, d_prev_N, direction, dev_snr, - snr_dt, time_sn_start, time_sn_end, G.H.n_step); - - CHECK(cudaMemcpy(&h_dti, d_dti, sizeof(Real), cudaMemcpyDeviceToHost)); - } - - #ifdef MPI_CHOLLA - h_dti = ReduceRealMax(h_dti); - MPI_Barrier(world); - #endif // MPI_CHOLLA - - if (h_dti != 0 && (C_cfl / h_dti < G.H.dt)) - { - // timestep too big: need to undo the last operation - direction = -1; - if (G.Particles.n_local > 0) { - hipLaunchKernelGGL( - Cluster_Feedback_Kernel, ngrid, TPB_FEEDBACK, 0, 0, - G.Particles.n_local, G.Particles.partIDs_dev, G.Particles.pos_x_dev, - G.Particles.pos_y_dev, G.Particles.pos_z_dev, G.Particles.mass_dev, - G.Particles.age_dev, G.H.xblocal, G.H.yblocal, G.H.zblocal, - G.H.xblocal_max, G.H.yblocal_max, G.H.zblocal_max, G.H.dx, G.H.dy, - G.H.dz, G.H.nx, G.H.ny, G.H.nz, G.H.n_ghost, G.H.t, G.H.dt, d_dti, - d_info, G.C.d_density, G.C.d_GasEnergy, G.C.d_Energy, - G.C.d_momentum_x, G.C.d_momentum_y, G.C.d_momentum_z, gama, - supernova::randStates, d_prev_dens, d_prev_N, direction, dev_snr, - snr_dt, time_sn_start, time_sn_end, G.H.n_step); - - CHECK(cudaDeviceSynchronize()); - } - G.H.dt = C_cfl / h_dti; - } - - } while (direction == -1); - - if (G.Particles.n_local > 0) { - CHECK(cudaMemcpy(&h_info, d_info, FEED_INFO_N * sizeof(Real), - cudaMemcpyDeviceToHost)); - CHECK(cudaFree(d_dti)); - CHECK(cudaFree(d_info)); - CHECK(cudaFree(d_prev_dens)); - CHECK(cudaFree(d_prev_N)); - } - - #ifdef MPI_CHOLLA - MPI_Reduce(&h_info, &info, FEED_INFO_N, MPI_CHREAL, MPI_SUM, root, world); - #else - info = h_info; - #endif - - analysis.countSN += (int)info[supernova::SN]; - analysis.countResolved += (int)info[supernova::RESOLVED]; - analysis.countUnresolved += (int)info[supernova::NOT_RESOLVED]; - analysis.totalEnergy += info[supernova::ENERGY]; - analysis.totalMomentum += info[supernova::MOMENTUM]; - analysis.totalUnresEnergy += info[supernova::UNRES_ENERGY]; - - Real resolved_ratio = 0.0; - if (info[supernova::RESOLVED] > 0 || info[supernova::NOT_RESOLVED] > 0) { - resolved_ratio = - info[supernova::RESOLVED] / - (info[supernova::RESOLVED] + info[supernova::NOT_RESOLVED]); - } - Real global_resolved_ratio = 0.0; - if (analysis.countResolved > 0 || analysis.countUnresolved > 0) { - global_resolved_ratio = analysis.countResolved / - (analysis.countResolved + analysis.countUnresolved); - } - - chprintf("iteration %d: number of SN: %d, ratio of resolved %.3e\n", - G.H.n_step, (long)info[supernova::SN], resolved_ratio); - chprintf( - " this iteration: energy: %.5e erg. momentum: %.5e S.M. km/s " - "unres_energy: %.5e erg\n", - info[supernova::ENERGY] * MASS_UNIT * LENGTH_UNIT * LENGTH_UNIT / - TIME_UNIT / TIME_UNIT, - info[supernova::MOMENTUM] * VELOCITY_UNIT / 1e5, - info[supernova::UNRES_ENERGY] * MASS_UNIT * LENGTH_UNIT * LENGTH_UNIT / - TIME_UNIT / TIME_UNIT); - chprintf( - " cummulative: #SN: %d, ratio of resolved (R: %d, UR: %d) = %.3e\n", - (long)analysis.countSN, (long)analysis.countResolved, - (long)analysis.countUnresolved, global_resolved_ratio); - chprintf( - " energy: %.5e erg. Total momentum: %.5e S.M. km/s, Total unres " - "energy: %.5e\n", - analysis.totalEnergy * MASS_UNIT * LENGTH_UNIT * LENGTH_UNIT / TIME_UNIT / - TIME_UNIT, - analysis.totalMomentum * VELOCITY_UNIT / 1e5, - analysis.totalUnresEnergy * MASS_UNIT * LENGTH_UNIT * LENGTH_UNIT / - TIME_UNIT / TIME_UNIT); - - #ifdef CPU_TIME - G.Timer.Feedback.End(); - #endif - - return h_dti; -} - -#endif // SUPERNOVA & PARTICLES_GPU & PARTICLE_IDS & PARTICLE_AGE diff --git a/src/particles/particles_3D.cpp b/src/particles/particles_3D.cpp index 36ffcf2ce..d4c137379 100644 --- a/src/particles/particles_3D.cpp +++ b/src/particles/particles_3D.cpp @@ -699,7 +699,9 @@ void Particles_3D::Initialize_Disk_Stellar_Clusters(struct parameters *P) // radius unsigned long int N = 13; //(long int)(6.5e6 * 0.9272485558395908); // // 15kpc radius Real total_mass = 0; - Real upper_limit_cluster_mass = 1e7; + Real upper_limit_cluster_mass = 8e7; + Real SFR = 1e3; + Real t_cluster_creation = -4e4; long lost_particles = 0; part_int_t id = -1; while (total_mass < upper_limit_cluster_mass) { @@ -716,6 +718,12 @@ void Particles_3D::Initialize_Disk_Stellar_Clusters(struct parameters *P) y = R * sin(phi); z = zDist(generator); + // set creation time of cluster on how long + // it would take star formation to add that + // much mass + t_cluster_creation += cluster_mass/SFR; + chprintf("cluster %d, age %.4e, mass %.4e\n", id, t_cluster_creation, cluster_mass); + if (x < G.xMin || x >= G.xMax) continue; if (y < G.yMin || y >= G.yMax) continue; if (z < G.zMin || z >= G.zMax) continue; @@ -738,7 +746,7 @@ void Particles_3D::Initialize_Disk_Stellar_Clusters(struct parameters *P) temp_grav_y.push_back(0.0); temp_grav_z.push_back(0.0); temp_mass.push_back(cluster_mass); - temp_age.push_back(0.0); + temp_age.push_back(t_cluster_creation); temp_ids.push_back(id); } diff --git a/src/particles/supernova.h b/src/particles/supernova.h deleted file mode 100644 index 6d6fc7d11..000000000 --- a/src/particles/supernova.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once -#if defined(PARTICLES_GPU) && defined(SUPERNOVA) - - #include "../analysis/feedback_analysis.h" - #include "../global/global.h" - #ifdef O_HIP - #include - #include - #else - #include - #include - #endif // O_HIP - -namespace supernova -{ -const int SN = 0, RESOLVED = 1, NOT_RESOLVED = 2, ENERGY = 3, MOMENTUM = 4, - UNRES_ENERGY = 5; - -// supernova rate: 1SN / 100 solar masses per 36 Myr -static const Real DEFAULT_SNR = 2.8e-7; -static const Real ENERGY_PER_SN = - 1e51 / MASS_UNIT * TIME_UNIT * TIME_UNIT / LENGTH_UNIT / LENGTH_UNIT; -static const Real MASS_PER_SN = 10.0; // 10 solarMasses per SN -static const Real FINAL_MOMENTUM = - 2.8e5 / LENGTH_UNIT * 1e5 * - TIME_UNIT; // 2.8e5 M_s km/s * n_0^{-0.17} -> eq.(34) Kim & Ostriker (2015) -static const Real MU = 0.6; -static const Real R_SH = - 0.0302; // 30.2 pc * n_0^{-0.46} -> eq.(31) Kim & Ostriker (2015) -static const Real DEFAULT_SN_END = - 40000; // default value for when SNe stop (40 Myr) -static const Real DEFAULT_SN_START = - 4000; // default value for when SNe start (4 Myr) - -void initState(struct parameters* P, part_int_t n_local, - Real allocation_factor = 1); -Real Cluster_Feedback(Grid3D& G, FeedbackAnalysis& sn_analysis); -} // namespace supernova -#endif // PARTICLES_GPU && SUPERNOVA diff --git a/src/utils/timing_functions.cpp b/src/utils/timing_functions.cpp index 050977e1b..ef26dcdea 100644 --- a/src/utils/timing_functions.cpp +++ b/src/utils/timing_functions.cpp @@ -111,12 +111,12 @@ void Time::Initialize() #ifdef CHEMISTRY_GPU &(Chemistry = OneTime("Chemistry")), #endif - #ifdef SUPERNOVA + #ifdef FEEDBACK &(Feedback = OneTime("Feedback")), #ifdef ANALYSIS &(FeedbackAnalysis = OneTime("FeedbackAnalysis")), #endif - #endif // SUPERNOVA + #endif // FEEDBACK &(Total = OneTime("Total")), }; From cc0d39923d8ded8e38a5da1abf0dfcd6809b2f62 Mon Sep 17 00:00:00 2001 From: ojwg Date: Tue, 16 May 2023 18:08:49 -0400 Subject: [PATCH 03/10] cloudy cooling bugfix and photo-e heating, plus feedback changes --- builds/make.type.disk | 2 +- src/analysis/feedback_analysis.cpp | 7 ++- src/analysis/feedback_analysis.h | 3 +- src/analysis/feedback_analysis_gpu.cu | 43 ++++++++++++---- src/cooling/cooling_cuda.cu | 30 ++++++++--- src/feedback/feedback.cu | 74 +++++++++------------------ src/feedback/feedback.h | 6 +-- src/global/global.cpp | 12 +++-- src/global/global.h | 13 +++-- src/grid/grid3D.cpp | 2 +- src/main.cpp | 17 +++--- src/model/disk_galaxy.h | 2 +- src/particles/particles_3D.cpp | 4 +- 13 files changed, 115 insertions(+), 100 deletions(-) diff --git a/builds/make.type.disk b/builds/make.type.disk index 5691853b5..d8d8d0f4c 100644 --- a/builds/make.type.disk +++ b/builds/make.type.disk @@ -36,7 +36,7 @@ DFLAGS += -DDISK_ICS DFLAGS += -DDENSITY_FLOOR DFLAGS += -DTEMPERATURE_FLOOR DFLAGS += -DCOOLING_GPU -#DFLAGS += -DCLOUDY_COOL +DFLAGS += -DCLOUDY_COOL DFLAGS += -DDE #DFLAGS += -DCPU_TIME DFLAGS += -DAVERAGE_SLOW_CELLS diff --git a/src/analysis/feedback_analysis.cpp b/src/analysis/feedback_analysis.cpp index 4f870a33c..0e5b49dc7 100644 --- a/src/analysis/feedback_analysis.cpp +++ b/src/analysis/feedback_analysis.cpp @@ -9,8 +9,13 @@ #define VRMS_CUTOFF_DENSITY (0.01 * 0.6 * MP / DENSITY_UNIT) -FeedbackAnalysis::FeedbackAnalysis(Grid3D& G) +FeedbackAnalysis::FeedbackAnalysis(Grid3D& G, struct parameters *P) { + // set distance limits beyond which contributions to the V_rms don't + // make sense, such as too close to the simulation volume edge or + // too high above the disk. + r_max = P->xlen / 2 * 0.975; + z_max = 0.15; // 150 pc above/below the disk plane // allocate arrays h_circ_vel_x = (Real*)malloc(G.H.n_cells * sizeof(Real)); h_circ_vel_y = (Real*)malloc(G.H.n_cells * sizeof(Real)); diff --git a/src/analysis/feedback_analysis.h b/src/analysis/feedback_analysis.h index 5e937996e..844da2cd3 100644 --- a/src/analysis/feedback_analysis.h +++ b/src/analysis/feedback_analysis.h @@ -8,6 +8,7 @@ class FeedbackAnalysis { Real *h_circ_vel_x, *h_circ_vel_y; + Real r_max, z_max; #ifdef PARTICLES_GPU Real *d_circ_vel_x, *d_circ_vel_y; @@ -24,7 +25,7 @@ class FeedbackAnalysis Real totalWindMomentum{0}; Real totalWindEnergy{0}; - FeedbackAnalysis(Grid3D& G); + FeedbackAnalysis(Grid3D& G, struct parameters *P); ~FeedbackAnalysis(); void Compute_Gas_Velocity_Dispersion(Grid3D& G); diff --git a/src/analysis/feedback_analysis_gpu.cu b/src/analysis/feedback_analysis_gpu.cu index 9ef268216..a7b2a06d3 100644 --- a/src/analysis/feedback_analysis_gpu.cu +++ b/src/analysis/feedback_analysis_gpu.cu @@ -6,7 +6,6 @@ #include "feedback_analysis.h" #ifdef PARTICLES_GPU - #define MU 0.6 // in cgs, this is 0.01 cm^{-3} #define MIN_DENSITY (0.01 * MP * MU * LENGTH_UNIT * LENGTH_UNIT * LENGTH_UNIT / MASS_UNIT) // 148279.7 #define TPB_ANALYSIS 1024 @@ -33,9 +32,13 @@ __device__ void warpReduce(volatile Real *buff, size_t tid) } } -void __global__ Reduce_Tubulence_kernel(int nx, int ny, int nz, int n_ghost, Real *density, Real *momentum_x, - Real *momentum_y, Real *momentum_z, Real *circ_vel_x, Real *circ_vel_y, - Real *partial_mass, Real *partial_vel) +void __global__ Reduce_Tubulence_kernel( + int nx, int ny, int nz, int n_ghost, + Real xbound, Real ybound, Real zbound, Real dx, + Real dy, Real dz, Real nx_local_start, Real ny_local_start, Real nz_local_start, + Real r_max, Real z_max, Real *density, Real *momentum_x, Real *momentum_y, + Real *momentum_z, Real *circ_vel_x, Real *circ_vel_y, Real *partial_mass, + Real *partial_vel) { __shared__ Real s_mass[TPB_ANALYSIS]; __shared__ Real s_vel[TPB_ANALYSIS]; @@ -50,15 +53,24 @@ void __global__ Reduce_Tubulence_kernel(int nx, int ny, int nz, int n_ghost, Rea s_mass[tid] = 0; s_vel[tid] = 0; Real vx, vy, vz; + Real x, y, z, r; + if (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && zid > n_ghost - 1 && zid < nz - n_ghost && density[id] > MIN_DENSITY) { - s_mass[tid] = density[id]; - vx = momentum_x[id] / density[id]; - vy = momentum_y[id] / density[id]; - vz = momentum_z[id] / density[id]; - s_vel[tid] = - ((vx - circ_vel_x[id]) * (vx - circ_vel_x[id]) + (vy - circ_vel_y[id]) * (vy - circ_vel_y[id]) + (vz * vz)) * - density[id]; + x = xbound + (nx_local_start + xid - n_ghost + 0.5) * dx; + y = ybound + (ny_local_start + yid - n_ghost + 0.5) * dy; + z = zbound + (nz_local_start + zid - n_ghost + 0.5) * dz; + r = sqrt(x*x + y*y + z*z); + + if ( r <= r_max && abs(z) <= z_max ) { + s_mass[tid] = density[id]; + vx = momentum_x[id] / density[id]; + vy = momentum_y[id] / density[id]; + vz = momentum_z[id] / density[id]; + s_vel[tid] = + ((vx - circ_vel_x[id]) * (vx - circ_vel_x[id]) + (vy - circ_vel_y[id]) * (vy - circ_vel_y[id]) + (vz * vz)) * + density[id]; + } } __syncthreads(); @@ -153,7 +165,16 @@ void FeedbackAnalysis::Compute_Gas_Velocity_Dispersion_GPU(Grid3D &G) Real total_mass = 0; Real total_vel = 0; + Real nx_loc_strt = 0, ny_loc_strt = 0, nz_loc_strt = 0; + #ifdef MPI_CHOLLA + nx_loc_strt = nx_local_start; + ny_loc_strt = ny_local_start; + nz_loc_strt = nz_local_start; + #endif + hipLaunchKernelGGL(Reduce_Tubulence_kernel, ngrid, TPB_ANALYSIS, 0, 0, G.H.nx, G.H.ny, G.H.nz, G.H.n_ghost, + G.H.xbound, G.H.ybound, G.H.zbound, G.H.dx, G.H.dy, G.H.dz, nx_loc_strt, ny_loc_strt, + nz_loc_strt, r_max, z_max, G.C.d_density, G.C.d_momentum_x, G.C.d_momentum_y, G.C.d_momentum_z, d_circ_vel_x, d_circ_vel_y, d_partial_mass, d_partial_vel); diff --git a/src/cooling/cooling_cuda.cu b/src/cooling/cooling_cuda.cu index 4b09527d0..9d3b816ab 100644 --- a/src/cooling/cooling_cuda.cu +++ b/src/cooling/cooling_cuda.cu @@ -154,7 +154,6 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int cool = Cloudy_cool(n, T, coolTexObj, heatTexObj); #else cool = CIE_cool(n, T); - // printf("%d %d %d %e %e %e\n", xid, yid, zid, n, T, cool); #endif // and send back from kernel @@ -327,6 +326,10 @@ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cuda Real lambda = 0.0; // cooling rate, erg s^-1 cm^3 Real H = 0.0; // heating rate, erg s^-1 cm^3 Real cool = 0.0; // cooling per unit volume, erg /s / cm^3 + // the following two values are for photoelectric heating + // based on description given in Kim et al. 2015. + Real n_av = 100.0; //TODO mean density in the sim volume, cm^3 (make this an argument?) + Real H_pe = 0.0; float log_n, log_T; log_n = log10(n); log_T = log10(T); @@ -341,15 +344,28 @@ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cuda // variable so it is treated as "x" This is why the Texture calls are T first, // then n: Bilinear_Texture(tex, log_T, log_n) - // don't cool below 10 K - if (log10(T) > 1.0) { + // cloudy cooling tables cut off at 10^9 K, use the CIE analytic fit above + // this temp. + if (log10(T) > 9.0) { + lambda = 0.45 * log10(T) - 26.065; + } else if (log10(T) >= 1.0) { // don't cool below 10 K lambda = Bilinear_Texture(coolTexObj, log_T, log_n); - } else - lambda = 0.0; - H = Bilinear_Texture(heatTexObj, log_T, log_n); + H = Bilinear_Texture(heatTexObj, log_T, log_n); + } + + // apply photoelectric heating under 10,000 K + if (log10(T) < 4.0) { + H_pe = n_av * 1.0e-26; + } // cooling rate per unit volume - cool = n * n * (powf(10, lambda) - powf(10, H)); + if (log10(T) > 9.0) { + cool = n * n * pow(10, lambda); + } else if (log10(T) >= 4.0) { + cool = n * n * (pow(10, lambda) - pow(10, H)); + } else { + cool = n * (n * (pow(10, lambda) - pow(10, H)) - H_pe); + } // printf("DEBUG Cloudy L350: %.17e\n",cool); return cool; } diff --git a/src/feedback/feedback.cu b/src/feedback/feedback.cu index aba2ad6fd..9b2b6f5fa 100644 --- a/src/feedback/feedback.cu +++ b/src/feedback/feedback.cu @@ -39,8 +39,6 @@ namespace feedback { - feedback_prng_t* randStates; - part_int_t n_states; Real *dev_snr, snr_dt, time_sn_start, time_sn_end; Real *dev_sw_p, *dev_sw_e, sw_dt, time_sw_start, time_sw_end; int snr_n; @@ -136,35 +134,21 @@ inline __device__ Real GetAverageDensity(Real* density, int xi, int yi, int zi, inline __device__ Real GetAverageNumberDensity_CGS(Real* density, int xi, int yi, int zi, int nx_grid, int ny_grid, int n_ghost) { - return GetAverageDensity(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * DENSITY_UNIT / - (feedback::MU * MP); + return GetAverageDensity(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * + DENSITY_UNIT / (MU * MP); } -__global__ void initState_kernel(unsigned int seed, - feedback_prng_t* states) -{ - int id = blockIdx.x * blockDim.x + threadIdx.x; - curand_init(seed, id, 0, &states[id]); -} - #ifndef NO_SN_FEEDBACK /** - * @brief Does 2 things: + * @brief * -# Read in SN rate data from Starburst 99. If no file exists, assume a * constant rate. - * -# Initialize the cuRAND state, which is analogous to the concept of - * generators in CPU code. The state object maintains configuration and status - * the cuRAND context for each thread on the GPU. Initialize more than the - * number of local particles since the latter will change through MPI transfers. * * @param P pointer to parameters struct. Passes in starburst 99 filename and * random number gen seed. - * @param n_local number of local particles on the GPU - * @param allocation_factor */ -void feedback::initState(struct parameters* P, part_int_t n_local, - Real allocation_factor) +void feedback::initState(struct parameters* P) { chprintf("feedback::initState start\n"); std::string snr_filename(P->snr_filename); @@ -222,20 +206,6 @@ void feedback::initState(struct parameters* P, part_int_t n_local, time_sn_start = DEFAULT_SN_START; time_sn_end = DEFAULT_SN_END; } - - // Now initialize the poisson random number generator state. - n_states = n_local * allocation_factor; - cudaMalloc((void**)&randStates, n_states * sizeof(feedback_prng_t)); - - int ngrid = (n_states + TPB_FEEDBACK - 1) / TPB_FEEDBACK; - dim3 grid(ngrid); - dim3 block(TPB_FEEDBACK); - - hipLaunchKernelGGL(initState_kernel, grid, block, 0, 0, P->prng_seed, - randStates); - CHECK(cudaDeviceSynchronize()); - chprintf("feedback::initState end: n_states=%ld, ngrid=%d, threads=%d\n", - n_states, ngrid, TPB_FEEDBACK); } #endif // NO_SN_FEEDBACK @@ -725,7 +695,7 @@ __device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, if (is_resolved) { // inject energy and density if (time_direction > 0) { - s_info[FEED_INFO_N * tid + i_RES] = 1.0; + s_info[FEED_INFO_N * tid + i_RES] = 1. * N; s_info[FEED_INFO_N * tid + i_ENERGY] = feedback_energy * dV; } local_dti = Apply_Resolved_SN(pos_x, pos_y, pos_z, @@ -740,7 +710,7 @@ __device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, feedback_momentum = time_direction * feedback::FINAL_MOMENTUM * pow(n_0, -0.17) * pow(fabsf(N), 0.93) / dV / sqrt(3.0); if (time_direction > 0) { - s_info[FEED_INFO_N * tid + i_UNRES] = time_direction * 1.0; + s_info[FEED_INFO_N * tid + i_UNRES] = 1. * N; s_info[FEED_INFO_N * tid + i_MOMENTUM] = feedback_momentum * dV * sqrt(3.0); s_info[FEED_INFO_N * tid + i_UNRES_ENERGY] = feedback_energy * dV; } @@ -875,8 +845,9 @@ __device__ void Cluster_Feedback_Helper( part_int_t n_local, Real* pos_x_dev, // Ignore this particle, exit if (ignore) return; - bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); - if (!is_alone) return; + //bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + //if (is_alone) kernel_printf(" particle not alone: step %d, id %ld\n", n_step, id_dev[gtid]); + //if (!is_alone) return; // note age_dev is actually the time of birth Real age = t - age_dev[gtid]; @@ -1019,8 +990,9 @@ __global__ void Adjust_Cluster_Mass_Kernel(part_int_t n_local, Real* pos_x_dev, // Ignore this particle, exit if (ignore) return; - bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); - if (!is_alone) return; + //bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + //if (is_alone) kernel_printf(" particle not alone: step %d, id %ld\n", n_step, id_dev[gtid]); + //if (!is_alone) return; Real age = t - age_dev[gtid]; @@ -1093,8 +1065,9 @@ __global__ void Set_Ave_Density_Kernel( // Ignore this particle, exit if (ignore) return; - bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); - if (!is_alone) return; + //bool is_alone = Particle_Is_Alone(pos_x_dev, pos_y_dev, pos_z_dev, n_local, gtid, 6*dx); + //if (is_alone) kernel_printf(" particle not alone: step %d, id %ld\n", n_step, id_dev[gtid]); + //if (!is_alone) return; bool is_sn_feedback = false; bool is_wind_feedback = false; @@ -1134,7 +1107,7 @@ __global__ void Set_Ave_Density_Kernel( // resolved SN feedback does not average densities. if (!is_resolved && N > 0) { - ave_dens = n_0 * feedback::MU * MP / DENSITY_UNIT; + ave_dens = n_0 * MU * MP / DENSITY_UNIT; Set_Average_Density(indx_x, indx_y, indx_z, nx_g, ny_g, n_ghost, density, ave_dens); } @@ -1173,7 +1146,7 @@ Real feedback::Cluster_Feedback(Grid3D& G, FeedbackAnalysis& analysis) CHECK(cudaMalloc(&d_prev_dens, G.Particles.n_local * sizeof(Real))); CHECK(cudaMemset(d_prev_dens, 0, G.Particles.n_local * sizeof(Real))); - ngrid = std::ceil((1. * G.Particles.n_local) / TPB_FEEDBACK); + ngrid = (G.Particles.n_local - 1) / TPB_FEEDBACK + 1; CHECK(cudaMalloc((void**)&d_info, FEED_INFO_N * sizeof(Real))); // before applying feedback, set gas density around clusters to the @@ -1292,24 +1265,23 @@ Real feedback::Cluster_Feedback(Grid3D& G, FeedbackAnalysis& analysis) if (procID == 0) { #endif - analysis.countSN += (int)info[feedback::SN]; - analysis.countResolved += (int)info[feedback::RESOLVED]; - analysis.countUnresolved += (int)info[feedback::NOT_RESOLVED]; + analysis.countSN += (long)info[feedback::SN]; + analysis.countResolved += (long)info[feedback::RESOLVED]; + analysis.countUnresolved += (long)info[feedback::NOT_RESOLVED]; analysis.totalEnergy += info[feedback::ENERGY]; analysis.totalMomentum += info[feedback::MOMENTUM]; analysis.totalUnresEnergy += info[feedback::UNRES_ENERGY]; analysis.totalWindMomentum += info[i_WIND_MOMENTUM]; analysis.totalWindEnergy+= info[i_WIND_ENERGY]; + chprintf("iteration %d, t %.4e, dt %.4e", G.H.n_step, G.H.t, G.H.dt); + + #ifndef NO_SN_FEEDBACK Real global_resolved_ratio = 0.0; if (analysis.countResolved > 0 || analysis.countUnresolved > 0) { global_resolved_ratio = analysis.countResolved / (analysis.countResolved + analysis.countUnresolved); } - - chprintf("iteration %d, t %.4e, dt %.4e", G.H.n_step, G.H.t, G.H.dt); - - #ifndef NO_SN_FEEDBACK chprintf(": number of SN: %d,(R: %d, UR: %d)\n", (int)info[feedback::SN], (long)info[feedback::RESOLVED], (long)info[feedback::NOT_RESOLVED]); chprintf( diff --git a/src/feedback/feedback.h b/src/feedback/feedback.h index f8c5839dc..47ccbae3e 100644 --- a/src/feedback/feedback.h +++ b/src/feedback/feedback.h @@ -27,7 +27,6 @@ namespace feedback static const Real MASS_PER_SN = 10.0; // 2.8e5 M_s km/s * n_0^{-0.17} -> eq.(34) Kim & Ostriker (2015) static const Real FINAL_MOMENTUM = 2.8e5 / LENGTH_UNIT * 1e5 * TIME_UNIT; - static const Real MU = 0.6; // 30.2 pc * n_0^{-0.46} -> eq.(31) Kim & Ostriker (2015) static const Real R_SH = 0.0302; // default value for when SNe stop (40 Myr) @@ -35,15 +34,12 @@ namespace feedback // default value for when SNe start (4 Myr) static const Real DEFAULT_SN_START = 4000; - extern feedback_prng_t* randStates; - extern part_int_t n_states; extern Real *dev_snr, snr_dt, time_sn_end, time_sn_start; extern Real *dev_sw_p, *dev_sw_e, sw_dt, time_sw_start, time_sw_end; #ifndef NO_SN_FEEDBACK - void initState(struct parameters* P, part_int_t n_local, - Real allocation_factor = 1); + void initState(struct parameters* P); #endif #ifndef NO_WIND_FEEDBACK void initWindState(struct parameters* P); diff --git a/src/global/global.cpp b/src/global/global.cpp index bda0ddf10..ac0648b2d 100644 --- a/src/global/global.cpp +++ b/src/global/global.cpp @@ -367,13 +367,15 @@ void parse_param(char *name, char *value, struct parameters *parms) } else if (strcmp(name, "prng_seed") == 0) { parms->prng_seed = atoi(value); #endif // PARTICLES -#if defined(FEEDBACK) && !defined(NO_SN_FEEDBACK) - } else if (strcmp(name, "snr_filename") == 0) +#ifdef FEEDBACK + #ifndef NO_SN_FEEDBACK + } else if (strcmp(name, "snr_filename") == 0) { strncpy(parms->snr_filename, value, MAXLEN); -#endif -#if defined(FEEDBACK) && !defined(NO_WIND_FEEDBACK) - } else if (strcmp(name, "sw_filename") == 0) + #endif + #ifndef NO_WIND_FEEDBACK + } else if (strcmp(name, "sw_filename") == 0) { strncpy(parms->sw_filename, value, MAXLEN); + #endif #endif #ifdef ROTATED_PROJECTION } else if (strcmp(name, "nxr") == 0) { diff --git a/src/global/global.h b/src/global/global.h index 612d4c974..b25dd3c3e 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -56,9 +56,14 @@ typedef double Real; #define LOG_FILE_NAME "run_output.log" // Conserved Floor Values +//#define TEMP_FLOOR 1e1 // 10K for cloudy cooling +//#define DENS_FLOOR 14.83 // 1e-6 cm^-3 #define TEMP_FLOOR 1e-3 #define DENS_FLOOR 1e-5 // in code units +// mean molecular weight +#define MU 0.6 + // Parameter for Enzo dual Energy Condition #define DE_ETA_1 \ 0.001 // Ratio of U to E for which Internal Energy is used to compute the @@ -271,11 +276,13 @@ struct parameters { // machine dependent seed will be generated. std::uint_fast64_t prng_seed = 0; #endif // PARTICLES -#if defined(FEEDBACK) && !defined(NO_SN_FEEDBACK) +#ifdef FEEDBACK + #ifndef NO_SN_FEEDBACK char snr_filename[MAXLEN]; -#endif -#if defined(FEEDBACK) && !defined(NO_WIND_FEEDBACK) + #endif + #ifndef NO_WIND_FEEDBACK char sw_filename[MAXLEN]; + #endif #endif #ifdef ROTATED_PROJECTION int nxr; diff --git a/src/grid/grid3D.cpp b/src/grid/grid3D.cpp index dac2795c8..c8519558e 100644 --- a/src/grid/grid3D.cpp +++ b/src/grid/grid3D.cpp @@ -420,7 +420,7 @@ Real Grid3D::Update_Grid(void) Real U_floor, density_floor; density_floor = H.density_floor; // Minimum of internal energy from minumum of temperature - U_floor = H.temperature_floor * KB / (gama - 1) / MP / SP_ENERGY_UNIT; + U_floor = H.temperature_floor * KB / (gama - 1) / MU / MP / SP_ENERGY_UNIT; #ifdef COSMOLOGY U_floor = H.temperature_floor / (gama - 1) / MP * KB * 1e-10; // ( km/s )^2 U_floor /= Cosmo.v_0_gas * Cosmo.v_0_gas / Cosmo.current_a / Cosmo.current_a; diff --git a/src/main.cpp b/src/main.cpp index acae425ea..eb2898a38 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -151,19 +151,14 @@ int main(int argc, char *argv[]) #endif #if defined(FEEDBACK) && defined(PARTICLE_AGE) - FeedbackAnalysis sn_analysis(G); + FeedbackAnalysis sn_analysis(G, &P); #ifndef NO_SN_FEEDBACK - #ifdef MPI_CHOLLA - feedback::initState(&P, G.Particles.n_total_initial); - #else - feedback::initState(&P, G.Particles.n_local); - #endif // MPI_CHOLLA + feedback::initState(&P); #endif // NO_SN_FEEDBACK -#endif // FEEDBACK && PARTICLE_AGE - -#ifndef NO_WIND_FEEDBACK - feedback::initWindState(&P); -#endif + #ifndef NO_WIND_FEEDBACK + feedback::initWindState(&P); + #endif +#endif // FEEDBACK && PARTICLE_AGE #ifdef STAR_FORMATION star_formation::Initialize(G); diff --git a/src/model/disk_galaxy.h b/src/model/disk_galaxy.h index 04249853f..004c2ccc1 100644 --- a/src/model/disk_galaxy.h +++ b/src/model/disk_galaxy.h @@ -184,7 +184,7 @@ namespace Galaxies { // all masses in M_sun and all distances in kpc // static DiskGalaxy MW(6.5e10, 3.5, (3.5/5.0), 1.0e12, 261, 20, 157.0); -static ClusteredDiskGalaxy MW(1e4, 5e5, 6.5e10, 2.7, 0.7, 1.077e12, 261, 18, 157.0); +static ClusteredDiskGalaxy MW(1e2, 5e5, 6.5e10, 2.7, 0.7, 1.077e12, 261, 18, 157.0); static DiskGalaxy M82(1.0e10, 0.8, 0.15, 5.0e10, 0.8 / 0.015, 10, 100.0); }; // namespace Galaxies diff --git a/src/particles/particles_3D.cpp b/src/particles/particles_3D.cpp index ed9716ead..247ddf21b 100644 --- a/src/particles/particles_3D.cpp +++ b/src/particles/particles_3D.cpp @@ -679,8 +679,8 @@ void Particles_3D::Initialize_Disk_Stellar_Clusters(struct parameters *P) // radius unsigned long int N = 13; //(long int)(6.5e6 * 0.9272485558395908); // // 15kpc radius Real total_mass = 0; - Real upper_limit_cluster_mass = 8e7; - Real SFR = 1e3; + Real upper_limit_cluster_mass = 8e6; + Real SFR = 1e2; Real t_cluster_creation = -4e4; long lost_particles = 0; part_int_t id = -1; From b1033e1e376b6fb9e7658e787d63b89e554236bf Mon Sep 17 00:00:00 2001 From: ojwg Date: Tue, 23 May 2023 23:46:19 -0400 Subject: [PATCH 04/10] cloudy cooling 'fabs' fix. Disk gas cutoff --- src/cooling/cooling_cuda.cu | 11 ++--------- src/model/disk_ICs.cpp | 8 ++++---- src/particles/particles_3D.cpp | 2 +- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/cooling/cooling_cuda.cu b/src/cooling/cooling_cuda.cu index 9d3b816ab..84801d33c 100644 --- a/src/cooling/cooling_cuda.cu +++ b/src/cooling/cooling_cuda.cu @@ -122,9 +122,9 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int del_T = cool * dt * TIME_UNIT * (gamma - 1.0) / (n * KB); // limit change in temperature to 1% - while (del_T / T > 0.01) { + while (fabs(del_T) / T > 0.01) { // what dt gives del_T = 0.01*T? - dt_sub = 0.01 * T * n * KB / (cool * TIME_UNIT * (gamma - 1.0)); + dt_sub = 0.01 * T * n * KB / (fabs(cool) * TIME_UNIT * (gamma - 1.0)); // apply that dt T -= cool * dt_sub * TIME_UNIT * (gamma - 1.0) / (n * KB); // how much time is left from the original timestep? @@ -149,13 +149,6 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int ge -= KB * del_T / (mu * MP * (gamma - 1.0) * SP_ENERGY_UNIT); #endif - // calculate cooling rate for new T - #ifdef CLOUDY_COOL - cool = Cloudy_cool(n, T, coolTexObj, heatTexObj); - #else - cool = CIE_cool(n, T); - #endif - // and send back from kernel dev_conserved[4 * n_cells + id] = E; #ifdef DE diff --git a/src/model/disk_ICs.cpp b/src/model/disk_ICs.cpp index 8e4bede3f..80fd30a54 100644 --- a/src/model/disk_ICs.cpp +++ b/src/model/disk_ICs.cpp @@ -65,16 +65,16 @@ Real Sigma_disk_D3D(Real r, Real *hdp) // return the exponential surface density Real Sigma_0 = hdp[9]; Real R_g = hdp[10]; - Real R_c = 4.5; + Real R_c = 1.9; Real Sigma; - Real delta = 0.1; + Real delta = 0.01; Real norm = log(1.0 / 3.0); Sigma = Sigma_0 * exp(-r / R_g); // taper the edge of the disk to 0 if (r < R_c) { - Sigma *= 2.0 - 1.0 / (1.0 - exp((r - (4.5 - delta * norm)) / delta)); + Sigma *= 2.0 - 1.0 / (1.0 - exp((r - (R_c - delta * norm)) / delta)); } else { - Sigma *= 1.0 / (1.0 - exp(((4.5 + delta * norm) - r) / delta)) - 1.0; + Sigma *= 1.0 / (1.0 - exp(((R_c + delta * norm) - r) / delta)) - 1.0; } return Sigma; } diff --git a/src/particles/particles_3D.cpp b/src/particles/particles_3D.cpp index 247ddf21b..b29cea281 100644 --- a/src/particles/particles_3D.cpp +++ b/src/particles/particles_3D.cpp @@ -657,7 +657,7 @@ void Particles_3D::Initialize_Disk_Stellar_Clusters(struct parameters *P) Real R_d = Galaxies::MW.getR_d(); // MW stellar disk scale length in kpc Real Z_d = Galaxies::MW.getZ_d(); // MW stellar height scale length in kpc Real R_max = sqrt(P->xlen * P->xlen + P->ylen * P->ylen) / 2; - R_max = P->xlen / 2.0; + R_max = 1.8; // P->xlen / 2.0; real_vector_t temp_pos_x; real_vector_t temp_pos_y; From 488defeaaa06a3eab4e3cfaa52b8a898c43cabbd Mon Sep 17 00:00:00 2001 From: ojwg Date: Tue, 30 May 2023 22:05:32 -0400 Subject: [PATCH 05/10] slow cell location; MW gas fraction --- src/grid/grid3D.cpp | 9 ++++++++- src/hydro/hydro_cuda.cu | 15 ++++++++++----- src/hydro/hydro_cuda.h | 6 ++++-- src/model/disk_ICs.cpp | 6 ++---- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/grid/grid3D.cpp b/src/grid/grid3D.cpp index e1868be3d..3f7fa3b81 100644 --- a/src/grid/grid3D.cpp +++ b/src/grid/grid3D.cpp @@ -513,7 +513,14 @@ Real Grid3D::Update_Grid(void) // Set the min_delta_t for averaging a slow cell Real max_dti_slow; max_dti_slow = 1 / H.min_dt_slow; - Average_Slow_Cells(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, H.dx, H.dy, H.dz, gama, max_dti_slow); + int nx_off = 0, ny_off = 0, nz_off = 0; + #ifdef MPI_CHOLLA + nx_off = nx_local_start; //offsets + ny_off = ny_local_start; + nz_off = nz_local_start; + #endif + Average_Slow_Cells(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, H.dx, H.dy, H.dz, gama, max_dti_slow, + H.xbound, H.ybound, H.zbound, nx_off, ny_off, nz_off); #endif // AVERAGE_SLOW_CELLS // ==Calculate the next time step using Calc_dt_GPU from hydro/hydro_cuda.h== diff --git a/src/hydro/hydro_cuda.cu b/src/hydro/hydro_cuda.cu index dad6f3b66..9bac3711f 100644 --- a/src/hydro/hydro_cuda.cu +++ b/src/hydro/hydro_cuda.cu @@ -597,7 +597,8 @@ Real Calc_dt_GPU(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n #ifdef AVERAGE_SLOW_CELLS void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, Real dy, - Real dz, Real gamma, Real max_dti_slow) + Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, int nx_offset, + int ny_offset, int nz_offset) { // set values for GPU kernels int n_cells = nx * ny * nz; @@ -609,12 +610,13 @@ void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost if (nx > 1 && ny > 1 && nz > 1) { // 3D hipLaunchKernelGGL(Average_Slow_Cells_3D, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, n_fields, - dx, dy, dz, gamma, max_dti_slow); + dx, dy, dz, gamma, max_dti_slow, xbound, ybound, zbound, nx_offset, ny_offset, nz_offset); } } __global__ void Average_Slow_Cells_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, - Real dy, Real dz, Real gamma, Real max_dti_slow) + Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, + Real zbound, int nx_offset, int ny_offset, int nz_offset) { int id, xid, yid, zid, n_cells; Real d, d_inv, vx, vy, vz, E, max_dti; @@ -644,11 +646,14 @@ __global__ void Average_Slow_Cells_3D(Real *dev_conserved, int nx, int ny, int n temp = (gamma - 1) * (E - 0.5 * (speed * speed) * d) * ENERGY_UNIT / (d * DENSITY_UNIT / 0.6 / MP) / KB; P = (E - 0.5 * d * (vx * vx + vy * vy + vz * vz)) * (gamma - 1.0); cs = sqrt(d_inv * gamma * P) * VELOCITY_UNIT * 1e-5; + Real x = xbound + (nx_offset + xid - n_ghost + 0.5) * dx; + Real y = ybound + (ny_offset + yid - n_ghost + 0.5) * dy; + Real z = zbound + (nz_offset + zid - n_ghost + 0.5) * dz; // Average this cell kernel_printf( - " Average Slow Cell [ %d %d %d ] -> dt_cell=%f dt_min=%f, n=%.3e, " + " Average Slow Cell [ %.5e %.5e %.5e ] -> dt_cell=%f dt_min=%f, n=%.3e, " "T=%.3e, v=%.3e (%.3e, %.3e, %.3e), cs=%.3e\n", - xid, yid, zid, 1. / max_dti, 1. / max_dti_slow, dev_conserved[id] * DENSITY_UNIT / 0.6 / MP, temp, + x, y, z, 1. / max_dti, 1. / max_dti_slow, dev_conserved[id] * DENSITY_UNIT / 0.6 / MP, temp, speed * VELOCITY_UNIT * 1e-5, vx * VELOCITY_UNIT * 1e-5, vy * VELOCITY_UNIT * 1e-5, vz * VELOCITY_UNIT * 1e-5, cs); Average_Cell_All_Fields(xid, yid, zid, nx, ny, nz, n_cells, n_fields, dev_conserved); diff --git a/src/hydro/hydro_cuda.h b/src/hydro/hydro_cuda.h index a5c4ab713..446da81af 100644 --- a/src/hydro/hydro_cuda.h +++ b/src/hydro/hydro_cuda.h @@ -78,10 +78,12 @@ __global__ void Sync_Energies_3D(Real *dev_conserved, int nx, int ny, int nz, in #ifdef AVERAGE_SLOW_CELLS void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, Real dy, - Real dz, Real gamma, Real max_dti_slow); + Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, int nx_offset, + int ny_offset, int nz_offset); __global__ void Average_Slow_Cells_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, - Real dy, Real dz, Real gamma, Real max_dti_slow); + Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, + int nx_offset, int ny_offset, int nz_offset); #endif #ifdef TEMPERATURE_FLOOR diff --git a/src/model/disk_ICs.cpp b/src/model/disk_ICs.cpp index 80fd30a54..66dfa3516 100644 --- a/src/model/disk_ICs.cpp +++ b/src/model/disk_ICs.cpp @@ -18,7 +18,6 @@ #include "../utils/error_handling.h" #include "disk_galaxy.h" -// #define DISK_ICS // function with logarithms used in NFW definitions Real log_func(Real y) { return log(1 + y) - y / (1 + y); } @@ -736,7 +735,6 @@ Real halo_density_D3D(Real r, Real *r_halo, Real *rho_halo, Real dr, int nr) * \brief Initialize the grid with a 3D disk. */ void Grid3D::Disk_3D(parameters p) { -#ifdef DISK_ICS int i, j, k, id; Real x_pos, y_pos, z_pos, r, phi; @@ -771,7 +769,8 @@ void Grid3D::Disk_3D(parameters p) mu = 0.6; R_g = 2.0 * R_d; // gas scale length in kpc - Sigma_0 = 0.25 * M_d / (2 * M_PI * R_g * R_g); // central surface density in Msun/kpc^2 + Sigma_0 = 0.15 * M_d / (2 * M_PI * R_g * R_g); // central surface density in Msun/kpc^2 + // FIXME 0.15 -> 0.25 for M82 H_g = z_d; // initial guess for gas scale height // rho_floor = 1.0e3; //ICs minimum density in Msun/kpc^3 @@ -1074,5 +1073,4 @@ void Grid3D::Disk_3D(parameters p) free(r_halo); free(rho_halo); -#endif // DISK_ICS } From df471769e13486011015af573053828fefb80cec Mon Sep 17 00:00:00 2001 From: ojwg Date: Fri, 21 Jul 2023 17:43:25 -0400 Subject: [PATCH 06/10] refactor to use naming conventions --- builds/make.type.disk | 11 +-- src/feedback/feedback.cu | 149 +++++++++++++++------------------------ src/feedback/feedback.h | 4 +- src/main.cpp | 4 +- 4 files changed, 62 insertions(+), 106 deletions(-) diff --git a/builds/make.type.disk b/builds/make.type.disk index d8d8d0f4c..959d97556 100644 --- a/builds/make.type.disk +++ b/builds/make.type.disk @@ -1,12 +1,10 @@ MPI_GPU = -DMPI_GPU DFLAGS += -DPARTICLES -#DFLAGS += -DPARTICLES_CPU DFLAGS += -DPARTICLES_GPU -#DFLAGS += -DONLY_PARTICLES DFLAGS += -DPARTICLE_IDS -#DFLAGS += -DSINGLE_PARTICLE_MASS DFLAGS += -DPARTICLE_AGE DFLAGS += -DFEEDBACK #this flag requires PARTICLE_AGE, PARTICLE_IDS +#DFLAGS += -DONLY_RESOLVED #DFLAGS += -DNO_WIND_FEEDBACK #DFLAGS += -DNO_SN_FEEDBACK DFLAGS += -DANALYSIS @@ -31,16 +29,13 @@ DFLAGS += -DPPMC DFLAGS += -DHLLC DFLAGS += -DVL -DFLAGS += -DDISK_ICS - DFLAGS += -DDENSITY_FLOOR DFLAGS += -DTEMPERATURE_FLOOR DFLAGS += -DCOOLING_GPU -DFLAGS += -DCLOUDY_COOL +#DFLAGS += -DCLOUDY_COOL DFLAGS += -DDE #DFLAGS += -DCPU_TIME DFLAGS += -DAVERAGE_SLOW_CELLS -DFLAGS += -DHYDRO_GPU OUTPUT ?= -DOUTPUT -DHDF5 -DSLICES -DPROJECTION DFLAGS += $(OUTPUT) @@ -50,4 +45,4 @@ DFLAGS += $(MPI_GPU) DFLAGS += -DPARALLEL_OMP DFLAGS += -DN_OMP_THREADS=$(OMP_NUM_THREADS) -#DFLAGS += -DCUDA_ERROR_CHECK +DFLAGS += -DCUDA_ERROR_CHECK diff --git a/src/feedback/feedback.cu b/src/feedback/feedback.cu index 9b2b6f5fa..a42d7ee48 100644 --- a/src/feedback/feedback.cu +++ b/src/feedback/feedback.cu @@ -83,19 +83,21 @@ inline __device__ Real Calc_Timestep(Real gamma, Real* density, should be dx*1/2. In the above the 1/2 factor is normalize over 2 cells/direction. */ -inline __device__ Real frac(int i, Real dx) +inline __device__ Real Frac(int i, Real dx) { return (-0.5 * i * i - 0.5 * i + 1 + i * dx) * 0.5; } -inline __device__ Real d_fr(int i, Real dx) +inline __device__ Real D_Frac(int i, Real dx) { return (dx > 0.5) * i * (1 - 2 * dx) + ((i + 1) * dx + 0.5 * (i - 1)) - 3 * (i - 1) * (i + 1) * (0.5 - dx); } - +/** This function used for debugging potential race conditions. Feedback from neighboring + particles could simultaneously alter one hydro cell's conserved quantities. + */ inline __device__ bool Particle_Is_Alone(Real* pos_x_dev, Real* pos_y_dev, Real* pos_z_dev, part_int_t n_local, int gtid, Real dx) { @@ -115,7 +117,7 @@ inline __device__ bool Particle_Is_Alone(Real* pos_x_dev, Real* pos_y_dev, } -inline __device__ Real GetAverageDensity(Real* density, int xi, int yi, int zi, +inline __device__ Real Get_Average_Density(Real* density, int xi, int yi, int zi, int nx_grid, int ny_grid, int n_ghost) { Real d_average = 0.0; @@ -131,10 +133,10 @@ inline __device__ Real GetAverageDensity(Real* density, int xi, int yi, int zi, } -inline __device__ Real GetAverageNumberDensity_CGS(Real* density, int xi, int yi, +inline __device__ Real Get_Average_Number_Density_CGS(Real* density, int xi, int yi, int zi, int nx_grid, int ny_grid, int n_ghost) { - return GetAverageDensity(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * + return Get_Average_Density(density, xi, yi, zi, nx_grid, ny_grid, n_ghost) * DENSITY_UNIT / (MU * MP); } @@ -148,9 +150,9 @@ inline __device__ Real GetAverageNumberDensity_CGS(Real* density, int xi, int yi * @param P pointer to parameters struct. Passes in starburst 99 filename and * random number gen seed. */ -void feedback::initState(struct parameters* P) +void feedback::Init_State(struct parameters* P) { - chprintf("feedback::initState start\n"); + chprintf("feedback::Init_State start\n"); std::string snr_filename(P->snr_filename); if (not snr_filename.empty()) { chprintf("Specified a SNR filename %s.\n", snr_filename.data()); @@ -218,9 +220,9 @@ void feedback::initState(struct parameters* P) * * @param P pointer to parameters struct. Passes in starburst 99 filepath */ - void feedback::initWindState(struct parameters* P) + void feedback::Init_Wind_State(struct parameters* P) { - chprintf("initWindState start\n"); + chprintf("Init_Wind_State start\n"); std::string sw_filename(P->sw_filename); if (sw_filename.empty()) { chprintf("must specify a stellar wind file.\n"); @@ -303,7 +305,7 @@ void feedback::initState(struct parameters* P) * @param t_end cluster age when stellar winds turn off (kyr). * @return flux (in Cholla force units) per solar mass. */ -__device__ Real GetWindFlux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, +__device__ Real Get_Wind_Flux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, Real t_end) { if (t < t_start || t >= t_end) return 0; @@ -325,7 +327,7 @@ __device__ Real GetWindFlux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, * @param t_end cluster age when stellar winds turn off (kyr). * @return power (in Cholla units) per solar mass. */ - __device__ Real GetWindPower(Real t, Real* dev_sw_e, Real sw_dt, Real t_start, + __device__ Real Get_Wind_Power(Real t, Real* dev_sw_e, Real sw_dt, Real t_start, Real t_end) { if (t < t_start || t >= t_end) return 0; @@ -346,7 +348,7 @@ __device__ Real GetWindFlux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, * @param flux * @return mass flux in g/s per solar mass */ - __device__ Real GetWindMass(Real flux, Real power) + __device__ Real Get_Wind_Mass(Real flux, Real power) { if (flux <= 0 || power <= 0) return 0; return flux * flux / power / 2; @@ -364,7 +366,7 @@ __device__ Real GetWindFlux(Real t, Real* dev_sw_p, Real sw_dt, Real t_start, * @param t_end cluster age when SNR drops to zero. * @return double number of SNe per kyr per solar mass */ -__device__ Real GetSNRate(Real t, Real* dev_snr, Real snr_dt, Real t_start, +__device__ Real Get_SN_Rate(Real t, Real* dev_snr, Real snr_dt, Real t_start, Real t_end) { if (t < t_start || t >= t_end) return 0; @@ -388,7 +390,7 @@ __device__ Real GetSNRate(Real t, Real* dev_snr, Real snr_dt, Real t_start, * @param cluster_id * @return number of supernovae */ -inline __device__ int GetNumberOfSNeForCluster(Real ave_num_sn, int n_step, +inline __device__ int Get_Number_Of_SNe_In_Cluster(Real ave_num_sn, int n_step, part_int_t cluster_id) { feedback_prng_t state; @@ -484,9 +486,9 @@ __device__ Real Apply_Unresolved_SN(Real pos_x, Real pos_y, Real pos_z, for (int i = -1; i < 2; i++) { for (int j = -1; j < 2; j++) { for (int k = -1; k < 2; k++) { - x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + x_frac = D_Frac(i, delta_x) * Frac(j, delta_y) * Frac(k, delta_z); + y_frac = Frac(i, delta_x) * D_Frac(j, delta_y) * Frac(k, delta_z); + z_frac = Frac(i, delta_x) * Frac(j, delta_y) * D_Frac(k, delta_z); mag += sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac); } @@ -501,9 +503,9 @@ __device__ Real Apply_Unresolved_SN(Real pos_x, Real pos_y, Real pos_z, (indx_y + j + n_ghost) * nx_g + (indx_z + k + n_ghost) * nx_g * ny_g; - x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + x_frac = D_Frac(i, delta_x) * Frac(j, delta_y) * Frac(k, delta_z); + y_frac = Frac(i, delta_x) * D_Frac(j, delta_y) * Frac(k, delta_z); + z_frac = Frac(i, delta_x) * Frac(j, delta_y) * D_Frac(k, delta_z); Real px = x_frac * feedback_momentum; Real py = y_frac * feedback_momentum; @@ -567,9 +569,9 @@ __device__ Real Apply_Wind(Real pos_x, Real pos_y, Real pos_z, Real xMin, for (int i = -1; i < 2; i++) { for (int j = -1; j < 2; j++) { for (int k = -1; k < 2; k++) { - x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + x_frac = D_Frac(i, delta_x) * Frac(j, delta_y) * Frac(k, delta_z); + y_frac = Frac(i, delta_x) * D_Frac(j, delta_y) * Frac(k, delta_z); + z_frac = Frac(i, delta_x) * Frac(j, delta_y) * D_Frac(k, delta_z); mag += sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac); } @@ -584,9 +586,9 @@ __device__ Real Apply_Wind(Real pos_x, Real pos_y, Real pos_z, Real xMin, (indx_y + j + n_ghost) * nx_g + (indx_z + k + n_ghost) * nx_g * ny_g; - x_frac = d_fr(i, delta_x) * frac(j, delta_y) * frac(k, delta_z); - y_frac = frac(i, delta_x) * d_fr(j, delta_y) * frac(k, delta_z); - z_frac = frac(i, delta_x) * frac(j, delta_y) * d_fr(k, delta_z); + x_frac = D_Frac(i, delta_x) * Frac(j, delta_y) * Frac(k, delta_z); + y_frac = Frac(i, delta_x) * D_Frac(j, delta_y) * Frac(k, delta_z); + z_frac = Frac(i, delta_x) * Frac(j, delta_y) * D_Frac(k, delta_z); Real px = x_frac * feedback_momentum; Real py = y_frac * feedback_momentum; @@ -596,7 +598,7 @@ __device__ Real Apply_Wind(Real pos_x, Real pos_y, Real pos_z, Real xMin, f_energy = sqrt(x_frac*x_frac + y_frac*y_frac + z_frac*z_frac) / mag * feedback_energy; - atomicAdd(&density[indx] , f_dens); + atomicAdd( &density[indx], f_dens); atomicAdd(&momentum_x[indx], px); atomicAdd(&momentum_y[indx], py); atomicAdd(&momentum_z[indx], pz); @@ -616,28 +618,8 @@ __device__ Real Apply_Wind(Real pos_x, Real pos_y, Real pos_z, Real xMin, Real cell_dti = Calc_Timestep(gamma, density, momentum_x, momentum_y, momentum_z, energy, indx, dx, dy, dz); local_dti = fmax(local_dti, cell_dti); - /* - s_info[FEED_INFO_N * threadIdx.x + i_WIND_ENERGY] += - (energy[indx] - priorEnergy) * dx * dy * dz; - - if (threadIdx.x == 0) { - kernel_printf("WF: mom: %.4e, pe: %.4e, e: %.4e, sinfo: %.7e\n", - feedback_momentum, priorEnergy, energy[indx], - s_info[FEED_INFO_N * threadIdx.x + i_WIND_ENERGY]); - } - */ - } - /* - if (threadIdx.x == 0 && i == -1 && j == 0 && k == 1) { - if (time_direction > 0) { - kernel_printf("~~~~ WF: n_s: %d, id: %lld, l: %d [%d, %d, %d], fe: %.5e, px: %.5e, py: %.5e, pz: %.5e, %.5e\n", - n_step, id, loop, i, j, k, f_energy, px, py, pz, fd); - } else { - kernel_printf("~~~~ -WF: n_s: %d, id: %lld, l: %d [%d, %d, %d], fe: %.5e, px: %.5e, py: %.5e, pz: %.5e, %.5e\n", - n_step, id, loop, i, j, k, f_energy, px, py, pz, fd); - } } - */ + } // k loop } // j loop } // i loop @@ -663,9 +645,9 @@ __device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, Real local_dti = 0.0; int n_cells = nx_g * ny_g * nz_g; - Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, + Real average_num_sn = Get_SN_Rate(age, dev_snr, snr_dt, time_sn_start, time_sn_end) * mass_dev[gtid] * dt; - int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]) + int N = Get_Number_Of_SNe_In_Cluster(average_num_sn, n_step, id_dev[gtid]) * time_direction; /* if (gtid == 0) { @@ -680,7 +662,7 @@ __device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, n_0 = prev_dens[gtid]; } else { Real* density = conserved_dev; - n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, + n_0 = Get_Average_Number_Density_CGS(density, indx_x, indx_y, indx_z, nx_g, ny_g, n_ghost); prev_dens[gtid] = n_0; s_info[FEED_INFO_N * tid] = 1. * N; @@ -690,7 +672,11 @@ __device__ void SN_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, feedback_density = N * feedback::MASS_PER_SN / dV; Real shell_radius = feedback::R_SH * pow(n_0, -0.46) * pow(fabsf(N), 0.29); + #ifdef ONLY_RESOLVED + bool is_resolved = true; + #else bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; + #endif if (is_resolved) { // inject energy and density @@ -749,11 +735,11 @@ __device__ void Wind_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, if (age < 0 || age > time_sw_end) return; - feedback_momentum = GetWindFlux(age, dev_sw_p, sw_dt, time_sw_start, time_sw_end); + feedback_momentum = Get_Wind_Flux(age, dev_sw_p, sw_dt, time_sw_start, time_sw_end); // no sense in proceeding if there is no feedback. if (feedback_momentum == 0) return; - feedback_energy = GetWindPower(age, dev_sw_e, sw_dt, time_sw_start, time_sw_end); - feedback_density = GetWindMass(feedback_momentum, feedback_energy); + feedback_energy = Get_Wind_Power(age, dev_sw_e, sw_dt, time_sw_start, time_sw_end); + feedback_density = Get_Wind_Mass(feedback_momentum, feedback_energy); // feedback_momentum now becomes momentum component along one direction. feedback_momentum *= mass_dev[gtid] * dt / dV / sqrt(3.0) * time_direction; @@ -773,35 +759,6 @@ __device__ void Wind_Feedback(Real pos_x, Real pos_y, Real pos_z, Real age, s_info[FEED_INFO_N * tid + i_WIND_ENERGY] = feedback_energy * dV; } - //if (tid == 0 ) { - /* - if (abs(feedback_momentum) > 0) { - if (feedback_momentum > 0) { - kernel_printf("WIND: %.5e yrs, %3f log dynes, %f log power \n", - age * 1000, - log10(abs(feedback_momentum)/mass_dev[gtid]*S_99_TOTAL_MASS*dV*sqrt(3.0)*FORCE_UNIT/dt), - log10(abs(feedback_energy)/mass_dev[gtid]*S_99_TOTAL_MASS*dV* - MASS_UNIT*VELOCITY_UNIT*VELOCITY_UNIT/TIME_UNIT/dt)); - } else { - kernel_printf("-WIND: %.5e yrs, %3f log dynes, %f log power \n", - age * 1000, - log10(abs(feedback_momentum)/mass_dev[gtid]*S_99_TOTAL_MASS*dV*sqrt(3.0)*FORCE_UNIT/dt), - log10(abs(feedback_energy)/mass_dev[gtid]*S_99_TOTAL_MASS*dV* - MASS_UNIT*VELOCITY_UNIT*VELOCITY_UNIT/TIME_UNIT/dt)); - } - */ - /* - if (time_direction > 0) { - kernel_printf(" WIND: n_s: %d, id: %lld, l: %d, p: %.9e, e: %.9e, d: %.9e\n", - n_step, id_dev[gtid], loop, feedback_momentum, feedback_energy, feedback_density); - } else { - kernel_printf(" -WIND: n_s: %d, id: %lld, l: %d, p: %.9e, e: %.9e, d: %.9e\n", - n_step, id_dev[gtid], loop, feedback_momentum, feedback_energy, feedback_density); - - } - */ - //} - local_dti = Apply_Wind( pos_x, pos_y, pos_z, xMin, yMin, zMin, dx, dy, dz, nx_g, ny_g, n_ghost, n_cells, gamma, conserved_dev, time_direction, feedback_density, feedback_momentum, feedback_energy, n_step, id_dev[gtid], @@ -997,18 +954,18 @@ __global__ void Adjust_Cluster_Mass_Kernel(part_int_t n_local, Real* pos_x_dev, Real age = t - age_dev[gtid]; #ifndef NO_SN_FEEDBACK - Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, + Real average_num_sn = Get_SN_Rate(age, dev_snr, snr_dt, time_sn_start, time_sn_end) * mass_dev[gtid] * dt; - int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]); + int N = Get_Number_Of_SNe_In_Cluster(average_num_sn, n_step, id_dev[gtid]); mass_dev[gtid] -= N * feedback::MASS_PER_SN; #endif #ifndef NO_WIND_FEEDBACK - Real feedback_momentum = GetWindFlux(age, dev_sw_p, sw_dt, time_sw_start, + Real feedback_momentum = Get_Wind_Flux(age, dev_sw_p, sw_dt, time_sw_start, time_sw_end); - Real feedback_energy = GetWindPower(age, dev_sw_e, sw_dt, time_sw_start, + Real feedback_energy = Get_Wind_Power(age, dev_sw_e, sw_dt, time_sw_start, time_sw_end); - Real feedback_mass_rate = GetWindMass(feedback_momentum, feedback_energy); + Real feedback_mass_rate = Get_Wind_Mass(feedback_momentum, feedback_energy); mass_dev[gtid] -= feedback_mass_rate * dt; #endif @@ -1082,7 +1039,7 @@ __global__ void Set_Ave_Density_Kernel( Real age = t - age_dev[gtid]; if (is_wind_feedback) { if (time_sw_start <= age && age <= time_sw_end) { - ave_dens = GetAverageDensity(density, indx_x, indx_y, indx_z, + ave_dens = Get_Average_Density(density, indx_x, indx_y, indx_z, nx_g, ny_g, n_ghost); Set_Average_Density(indx_x, indx_y, indx_z, nx_g, ny_g, n_ghost, density, ave_dens); @@ -1093,17 +1050,21 @@ __global__ void Set_Ave_Density_Kernel( } if (is_sn_feedback) { if (time_sn_start <= age && age <= time_sn_end) { - Real average_num_sn = GetSNRate(age, dev_snr, snr_dt, time_sn_start, - time_sn_end) * mass_dev[gtid] * dt; - int N = GetNumberOfSNeForCluster(average_num_sn, n_step, id_dev[gtid]); + Real average_num_sn = Get_SN_Rate(age, dev_snr, snr_dt, time_sn_start, + time_sn_end) * mass_dev[gtid] * dt; + int N = Get_Number_Of_SNe_In_Cluster(average_num_sn, n_step, id_dev[gtid]); /* if (gtid == 0) { kernel_printf("AVEDENS n_step: %d, id: %lld, N: %d\n", n_step, id_dev[gtid], N); }*/ - Real n_0 = GetAverageNumberDensity_CGS(density, indx_x, indx_y, indx_z, + Real n_0 = Get_Average_Number_Density_CGS(density, indx_x, indx_y, indx_z, nx_g, ny_g, n_ghost); Real shell_radius = feedback::R_SH * pow(n_0, -0.46) * pow(N, 0.29); + #ifdef ONLY_RESOLVED + bool is_resolved = true; + #else bool is_resolved = 3 * max(dx, max(dy, dz)) <= shell_radius; + #endif // resolved SN feedback does not average densities. if (!is_resolved && N > 0) { diff --git a/src/feedback/feedback.h b/src/feedback/feedback.h index 47ccbae3e..7e344e84d 100644 --- a/src/feedback/feedback.h +++ b/src/feedback/feedback.h @@ -39,10 +39,10 @@ namespace feedback #ifndef NO_SN_FEEDBACK - void initState(struct parameters* P); + void Init_State(struct parameters* P); #endif #ifndef NO_WIND_FEEDBACK - void initWindState(struct parameters* P); + void Init_Wind_State(struct parameters* P); #endif Real Cluster_Feedback(Grid3D& G, FeedbackAnalysis& sn_analysis); } // namespace supernova diff --git a/src/main.cpp b/src/main.cpp index 81d5d7f24..86ecf0d1d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -162,10 +162,10 @@ int main(int argc, char *argv[]) #if defined(FEEDBACK) && defined(PARTICLE_AGE) FeedbackAnalysis sn_analysis(G, &P); #ifndef NO_SN_FEEDBACK - feedback::initState(&P); + feedback::Init_State(&P); #endif // NO_SN_FEEDBACK #ifndef NO_WIND_FEEDBACK - feedback::initWindState(&P); + feedback::Init_Wind_State(&P); #endif #endif // FEEDBACK && PARTICLE_AGE From ecb1ccda46627e7de518a9f342d5fe4f88af133f Mon Sep 17 00:00:00 2001 From: Alwin Date: Mon, 14 Aug 2023 15:10:24 -0400 Subject: [PATCH 07/10] add temperature extreme value check --- src/utils/debug_utilities.cu | 61 ++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/utils/debug_utilities.cu b/src/utils/debug_utilities.cu index 9a1157aca..5c90fe19f 100644 --- a/src/utils/debug_utilities.cu +++ b/src/utils/debug_utilities.cu @@ -5,6 +5,67 @@ #include "../io/io.h" // provides chprintf #include "../utils/error_handling.h" // provides chexit +__global__ void Check_For_Extreme_Temperature_Kernel(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, Real upper_limit, int marker, bool* out_bool) +{ + // Calculate temperature + int id = threadIdx.x + blockIdx.x * blockDim.x; + if (id >= n_cells) { + return; + } + const Real d = dev_conserved[id]; + const Real E = dev_conserved[id + n_cells * grid_enum::Energy]; + const Real px = dev_conserved[id + n_cells * grid_enum::momentum_x]; + const Real py = dev_conserved[id + n_cells * grid_enum::momentum_y]; + const Real pz = dev_conserved[id + n_cells * grid_enum::momentum_z]; + + const Real E_kinetic = 0.5 * (px * px + py * py + pz * pz) / d; + /* + const Real vx = dev_conserved[1 * n_cells + id] / d; + const Real vy = dev_conserved[2 * n_cells + id] / d; + const Real vz = dev_conserved[3 * n_cells + id] / d; + const Real E_kinetic = 0.5 * d * (vx * vx + vy * vy + vz * vz); + */ + + + const Real E_thermal = E - E_kinetic; + const Real p = (E_thermal) * (gamma - 1.0); + + const Real mu = 0.6; + const Real n = d * DENSITY_UNIT / (mu * MP); + const Real T_kelvin = p * PRESSURE_UNIT / (n * KB); + + + if (T_kelvin <= lower_limit || T_kelvin >= upper_limit) { + out_bool[0] = true; + kernel_printf("Check_For_Extreme_Temperature_Kernel found Value: %g E: %g E_thermal: %g E_kinetic: %g Marker: %d Thread: %d\n", T_kelvin, E, E_thermal, E_kinetic, marker, id); + } +} + + +void Check_For_Extreme_Temperature(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, Real upper_limit, int marker) +{ + bool host_out_bool[1] = {false}; + bool* out_bool; + cudaMalloc((void**)&out_bool, sizeof(bool)); + cudaMemcpy(out_bool, host_out_bool, sizeof(bool), cudaMemcpyHostToDevice); + int ngrid = (n_cells + TPB - 1) / TPB; + dim3 dim1dGrid(ngrid, 1, 1); + dim3 dim1dBlock(TPB, 1, 1); + + hipLaunchKernelGGL(Check_For_Extreme_Temperature_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, n_cells, gamma, lower_limit, upper_limit, marker, out_bool); + + cudaMemcpy(host_out_bool, out_bool, sizeof(bool), cudaMemcpyDeviceToHost); + cudaFree(out_bool); + + if (host_out_bool[0]) { + chexit(-1); + } +} + + + + + __global__ void Dump_Values_Kernel(Real* device_array, int array_size, int marker) { int tid = threadIdx.x + blockIdx.x * blockDim.x; From c4dd94f4b51b9b3503c3614d4c9782f21974a595 Mon Sep 17 00:00:00 2001 From: Alwin Date: Mon, 14 Aug 2023 15:25:15 -0400 Subject: [PATCH 08/10] add TI, rework cooling loop, separate photoelectric heating --- src/cooling/cooling_cuda.cu | 172 ++++++++++++++++++++++++------------ 1 file changed, 117 insertions(+), 55 deletions(-) diff --git a/src/cooling/cooling_cuda.cu b/src/cooling/cooling_cuda.cu index 265548617..42e5fe810 100644 --- a/src/cooling/cooling_cuda.cu +++ b/src/cooling/cooling_cuda.cu @@ -18,6 +18,9 @@ cudaTextureObject_t coolTexObj = 0; cudaTextureObject_t heatTexObj = 0; +__device__ Real Photoelectric_Heating(Real n, Real T, Real n_av); +__device__ Real TI_cool(Real n, Real T); + void Cooling_Update(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dt, Real gamma) { int n_cells = nx * ny * nz; @@ -58,8 +61,8 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int } Real d, E; - Real n, T, T_init; - Real del_T, dt_sub; + Real n, T; + Real del_T; Real mu; // mean molecular weight Real cool; // cooling rate per volume, erg/s/cm^3 // #ifndef DE @@ -84,10 +87,17 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int // load values of density and pressure d = dev_conserved[id]; E = dev_conserved[4 * n_cells + id]; + + // don't apply cooling if this thread crashed if (E < 0.0 || E != E) { + /* Alwin: this is a leftover debug printf from when we were checking cooling for small temperature outputs. Delete at leisure. + kernel_printf("WARNING: bad energy in cooling_cuda E=%e [xid, yid, zid] = [%d, %d, %d] \n", E, xid, yid, zid); + */ return; } + + // #ifndef DE vx = dev_conserved[1 * n_cells + id] / d; vy = dev_conserved[2 * n_cells + id] / d; @@ -96,7 +106,7 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int p = fmax(p, (Real)TINY_NUMBER); // #endif #ifdef DE - ge = dev_conserved[(n_fields - 1) * n_cells + id] / d; + ge = dev_conserved[(n_fields - 1) * n_cells + id]; ge = fmax(ge, (Real)TINY_NUMBER); #endif @@ -104,56 +114,68 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int n = d * DENSITY_UNIT / (mu * MP); // calculate the temperature of the gas - T_init = p * PRESSURE_UNIT / (n * KB); + #ifdef DE - T_init = d * ge * (gamma - 1.0) * PRESSURE_UNIT / (n * KB); + const Real T_init = ge * (gamma - 1.0) * PRESSURE_UNIT / (n * KB); + #else + const Real T_init = p * PRESSURE_UNIT / (n * KB); #endif + const Real dt_init = dt; // calculate cooling rate per volume T = T_init; - // call the cooling function - #ifdef CLOUDY_COOL - cool = Cloudy_cool(n, T, coolTexObj, heatTexObj); - #else - cool = CIE_cool(n, T); - #endif - // calculate change in temperature given dt - del_T = cool * dt * TIME_UNIT * (gamma - 1.0) / (n * KB); - - // limit change in temperature to 1% - while (fabs(del_T) / T > 0.01) { - // what dt gives del_T = 0.01*T? - dt_sub = 0.01 * T * n * KB / (fabs(cool) * TIME_UNIT * (gamma - 1.0)); - // apply that dt - T -= cool * dt_sub * TIME_UNIT * (gamma - 1.0) / (n * KB); - // how much time is left from the original timestep? - dt -= dt_sub; - // calculate cooling again + while (true) { #ifdef CLOUDY_COOL cool = Cloudy_cool(n, T, coolTexObj, heatTexObj); + cool -= Photoelectric_Heating(n, T, 100.0); #else cool = CIE_cool(n, T); #endif - // calculate new change in temperature + + #ifdef TI_COOL + cool = TI_cool(n, T); + #endif del_T = cool * dt * TIME_UNIT * (gamma - 1.0) / (n * KB); + if (fabs(del_T) > 0.01 * T) { + // Evolve T and dt so T changes by 1% + dt -= dt * 0.01 * T / fabs(del_T); + T -= del_T * 0.01 * T / fabs(del_T); + } else { + T -= del_T; + break; + } } - // calculate final temperature - T -= del_T; + const Real T_final = T; + + /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete at leisure. + // If the temperature has changed by a lot, whine vehemently + if ((T_init / T_final > 1e3) || (T_final / T_init > 1e3)) { + //kernel_printf("Cooling Cuda Large Temperature Change whine: | T_init (K): %e | T_final (K): %e | dt (cholla): %e | n (cgs): %e \n", T_init, T_final, dt_init, n); + } + */ // adjust value of energy based on total change in temperature - del_T = T_init - T; // total change in T - E -= n * KB * del_T / ((gamma - 1.0) * ENERGY_UNIT); + const Real delta_E = n * KB * (T_final - T_init) / ((gamma - 1.0) * ENERGY_UNIT); + E += delta_E; #ifdef DE - ge -= KB * del_T / (mu * MP * (gamma - 1.0) * SP_ENERGY_UNIT); + ge += delta_E; #endif // and send back from kernel dev_conserved[4 * n_cells + id] = E; #ifdef DE - dev_conserved[(n_fields - 1) * n_cells + id] = d * ge; + dev_conserved[(n_fields - 1) * n_cells + id] = ge; #endif + + + /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete at leisure. + if (T_final < 1e2) { + kernel_printf("Cooling Cuda Small Temperature Whine: | T_init (K): %e | T_final (K): %e | dt (cholla): %e | n (cgs): %e \n", T_init, T_final, dt_init, n); + } + */ + } } @@ -316,13 +338,11 @@ __device__ Real CIE_cool(Real n, Real T) tables at z = 0 with solar metallicity and an HM05 UV background. */ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cudaTextureObject_t heatTexObj) { - Real lambda = 0.0; // cooling rate, erg s^-1 cm^3 - Real H = 0.0; // heating rate, erg s^-1 cm^3 - Real cool = 0.0; // cooling per unit volume, erg /s / cm^3 - // the following two values are for photoelectric heating - // based on description given in Kim et al. 2015. - Real n_av = 100.0; // TODO mean density in the sim volume, cm^3 (make this an argument?) - Real H_pe = 0.0; + Real lambda = 0.0; // log cooling rate, erg s^-1 cm^3 + Real cooling = 0.0; // cooling per unit volume, erg /s / cm^3 + Real heating = 0.0; // cooling per unit volume, erg /s / cm^3 + + // To keep texture code simple, we use floats (which have built-in support) as opposed to doubles (which would require casting) float log_n, log_T; log_n = log10(n); log_T = log10(T); @@ -330,39 +350,81 @@ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cuda // remap coordinates for texture // remapped = (input - TABLE_MIN_VALUE)*(1/TABLE_SPACING) // remapped = (input - TABLE_MIN_VALUE)*(NUM_CELLS_PER_DECADE) - log_T = (log_T - 1.0) * 10; - log_n = (log_n + 6.0) * 10; + const Real remap_log_T = (log_T - 1.0) * 10; + const Real remap_log_n = (log_n + 6.0) * 10; // Note: although the cloudy table columns are n,T,L,H , T is the fastest // variable so it is treated as "x" This is why the Texture calls are T first, - // then n: Bilinear_Texture(tex, log_T, log_n) + // then n: Bilinear_Texture(tex, remap_log_T, remap_log_n) // cloudy cooling tables cut off at 10^9 K, use the CIE analytic fit above // this temp. if (log10(T) > 9.0) { lambda = 0.45 * log10(T) - 26.065; - } else if (log10(T) >= 1.0) { // don't cool below 10 K - lambda = Bilinear_Texture(coolTexObj, log_T, log_n); - H = Bilinear_Texture(heatTexObj, log_T, log_n); + } else if (log10(T) >= 1.0) { + lambda = Bilinear_Texture(coolTexObj, remap_log_T, remap_log_n); + const Real H = Bilinear_Texture(heatTexObj, remap_log_T, remap_log_n); + heating = pow(10, H); + } else { + // Do nothing below 10 K + return 0.0; } - // apply photoelectric heating under 10,000 K - if (log10(T) < 4.0) { - H_pe = n_av * 1.0e-26; - } + cooling = pow(10, lambda); + return n * n * (cooling - heating); +} + #endif // CLOUDY_COOL - // cooling rate per unit volume - if (log10(T) > 9.0) { - cool = n * n * pow(10, lambda); - } else if (log10(T) >= 4.0) { - cool = n * n * (pow(10, lambda) - pow(10, H)); +__device__ Real Photoelectric_Heating(Real n, Real T, Real n_av) +{ +// Photoelectric heating based on description given in Kim et al. 2015 +// n_av is mean density in the sim volume, cm^-3 +// Returns a positive value, expect sign conversion elsewhere for cooling + if (T < 1e4) { + return n * n_av * 1.0e-26; } else { - cool = n * (n * (pow(10, lambda) - pow(10, H)) - H_pe); + return 0.0; + } +} + +/* \fn __device__ Real TI_cool(Real n, Real T) + * \brief Estimated cooling / photoelectric heating function + based on description given in Kim et al. 2015. */ +__device__ Real TI_cool(Real n, Real T) +{ + Real lambda = 0.0; //cooling rate, erg s^-1 cm^3 + Real H = 0.0; //heating rate, erg s^-1 + Real cool = 0.0; //cooling per unit volume, erg /s / cm^3 + Real n_av = 100.0; //mean density in the sim volume + + // Below 10K only include photoelectric heating + if (log10(T) < 1.0) { + H = n_av*1.0e-26; + } + // Koyama & Inutsaka 2002 analytic fit + if (log10(T) >= 1.0 && log10(T) < 4.0) { + lambda = 2e-26 * (1e7 * exp(-1.148e5 / (T+1000.0)) + 1.4e-2 * sqrt(T) * exp(-92.0/T)); + H = n_av*1.0e-26; + } + // fit to cloudy CIE cooling function + if (log10(T) >= 4.0 && log10(T) < 5.9) { + lambda = powf(10.0, (-1.3 * (log10(T) - 5.25) * (log10(T) - 5.25) - 21.25)); + } + if (log10(T) >= 5.9 && log10(T) < 7.4) { + lambda = powf(10.0, (0.7 * (log10(T) - 7.1) * (log10(T) - 7.1) - 22.8)); + } + if (log10(T) >= 7.4) { + lambda = powf(10.0, (0.45*log10(T) - 26.065)); } - // printf("DEBUG Cloudy L350: %.17e\n",cool); + + // cooling rate per unit volume + cool = n*(n*lambda - H); + //if (cool > 1.0e-20) printf("n: %e T: %e lambda: %e H: %e\n", n, T, lambda, H); + return cool; } - #endif // CLOUDY_COOL + + #endif // COOLING_GPU #endif // CUDA From 22fe5af47c65404ade9452c07267c7b94f725a6b Mon Sep 17 00:00:00 2001 From: Alwin Date: Mon, 14 Aug 2023 15:36:44 -0400 Subject: [PATCH 09/10] add velocity and temperature ceilings --- src/grid/grid3D.cpp | 15 +++++ src/hydro/hydro_cuda.cu | 120 ++++++++++++++++++++++++++++++++++++++++ src/hydro/hydro_cuda.h | 14 ++++- 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/src/grid/grid3D.cpp b/src/grid/grid3D.cpp index 1777a4d81..5fd4ac7d8 100644 --- a/src/grid/grid3D.cpp +++ b/src/grid/grid3D.cpp @@ -499,6 +499,21 @@ Real Grid3D::Update_Grid(void) #endif #endif + #ifdef VELOCITY_CEILING + const Real V_ceiling_cholla = 0.005; // roughly 10000 km/s + Velocity_Ceiling(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, gama, V_ceiling_cholla); +#endif //VELOCITY_CEILING + + // Temperature Ceiling + #ifdef TEMPERATURE_CEILING + // 1e51 ergs / (m_p * (pc/cm)^3) = 45000 km/s + // sqrt(1e10 K * kB/ m_mp) = 9000 km/s + const Real T_ceiling_kelvin = 5e9;// 1e10; + Temperature_Ceiling(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, gama, T_ceiling_kelvin); +#endif //TEMPERATURE_CEILING + + + #ifdef AVERAGE_SLOW_CELLS // Set the min_delta_t for averaging a slow cell Real max_dti_slow; diff --git a/src/hydro/hydro_cuda.cu b/src/hydro/hydro_cuda.cu index eb50990de..da4c09552 100644 --- a/src/hydro/hydro_cuda.cu +++ b/src/hydro/hydro_cuda.cu @@ -594,6 +594,126 @@ Real Calc_dt_GPU(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n return dev_dti[0]; } + +#ifdef VELOCITY_CEILING + +__global__ void Velocity_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling, int* counter) +{ + const int id = threadIdx.x + blockIdx.x * blockDim.x; + const int n_cells = nx * ny * nz; + int xid, yid, zid; + cuda_utilities::compute3DIndices(id, nx, ny, xid, yid, zid); + const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && zid > n_ghost - 1 && zid < nz - n_ghost); + if (!real_cell) return; + + const Real d = dev_conserved[id]; + const Real max_momentum = d * V_ceiling; + + const Real d_inv = 1.0 / d; + /* + const Real vx = dev_conserved[1 * n_cells + id] * d_inv; + const Real vy = dev_conserved[2 * n_cells + id] * d_inv; + const Real vz = dev_conserved[3 * n_cells + id] * d_inv; + const Real E = dev_conserved[4 * n_cells + id]; + */ + for (int momentum_index = 1; momentum_index <= 3; momentum_index++) { + // Reduce momentum if velocity is too large + + const Real momentum = dev_conserved[momentum_index * n_cells + id]; + if (abs(momentum) > max_momentum) { + const Real new_momentum = max_momentum*momentum/abs(momentum); + const Real diff_energy = 0.5 * d_inv * ((momentum * momentum) - (max_momentum*max_momentum)); + // Write in the new momentum + dev_conserved[momentum_index * n_cells + id] = new_momentum; + // Thermalize the energy + #ifdef DE + dev_conserved[(n_fields - 1) * n_cells + id] += diff_energy; + #endif //DE + atomicAdd(counter, 1); + } + } + + +} + +void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling) +{ + + int n_cells = nx * ny * nz; + int ngrid = (n_cells + TPB - 1) / TPB; + dim3 dim1dGrid(ngrid, 1, 1); + dim3 dim1dBlock(TPB, 1, 1); + + cuda_utilities::DeviceVector counter(1, true); + int* dev_counter = counter.data(); + + if (nx > 1 && ny > 1 && nz > 1) { // 3D + hipLaunchKernelGGL(Velocity_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, n_fields, gamma, V_ceiling, dev_counter); + } + int host_counter = counter[0]; + if (host_counter > 0) { + printf("HYDRO WARNING: Velocity Ceiling applied to num_cells: %d \n", host_counter); + } +} + + +#endif + + +#ifdef TEMPERATURE_CEILING + +__global__ void Temperature_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling, int* counter) +{ + const int id = threadIdx.x + blockIdx.x * blockDim.x; + const int n_cells = nx * ny * nz; + int xid, yid, zid; + cuda_utilities::compute3DIndices(id, nx, ny, xid, yid, zid); + const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && zid > n_ghost - 1 && zid < nz - n_ghost); + if (!real_cell) return; + + const Real d = dev_conserved[id]; + const Real d_inv = 1.0 / d; + const Real vx = dev_conserved[1 * n_cells + id] * d_inv; + const Real vy = dev_conserved[2 * n_cells + id] * d_inv; + const Real vz = dev_conserved[3 * n_cells + id] * d_inv; + const Real E = dev_conserved[4 * n_cells + id]; + + const Real temperature_Kelvin = (gamma - 1) * (E - 0.5 * (vx*vx + vy*vy + vz*vz) * d) * ENERGY_UNIT / (d * DENSITY_UNIT / 0.6 / MP) / KB; + + if (temperature_Kelvin <= T_ceiling) return; + + const Real factor = T_ceiling / temperature_Kelvin; + + dev_conserved[4 * n_cells + id] *= factor; + #ifdef DE + dev_conserved[(n_fields - 1) * n_cells + id] *= factor; + #endif //DE + atomicAdd(counter, 1); +} + +void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling) +{ + + int n_cells = nx * ny * nz; + int ngrid = (n_cells + TPB - 1) / TPB; + dim3 dim1dGrid(ngrid, 1, 1); + dim3 dim1dBlock(TPB, 1, 1); + + cuda_utilities::DeviceVector counter(1, true); + int* dev_counter = counter.data(); + + if (nx > 1 && ny > 1 && nz > 1) { // 3D + hipLaunchKernelGGL(Temperature_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, n_fields, gamma, T_ceiling, dev_counter); + } + int host_counter = counter[0]; + if (host_counter > 0) { + printf("HYDRO WARNING: Temperature Ceiling applied to num_cells: %d \n", host_counter); + } +} + + +#endif //TEMPERATURE_CEILING + #ifdef AVERAGE_SLOW_CELLS void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, Real dy, diff --git a/src/hydro/hydro_cuda.h b/src/hydro/hydro_cuda.h index 9b4403be0..c071aa33f 100644 --- a/src/hydro/hydro_cuda.h +++ b/src/hydro/hydro_cuda.h @@ -75,15 +75,23 @@ __global__ void Sync_Energies_2D(Real *dev_conserved, int nx, int ny, int n_ghos __global__ void Sync_Energies_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, Real gamma, int n_fields); +#ifdef VELOCITY_CEILING +void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling); +#endif // VELOCITY CEILING + +#ifdef TEMPERATURE_CEILING +void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling); +#endif // TEMPERATURE CEILING + #ifdef AVERAGE_SLOW_CELLS void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, int nx_offset, - int ny_offset, int nz_offset); + int ny_offset, int nz_offset); __global__ void Average_Slow_Cells_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, - Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, - Real zbound, int nx_offset, int ny_offset, int nz_offset); + Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, + int nx_offset, int ny_offset, int nz_offset); #endif #ifdef TEMPERATURE_FLOOR From 2d331a81f167a7dbe36f681c6fb27ae33a25edf2 Mon Sep 17 00:00:00 2001 From: Alwin Date: Mon, 14 Aug 2023 16:28:13 -0400 Subject: [PATCH 10/10] formatting --- src/cooling/cooling_cuda.cu | 82 ++++++++++++++++++------------------ src/grid/grid3D.cpp | 10 ++--- src/hydro/hydro_cuda.cu | 79 +++++++++++++++++----------------- src/hydro/hydro_cuda.h | 20 +++++---- src/utils/debug_utilities.cu | 31 +++++++------- 5 files changed, 110 insertions(+), 112 deletions(-) diff --git a/src/cooling/cooling_cuda.cu b/src/cooling/cooling_cuda.cu index 42e5fe810..b4bb60782 100644 --- a/src/cooling/cooling_cuda.cu +++ b/src/cooling/cooling_cuda.cu @@ -88,16 +88,15 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int d = dev_conserved[id]; E = dev_conserved[4 * n_cells + id]; - // don't apply cooling if this thread crashed if (E < 0.0 || E != E) { - /* Alwin: this is a leftover debug printf from when we were checking cooling for small temperature outputs. Delete at leisure. - kernel_printf("WARNING: bad energy in cooling_cuda E=%e [xid, yid, zid] = [%d, %d, %d] \n", E, xid, yid, zid); + /* Alwin: this is a leftover debug printf from when we were checking cooling for small temperature outputs. Delete + at leisure. kernel_printf("WARNING: bad energy in cooling_cuda E=%e [xid, yid, zid] = [%d, %d, %d] \n", E, xid, + yid, zid); */ return; } - // #ifndef DE vx = dev_conserved[1 * n_cells + id] / d; vy = dev_conserved[2 * n_cells + id] / d; @@ -113,7 +112,7 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int // calculate the number density of the gas (in cgs) n = d * DENSITY_UNIT / (mu * MP); - // calculate the temperature of the gas + // calculate the temperature of the gas #ifdef DE const Real T_init = ge * (gamma - 1.0) * PRESSURE_UNIT / (n * KB); @@ -138,21 +137,23 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int #endif del_T = cool * dt * TIME_UNIT * (gamma - 1.0) / (n * KB); if (fabs(del_T) > 0.01 * T) { - // Evolve T and dt so T changes by 1% - dt -= dt * 0.01 * T / fabs(del_T); - T -= del_T * 0.01 * T / fabs(del_T); + // Evolve T and dt so T changes by 1% + dt -= dt * 0.01 * T / fabs(del_T); + T -= del_T * 0.01 * T / fabs(del_T); } else { - T -= del_T; - break; + T -= del_T; + break; } } const Real T_final = T; - /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete at leisure. + /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete + at leisure. // If the temperature has changed by a lot, whine vehemently if ((T_init / T_final > 1e3) || (T_final / T_init > 1e3)) { - //kernel_printf("Cooling Cuda Large Temperature Change whine: | T_init (K): %e | T_final (K): %e | dt (cholla): %e | n (cgs): %e \n", T_init, T_final, dt_init, n); + //kernel_printf("Cooling Cuda Large Temperature Change whine: | T_init (K): %e | T_final (K): %e | dt (cholla): %e + | n (cgs): %e \n", T_init, T_final, dt_init, n); } */ @@ -168,14 +169,12 @@ __global__ void cooling_kernel(Real *dev_conserved, int nx, int ny, int nz, int #ifdef DE dev_conserved[(n_fields - 1) * n_cells + id] = ge; #endif - - - /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete at leisure. - if (T_final < 1e2) { - kernel_printf("Cooling Cuda Small Temperature Whine: | T_init (K): %e | T_final (K): %e | dt (cholla): %e | n (cgs): %e \n", T_init, T_final, dt_init, n); + + /* Alwin: this is a leftover debug function from when we were checking cooling for small temperature outputs. Delete + at leisure. if (T_final < 1e2) { kernel_printf("Cooling Cuda Small Temperature Whine: | T_init (K): %e | T_final + (K): %e | dt (cholla): %e | n (cgs): %e \n", T_init, T_final, dt_init, n); } */ - } } @@ -338,11 +337,12 @@ __device__ Real CIE_cool(Real n, Real T) tables at z = 0 with solar metallicity and an HM05 UV background. */ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cudaTextureObject_t heatTexObj) { - Real lambda = 0.0; // log cooling rate, erg s^-1 cm^3 - Real cooling = 0.0; // cooling per unit volume, erg /s / cm^3 - Real heating = 0.0; // cooling per unit volume, erg /s / cm^3 + Real lambda = 0.0; // log cooling rate, erg s^-1 cm^3 + Real cooling = 0.0; // cooling per unit volume, erg /s / cm^3 + Real heating = 0.0; // cooling per unit volume, erg /s / cm^3 - // To keep texture code simple, we use floats (which have built-in support) as opposed to doubles (which would require casting) + // To keep texture code simple, we use floats (which have built-in support) as opposed to doubles (which would require + // casting) float log_n, log_T; log_n = log10(n); log_T = log10(T); @@ -362,9 +362,9 @@ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cuda if (log10(T) > 9.0) { lambda = 0.45 * log10(T) - 26.065; } else if (log10(T) >= 1.0) { - lambda = Bilinear_Texture(coolTexObj, remap_log_T, remap_log_n); + lambda = Bilinear_Texture(coolTexObj, remap_log_T, remap_log_n); const Real H = Bilinear_Texture(heatTexObj, remap_log_T, remap_log_n); - heating = pow(10, H); + heating = pow(10, H); } else { // Do nothing below 10 K return 0.0; @@ -377,9 +377,9 @@ __device__ Real Cloudy_cool(Real n, Real T, cudaTextureObject_t coolTexObj, cuda __device__ Real Photoelectric_Heating(Real n, Real T, Real n_av) { -// Photoelectric heating based on description given in Kim et al. 2015 -// n_av is mean density in the sim volume, cm^-3 -// Returns a positive value, expect sign conversion elsewhere for cooling + // Photoelectric heating based on description given in Kim et al. 2015 + // n_av is mean density in the sim volume, cm^-3 + // Returns a positive value, expect sign conversion elsewhere for cooling if (T < 1e4) { return n * n_av * 1.0e-26; } else { @@ -392,39 +392,37 @@ __device__ Real Photoelectric_Heating(Real n, Real T, Real n_av) based on description given in Kim et al. 2015. */ __device__ Real TI_cool(Real n, Real T) { - Real lambda = 0.0; //cooling rate, erg s^-1 cm^3 - Real H = 0.0; //heating rate, erg s^-1 - Real cool = 0.0; //cooling per unit volume, erg /s / cm^3 - Real n_av = 100.0; //mean density in the sim volume + Real lambda = 0.0; // cooling rate, erg s^-1 cm^3 + Real H = 0.0; // heating rate, erg s^-1 + Real cool = 0.0; // cooling per unit volume, erg /s / cm^3 + Real n_av = 100.0; // mean density in the sim volume // Below 10K only include photoelectric heating if (log10(T) < 1.0) { - H = n_av*1.0e-26; + H = n_av * 1.0e-26; } // Koyama & Inutsaka 2002 analytic fit if (log10(T) >= 1.0 && log10(T) < 4.0) { - lambda = 2e-26 * (1e7 * exp(-1.148e5 / (T+1000.0)) + 1.4e-2 * sqrt(T) * exp(-92.0/T)); - H = n_av*1.0e-26; + lambda = 2e-26 * (1e7 * exp(-1.148e5 / (T + 1000.0)) + 1.4e-2 * sqrt(T) * exp(-92.0 / T)); + H = n_av * 1.0e-26; } - // fit to cloudy CIE cooling function + // fit to cloudy CIE cooling function if (log10(T) >= 4.0 && log10(T) < 5.9) { lambda = powf(10.0, (-1.3 * (log10(T) - 5.25) * (log10(T) - 5.25) - 21.25)); } if (log10(T) >= 5.9 && log10(T) < 7.4) { lambda = powf(10.0, (0.7 * (log10(T) - 7.1) * (log10(T) - 7.1) - 22.8)); } - if (log10(T) >= 7.4) { - lambda = powf(10.0, (0.45*log10(T) - 26.065)); + if (log10(T) >= 7.4) { + lambda = powf(10.0, (0.45 * log10(T) - 26.065)); } - + // cooling rate per unit volume - cool = n*(n*lambda - H); - //if (cool > 1.0e-20) printf("n: %e T: %e lambda: %e H: %e\n", n, T, lambda, H); + cool = n * (n * lambda - H); + // if (cool > 1.0e-20) printf("n: %e T: %e lambda: %e H: %e\n", n, T, lambda, H); return cool; } - - #endif // COOLING_GPU #endif // CUDA diff --git a/src/grid/grid3D.cpp b/src/grid/grid3D.cpp index 5fd4ac7d8..7a1fad7a8 100644 --- a/src/grid/grid3D.cpp +++ b/src/grid/grid3D.cpp @@ -500,19 +500,17 @@ Real Grid3D::Update_Grid(void) #endif #ifdef VELOCITY_CEILING - const Real V_ceiling_cholla = 0.005; // roughly 10000 km/s + const Real V_ceiling_cholla = 0.005; // roughly 10000 km/s Velocity_Ceiling(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, gama, V_ceiling_cholla); -#endif //VELOCITY_CEILING + #endif // VELOCITY_CEILING // Temperature Ceiling #ifdef TEMPERATURE_CEILING // 1e51 ergs / (m_p * (pc/cm)^3) = 45000 km/s // sqrt(1e10 K * kB/ m_mp) = 9000 km/s - const Real T_ceiling_kelvin = 5e9;// 1e10; + const Real T_ceiling_kelvin = 5e9; // 1e10; Temperature_Ceiling(C.device, H.nx, H.ny, H.nz, H.n_ghost, H.n_fields, gama, T_ceiling_kelvin); -#endif //TEMPERATURE_CEILING - - + #endif // TEMPERATURE_CEILING #ifdef AVERAGE_SLOW_CELLS // Set the min_delta_t for averaging a slow cell diff --git a/src/hydro/hydro_cuda.cu b/src/hydro/hydro_cuda.cu index da4c09552..4e8e49328 100644 --- a/src/hydro/hydro_cuda.cu +++ b/src/hydro/hydro_cuda.cu @@ -594,19 +594,20 @@ Real Calc_dt_GPU(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n return dev_dti[0]; } + #ifdef VELOCITY_CEILING -#ifdef VELOCITY_CEILING - -__global__ void Velocity_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling, int* counter) +__global__ void Velocity_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, + Real gamma, Real V_ceiling, int *counter) { - const int id = threadIdx.x + blockIdx.x * blockDim.x; - const int n_cells = nx * ny * nz; + const int id = threadIdx.x + blockIdx.x * blockDim.x; + const int n_cells = nx * ny * nz; int xid, yid, zid; cuda_utilities::compute3DIndices(id, nx, ny, xid, yid, zid); - const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && zid > n_ghost - 1 && zid < nz - n_ghost); + const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && + zid > n_ghost - 1 && zid < nz - n_ghost); if (!real_cell) return; - const Real d = dev_conserved[id]; + const Real d = dev_conserved[id]; const Real max_momentum = d * V_ceiling; const Real d_inv = 1.0 / d; @@ -621,54 +622,53 @@ __global__ void Velocity_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int const Real momentum = dev_conserved[momentum_index * n_cells + id]; if (abs(momentum) > max_momentum) { - const Real new_momentum = max_momentum*momentum/abs(momentum); - const Real diff_energy = 0.5 * d_inv * ((momentum * momentum) - (max_momentum*max_momentum)); + const Real new_momentum = max_momentum * momentum / abs(momentum); + const Real diff_energy = 0.5 * d_inv * ((momentum * momentum) - (max_momentum * max_momentum)); // Write in the new momentum dev_conserved[momentum_index * n_cells + id] = new_momentum; - // Thermalize the energy - #ifdef DE + // Thermalize the energy + #ifdef DE dev_conserved[(n_fields - 1) * n_cells + id] += diff_energy; - #endif //DE + #endif // DE atomicAdd(counter, 1); } } - - } -void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling) +void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, + Real V_ceiling) { - int n_cells = nx * ny * nz; - int ngrid = (n_cells + TPB - 1) / TPB; + int ngrid = (n_cells + TPB - 1) / TPB; dim3 dim1dGrid(ngrid, 1, 1); dim3 dim1dBlock(TPB, 1, 1); cuda_utilities::DeviceVector counter(1, true); - int* dev_counter = counter.data(); + int *dev_counter = counter.data(); if (nx > 1 && ny > 1 && nz > 1) { // 3D - hipLaunchKernelGGL(Velocity_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, n_fields, gamma, V_ceiling, dev_counter); + hipLaunchKernelGGL(Velocity_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, + n_fields, gamma, V_ceiling, dev_counter); } int host_counter = counter[0]; if (host_counter > 0) { - printf("HYDRO WARNING: Velocity Ceiling applied to num_cells: %d \n", host_counter); + printf("HYDRO WARNING: Velocity Ceiling applied to num_cells: %d \n", host_counter); } } + #endif -#endif - - -#ifdef TEMPERATURE_CEILING + #ifdef TEMPERATURE_CEILING -__global__ void Temperature_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling, int* counter) +__global__ void Temperature_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, + Real gamma, Real T_ceiling, int *counter) { - const int id = threadIdx.x + blockIdx.x * blockDim.x; - const int n_cells = nx * ny * nz; + const int id = threadIdx.x + blockIdx.x * blockDim.x; + const int n_cells = nx * ny * nz; int xid, yid, zid; cuda_utilities::compute3DIndices(id, nx, ny, xid, yid, zid); - const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && zid > n_ghost - 1 && zid < nz - n_ghost); + const bool real_cell = (xid > n_ghost - 1 && xid < nx - n_ghost && yid > n_ghost - 1 && yid < ny - n_ghost && + zid > n_ghost - 1 && zid < nz - n_ghost); if (!real_cell) return; const Real d = dev_conserved[id]; @@ -678,41 +678,42 @@ __global__ void Temperature_Ceiling_Kernel(Real *dev_conserved, int nx, int ny, const Real vz = dev_conserved[3 * n_cells + id] * d_inv; const Real E = dev_conserved[4 * n_cells + id]; - const Real temperature_Kelvin = (gamma - 1) * (E - 0.5 * (vx*vx + vy*vy + vz*vz) * d) * ENERGY_UNIT / (d * DENSITY_UNIT / 0.6 / MP) / KB; + const Real temperature_Kelvin = + (gamma - 1) * (E - 0.5 * (vx * vx + vy * vy + vz * vz) * d) * ENERGY_UNIT / (d * DENSITY_UNIT / 0.6 / MP) / KB; if (temperature_Kelvin <= T_ceiling) return; const Real factor = T_ceiling / temperature_Kelvin; dev_conserved[4 * n_cells + id] *= factor; - #ifdef DE + #ifdef DE dev_conserved[(n_fields - 1) * n_cells + id] *= factor; - #endif //DE + #endif // DE atomicAdd(counter, 1); } -void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling) +void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, + Real T_ceiling) { - int n_cells = nx * ny * nz; - int ngrid = (n_cells + TPB - 1) / TPB; + int ngrid = (n_cells + TPB - 1) / TPB; dim3 dim1dGrid(ngrid, 1, 1); dim3 dim1dBlock(TPB, 1, 1); cuda_utilities::DeviceVector counter(1, true); - int* dev_counter = counter.data(); + int *dev_counter = counter.data(); if (nx > 1 && ny > 1 && nz > 1) { // 3D - hipLaunchKernelGGL(Temperature_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, n_fields, gamma, T_ceiling, dev_counter); + hipLaunchKernelGGL(Temperature_Ceiling_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, nx, ny, nz, n_ghost, + n_fields, gamma, T_ceiling, dev_counter); } int host_counter = counter[0]; if (host_counter > 0) { - printf("HYDRO WARNING: Temperature Ceiling applied to num_cells: %d \n", host_counter); + printf("HYDRO WARNING: Temperature Ceiling applied to num_cells: %d \n", host_counter); } } - -#endif //TEMPERATURE_CEILING + #endif // TEMPERATURE_CEILING #ifdef AVERAGE_SLOW_CELLS diff --git a/src/hydro/hydro_cuda.h b/src/hydro/hydro_cuda.h index c071aa33f..d6ac09bee 100644 --- a/src/hydro/hydro_cuda.h +++ b/src/hydro/hydro_cuda.h @@ -75,23 +75,25 @@ __global__ void Sync_Energies_2D(Real *dev_conserved, int nx, int ny, int n_ghos __global__ void Sync_Energies_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, Real gamma, int n_fields); -#ifdef VELOCITY_CEILING -void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real V_ceiling); -#endif // VELOCITY CEILING + #ifdef VELOCITY_CEILING +void Velocity_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, + Real V_ceiling); + #endif // VELOCITY CEILING -#ifdef TEMPERATURE_CEILING -void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, Real T_ceiling); -#endif // TEMPERATURE CEILING + #ifdef TEMPERATURE_CEILING +void Temperature_Ceiling(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real gamma, + Real T_ceiling); + #endif // TEMPERATURE CEILING #ifdef AVERAGE_SLOW_CELLS void Average_Slow_Cells(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, int nx_offset, - int ny_offset, int nz_offset); + int ny_offset, int nz_offset); __global__ void Average_Slow_Cells_3D(Real *dev_conserved, int nx, int ny, int nz, int n_ghost, int n_fields, Real dx, - Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, Real zbound, - int nx_offset, int ny_offset, int nz_offset); + Real dy, Real dz, Real gamma, Real max_dti_slow, Real xbound, Real ybound, + Real zbound, int nx_offset, int ny_offset, int nz_offset); #endif #ifdef TEMPERATURE_FLOOR diff --git a/src/utils/debug_utilities.cu b/src/utils/debug_utilities.cu index 5c90fe19f..604ad7c8b 100644 --- a/src/utils/debug_utilities.cu +++ b/src/utils/debug_utilities.cu @@ -5,15 +5,16 @@ #include "../io/io.h" // provides chprintf #include "../utils/error_handling.h" // provides chexit -__global__ void Check_For_Extreme_Temperature_Kernel(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, Real upper_limit, int marker, bool* out_bool) +__global__ void Check_For_Extreme_Temperature_Kernel(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, + Real upper_limit, int marker, bool* out_bool) { // Calculate temperature int id = threadIdx.x + blockIdx.x * blockDim.x; if (id >= n_cells) { return; } - const Real d = dev_conserved[id]; - const Real E = dev_conserved[id + n_cells * grid_enum::Energy]; + const Real d = dev_conserved[id]; + const Real E = dev_conserved[id + n_cells * grid_enum::Energy]; const Real px = dev_conserved[id + n_cells * grid_enum::momentum_x]; const Real py = dev_conserved[id + n_cells * grid_enum::momentum_y]; const Real pz = dev_conserved[id + n_cells * grid_enum::momentum_z]; @@ -25,24 +26,25 @@ __global__ void Check_For_Extreme_Temperature_Kernel(Real* dev_conserved, int n_ const Real vz = dev_conserved[3 * n_cells + id] / d; const Real E_kinetic = 0.5 * d * (vx * vx + vy * vy + vz * vz); */ - const Real E_thermal = E - E_kinetic; - const Real p = (E_thermal) * (gamma - 1.0); + const Real p = (E_thermal) * (gamma - 1.0); - const Real mu = 0.6; - const Real n = d * DENSITY_UNIT / (mu * MP); + const Real mu = 0.6; + const Real n = d * DENSITY_UNIT / (mu * MP); const Real T_kelvin = p * PRESSURE_UNIT / (n * KB); - if (T_kelvin <= lower_limit || T_kelvin >= upper_limit) { out_bool[0] = true; - kernel_printf("Check_For_Extreme_Temperature_Kernel found Value: %g E: %g E_thermal: %g E_kinetic: %g Marker: %d Thread: %d\n", T_kelvin, E, E_thermal, E_kinetic, marker, id); + kernel_printf( + "Check_For_Extreme_Temperature_Kernel found Value: %g E: %g E_thermal: %g E_kinetic: %g Marker: %d Thread: " + "%d\n", + T_kelvin, E, E_thermal, E_kinetic, marker, id); } } - -void Check_For_Extreme_Temperature(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, Real upper_limit, int marker) +void Check_For_Extreme_Temperature(Real* dev_conserved, int n_cells, Real gamma, Real lower_limit, Real upper_limit, + int marker) { bool host_out_bool[1] = {false}; bool* out_bool; @@ -52,7 +54,8 @@ void Check_For_Extreme_Temperature(Real* dev_conserved, int n_cells, Real gamma, dim3 dim1dGrid(ngrid, 1, 1); dim3 dim1dBlock(TPB, 1, 1); - hipLaunchKernelGGL(Check_For_Extreme_Temperature_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, n_cells, gamma, lower_limit, upper_limit, marker, out_bool); + hipLaunchKernelGGL(Check_For_Extreme_Temperature_Kernel, dim1dGrid, dim1dBlock, 0, 0, dev_conserved, n_cells, gamma, + lower_limit, upper_limit, marker, out_bool); cudaMemcpy(host_out_bool, out_bool, sizeof(bool), cudaMemcpyDeviceToHost); cudaFree(out_bool); @@ -62,10 +65,6 @@ void Check_For_Extreme_Temperature(Real* dev_conserved, int n_cells, Real gamma, } } - - - - __global__ void Dump_Values_Kernel(Real* device_array, int array_size, int marker) { int tid = threadIdx.x + blockIdx.x * blockDim.x;