Added support for MPI

src/Peridynamics.jl

@@ -9,7 +9,7 @@ end
 export BBMaterial, CKIMaterial, NOSBMaterial, OSBMaterial, Body, point_set!,
        failure_permit!, material!, velocity_bc!, velocity_ic!, forcedensity_bc!, precrack!,
        VelocityVerlet, MultibodySetup, contact!, Job, read_vtk, uniform_box, submit,
-       process_each_export
+       process_each_export, mpi_isroot
 
 const MPI_INITIALIZED = Ref(false)
 const MPI_RANK = Ref(-1)
@@ -25,6 +25,8 @@ const QUIET = Ref(false)
     QUIET[] = b
     return nothing
 end
+@inline mpi_chunk_id() = mpi_rank() + 1
+@inline mpi_isroot() = mpi_rank() == 0
 
 const TO = TimerOutput()
 
@@ -113,6 +115,7 @@ include("core/job.jl")
 include("core/submit.jl")
 include("core/halo_exchange.jl")
 include("core/threads_data_handler.jl")
+include("core/mpi_data_handler.jl")
 
 include("time_solvers/solve_velocity_verlet.jl")
 

src/auxiliary/io.jl

@@ -76,7 +76,6 @@ function check_storage_fields(::Type{S}, fields::Vector{Symbol}) where {S}
     return nothing
 end
 
-
 function export_results(dh::AbstractDataHandler, options::ExportOptions, chunk_id::Int,
                         timestep::Int, time::Float64)
     options.exportflag || return nothing
@@ -96,7 +95,8 @@ end
 
 function _export_results(b::AbstractBodyChunk, chunk_id::Int, n_chunks::Int,
                          options::ExportOptions, n::Int, t::Float64)
-    filename = @sprintf("timestep_%05d", n)
+    # filename = @sprintf("timestep_%05d", n)
+    filename = joinpath(options.vtk, @sprintf("timestep_%05d", n))
     position = get_loc_position(b)
     pvtk_grid(filename, position, b.cells; part=chunk_id, nparts=n_chunks) do vtk
         for field in options.fields

src/core/halo_exchange.jl

@@ -1,64 +1,83 @@
+function exchange!(dest::Matrix{T}, src::Matrix{T}, dest_idxs::Vector{Int},
+                   src_idxs::Vector{Int}) where {T}
+    for i in eachindex(dest_idxs)
+        for d in axes(dest, 1)
+            @inbounds dest[d, dest_idxs[i]] = src[d, src_idxs[i]]
+        end
+    end
+    return nothing
+end
 
-struct HaloExchange
-    src_chunk_id::Int
-    dest_chunk_id::Int
-    src_idxs::Vector{Int}
-    dest_idxs::Vector{Int}
+function exchange_from_buf!(dest::T, buf::T, dest_idxs::Vector{Int}) where {T<:Matrix}
+    for i in eachindex(dest_idxs)
+        for d in axes(dest, 1)
+            dest[d, dest_idxs[i]] = buf[d, i]
+        end
+    end
+    return nothing
 end
 
-function find_halo_exchanges(chunks::Vector{B}) where {B<:AbstractBodyChunk}
-    has_read = has_read_halos(chunks)
-    has_write = has_write_halos(chunks)
-    read_halo_exs = Vector{Vector{HaloExchange}}(undef, length(chunks))
-    write_halo_exs = Vector{Vector{HaloExchange}}(undef, length(chunks))
-    @threads :static for chunk_id in eachindex(chunks)
-        read_exs, write_exs = find_exs(chunks, chunk_id, has_read, has_write)
-        read_halo_exs[chunk_id] = read_exs
-        write_halo_exs[chunk_id] = write_exs
+function exchange_to_buf!(buf::T, src::T, src_idxs::Vector{Int}) where {T<:Matrix}
+    for i in eachindex(src_idxs)
+        for d in axes(buf, 1)
+            buf[d, i] = src[d, src_idxs[i]]
+        end
     end
-    reorder_write_exs!(write_halo_exs)
-    return read_halo_exs, write_halo_exs
+    return nothing
 end
 
-function has_read_halos(chunk::AbstractBodyChunk)
-    hrfs = get_halo_read_fields(chunk.store)
-    isempty(hrfs) && return false
-    return true
+function exchange!(dest::Vector{T}, src::Vector{T}, dest_idxs::Vector{Int},
+                   src_idxs::Vector{Int}) where {T}
+    for i in eachindex(dest_idxs)
+        @inbounds dest[dest_idxs[i]] = src[src_idxs[i]]
+    end
+    return nothing
 end
 
-function has_read_halos(chunks::Vector{B}) where {B<:AbstractBodyChunk}
-    return has_read_halos(first(chunks))
+function exchange_from_buf!(dest::T, buf::T, dest_idxs::Vector{Int}) where {T<:Vector}
+    for i in eachindex(dest_idxs)
+        dest[dest_idxs[i]] = buf[i]
+    end
+    return nothing
 end
 
-function has_write_halos(chunk::AbstractBodyChunk)
-    hwfs = get_halo_write_fields(chunk.store)
-    isempty(hwfs) && return false
-    return true
+function exchange_to_buf!(buf::T, src::T, src_idxs::Vector{Int}) where {T<:Vector}
+    for i in eachindex(src_idxs)
+        buf[i] = src[src_idxs[i]]
+    end
+    return nothing
 end
 
-function has_write_halos(chunks::Vector{B}) where {B<:AbstractBodyChunk}
-    return has_write_halos(first(chunks))
+function exchange_add!(dest::Matrix{T}, src::Matrix{T}, dest_idxs::Vector{Int},
+                       src_idxs::Vector{Int}) where {T<:Number}
+    for i in eachindex(dest_idxs)
+        for d in axes(dest, 1)
+            @inbounds dest[d, dest_idxs[i]] += src[d, src_idxs[i]]
+        end
+    end
+    return nothing
 end
 
-function find_exs(chunks::Vector{B}, chunk_id::Int, hasread::Bool,
-                  haswrite::Bool) where {B<:AbstractBodyChunk}
-    readexs = Vector{HaloExchange}()
-    writeexs = Vector{HaloExchange}()
-    chunk = chunks[chunk_id]
-    for (halo_chunk_id, idxs) in chunk.ch.halo_by_src
-        halo_chunk = chunks[halo_chunk_id]
-        halo_idxs = chunk.ch.point_ids[idxs]
-        localize!(halo_idxs, halo_chunk.ch.localizer)
-        hasread && push!(readexs, HaloExchange(halo_chunk_id, chunk_id, halo_idxs, idxs))
-        haswrite && push!(writeexs, HaloExchange(chunk_id, halo_chunk_id, idxs, halo_idxs))
+function exchange_from_buf_add!(dest::T, buf::T, dest_idxs::Vector{Int}) where {T<:Matrix}
+    for i in eachindex(dest_idxs)
+        for d in axes(dest, 1)
+            dest[d, dest_idxs[i]] += buf[d, i]
+        end
     end
-    return readexs, writeexs
+    return nothing
+end
+
+function exchange_add!(dest::Vector{T}, src::Vector{T}, dest_idxs::Vector{Int},
+                       src_idxs::Vector{Int}) where {T<:Number}
+    for i in eachindex(dest_idxs)
+        @inbounds dest[dest_idxs[i]] += src[src_idxs[i]]
+    end
+    return nothing
 end
 
-function reorder_write_exs!(write_halo_exs::Vector{Vector{HaloExchange}})
-    all_write_exs = reduce(vcat, write_halo_exs)
-    @threads :static for chunk_id in eachindex(write_halo_exs)
-        write_halo_exs[chunk_id] = filter(x -> x.dest_chunk_id == chunk_id, all_write_exs)
+function exchange_from_buf_add!(dest::T, buf::T, dest_idxs::Vector{Int}) where {T<:Vector}
+    for i in eachindex(dest_idxs)
+        dest[dest_idxs[i]] += buf[i]
     end
     return nothing
 end

src/core/job.jl

@@ -15,6 +15,6 @@ end
 function Job(spatial_setup::S, time_solver::T; kwargs...) where {S,T}
     o = Dict{Symbol,Any}(kwargs)
     check_kwargs(o, JOB_KWARGS)
-    options = get_export_options(storage_type(spatial_setup.mat, time_solver), o)
+    options = get_export_options(storage_type(spatial_setup, time_solver), o)
     return Job(spatial_setup, time_solver, options)
 end