From f4c6e489c7b3ca6151f515415e02c6f3565ed222 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 30 May 2024 17:29:31 +0200 Subject: [PATCH 1/8] allreduce --- allreduce-perf/common/launch_wrapper | 94 ++++++++++ allreduce-perf/mpi-cpp/env/README | 14 ++ .../mpi-cpp/env/recipe/compilers.yaml | 5 + allreduce-perf/mpi-cpp/env/recipe/config.yaml | 7 + .../mpi-cpp/env/recipe/environments.yaml | 26 +++ allreduce-perf/mpi-cpp/run/run.sh | 81 +++++++++ allreduce-perf/mpi-cpp/run/run_all.sh | 17 ++ allreduce-perf/mpi-cpp/src/CMakeLists.txt | 50 ++++++ allreduce-perf/mpi-cpp/src/allreduce.cu | 24 +++ allreduce-perf/mpi-cpp/src/allreduce.hpp | 87 +++++++++ allreduce-perf/mpi-cpp/src/bench.cpp | 166 ++++++++++++++++++ allreduce-perf/mpi-cpp/src/memory.cu | 25 +++ allreduce-perf/mpi-cpp/src/memory.hpp | 131 ++++++++++++++ allreduce-perf/nccl-pytorch/env/Dockerfile | 13 ++ .../nccl-pytorch/env/podman_build.sh | 4 + allreduce-perf/nccl-pytorch/run/ngc.toml | 16 ++ allreduce-perf/nccl-pytorch/run/run.sh | 62 +++++++ allreduce-perf/nccl-pytorch/run/run_all.sh | 10 ++ allreduce-perf/nccl-pytorch/src/all_reduce.py | 150 ++++++++++++++++ allreduce-perf/nccl-tests-cpp/env/import.sh | 3 + .../nccl-tests-cpp/run/nccl-test.toml | 22 +++ allreduce-perf/nccl-tests-cpp/run/run.sh | 52 ++++++ allreduce-perf/nccl-tests-cpp/run/run_all.sh | 17 ++ 23 files changed, 1076 insertions(+) create mode 100755 allreduce-perf/common/launch_wrapper create mode 100644 allreduce-perf/mpi-cpp/env/README create mode 100644 allreduce-perf/mpi-cpp/env/recipe/compilers.yaml create mode 100644 allreduce-perf/mpi-cpp/env/recipe/config.yaml create mode 100644 allreduce-perf/mpi-cpp/env/recipe/environments.yaml create mode 100755 allreduce-perf/mpi-cpp/run/run.sh create mode 100755 allreduce-perf/mpi-cpp/run/run_all.sh create mode 100644 allreduce-perf/mpi-cpp/src/CMakeLists.txt create mode 100644 allreduce-perf/mpi-cpp/src/allreduce.cu create mode 100644 allreduce-perf/mpi-cpp/src/allreduce.hpp create mode 100644 allreduce-perf/mpi-cpp/src/bench.cpp create mode 100644 allreduce-perf/mpi-cpp/src/memory.cu create mode 100644 allreduce-perf/mpi-cpp/src/memory.hpp create mode 100644 allreduce-perf/nccl-pytorch/env/Dockerfile create mode 100755 allreduce-perf/nccl-pytorch/env/podman_build.sh create mode 100644 allreduce-perf/nccl-pytorch/run/ngc.toml create mode 100755 allreduce-perf/nccl-pytorch/run/run.sh create mode 100755 allreduce-perf/nccl-pytorch/run/run_all.sh create mode 100644 allreduce-perf/nccl-pytorch/src/all_reduce.py create mode 100755 allreduce-perf/nccl-tests-cpp/env/import.sh create mode 100644 allreduce-perf/nccl-tests-cpp/run/nccl-test.toml create mode 100755 allreduce-perf/nccl-tests-cpp/run/run.sh create mode 100755 allreduce-perf/nccl-tests-cpp/run/run_all.sh diff --git a/allreduce-perf/common/launch_wrapper b/allreduce-perf/common/launch_wrapper new file mode 100755 index 0000000..ff84c40 --- /dev/null +++ b/allreduce-perf/common/launch_wrapper @@ -0,0 +1,94 @@ +#!/bin/bash + +# get the current process ID +pid=$$ + +# retrieve the CPU affinity mask in hexadecimal format +affinity_mask=$(taskset -p $pid | awk -F': ' '{print $2}') + +# get numerical list (decimal) of numa nodes and cores in the affinity mask +numa_list=$(hwloc-calc --physical --intersect NUMAnode 0x$affinity_mask) +cpu_list=$(hwloc-calc --physical --intersect core 0x$affinity_mask) +cpu_list_short=$(taskset -pc $pid | awk -F': ' '{print $2}') + +# calculate number of numa nodes / cpus +IFS=',' read -r -a numa_array <<< "$numa_list" +numa_count=${#numa_array[@]} +IFS=',' read -r -a cpu_array <<< "$cpu_list" +cpu_count=${#cpu_array[@]} + +# each GPU is associated with a NUMA node: +gpu_list=$numa_list +gpu_count=$numa_count + +# make devices visible: only export devices for which the current process has affinity +export CUDA_VISIBLE_DEVICES=$gpu_list + +# export the nic - doesn't work with pytorch +#IFS=',' read -r first_node other_nodes <<< "$numa_list" +#first_nic="cxi${first_node}" +#export FI_CXI_DEVICE_NAME=$first_nic + +# get local and global ranks from either Slurm or OMPI environment variables +lrank=0 +grank=0 +if [ -z ${OMPI_COMM_WORLD_LOCAL_RANK+x} ] +then + let lrank=$SLURM_LOCALID + let grank=$SLURM_PROCID + + # MPICH options + # is required for CUDA-aware MPI to work + export MPICH_GPU_SUPPORT_ENABLED=1 + # MPICH_GPU_SUPPORT_ENABLED=1 and MPICH_SMP_SINGLE_COPY_MODE=XPMEM are + # mutually exclusive, and MPICH will fall back to CMA if GPU support is + # enabled. + #export MPICH_SMP_SINGLE_COPY_MODE=xpmem + export MPICH_SMP_SINGLE_COPY_MODE=CMA +else + let lrank=$OMPI_COMM_WORLD_LOCAL_RANK + let grank=$OMPI_COMM_WORLD_RANK + + # OPENMPI options + #export OMPI_MCA_btl_ofi_mode=2 + #export OMPI_MCA_pml_ob1_max_rdma_per_request=1 +fi + +# print info about distribution of jobs +if [[ $grank == 0 ]] +then + echo "Slurm Job Hostlist: $SLURM_JOB_NODELIST" +fi +echo "Hostname: $(hostname) Rank: $grank, Local $lrank, GPUs $gpu_list (count=$gpu_count), CPUs $cpu_list_short (count=$cpu_count)" + +# export variables for pytorch +export LOCAL_RANK=${lrank} +export RANK=${grank} +export MASTER_ADDR=$(scontrol show hostname $SLURM_NODELIST | head -n1) +export MASTER_PORT=29500 # default from torch launcher +export WORLD_SIZE=$SLURM_NTASKS +export TORCH_CUDA_ARCH_LIST=9.0 +export MAX_JOBS=$cpu_count +export CXX=`which g++` +export CC=`which gcc` + +# would make the code print out a stacktrace in case of a crash - only works when compiled with -traceback compiler option +#export NVCOMPILER_TERM=trace + +## OpenACC options +## makes small H2D copies faster +#export NVCOMPILER_ACC_DEFER_UPLOADS=1 +#export NVCOMPILER_ACC_SYNCHRONOUS=1 +#export NVCOMPILER_ACC_USE_GRAPH=1 +#export NV_ACC_CUDA_MEMALLOCASYNC=1 +#export NV_ACC_CUDA_MEMALLOCASYNC_POOLSIZE=500000000000 + +## is required to avoid hangs at scale +#export FI_MR_CACHE_MONITOR=disabled +#export FI_CXI_SAFE_DEVMEM_COPY_THRESHOLD=0 +# +## speeds up (or rather recovers good performance) GPU direct communications over the network. Without it MPI takes 3-5x longer when G2G is enabled +#export FI_CXI_RX_MATCH_MODE=software + +# run the command +"$@" diff --git a/allreduce-perf/mpi-cpp/env/README b/allreduce-perf/mpi-cpp/env/README new file mode 100644 index 0000000..f03ec78 --- /dev/null +++ b/allreduce-perf/mpi-cpp/env/README @@ -0,0 +1,14 @@ + + +build instructions for uenv + +run on compute node: +==================== + +/path/to/stackinator/bin/stack-config -c /path/to/build-cace-config.yaml -r ./recipe -b /dev/shm//allreduce -s /path/to/alps-cluster-config/santis -m /user-environment --develop + +cd /dev/shm//allreduce + +env --ignore-environment PATH=/usr/bin:/bin:`pwd`/spack/bin HOME=$HOME https_proxy=$https_proxy http_proxy=$http_proxy no_proxy="$no_proxy" make store.squashfs -j288 + +cp store.squashfs /this/path/ diff --git a/allreduce-perf/mpi-cpp/env/recipe/compilers.yaml b/allreduce-perf/mpi-cpp/env/recipe/compilers.yaml new file mode 100644 index 0000000..199f37c --- /dev/null +++ b/allreduce-perf/mpi-cpp/env/recipe/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@12.3 +gcc: + specs: + - gcc@12.3 diff --git a/allreduce-perf/mpi-cpp/env/recipe/config.yaml b/allreduce-perf/mpi-cpp/env/recipe/config.yaml new file mode 100644 index 0000000..f4984b7 --- /dev/null +++ b/allreduce-perf/mpi-cpp/env/recipe/config.yaml @@ -0,0 +1,7 @@ +name: test_mpi +spack: + commit: origin/develop + repo: https://github.com/spack/spack.git +store: /user-environment +description: Test MPI in CMake + diff --git a/allreduce-perf/mpi-cpp/env/recipe/environments.yaml b/allreduce-perf/mpi-cpp/env/recipe/environments.yaml new file mode 100644 index 0000000..57965c2 --- /dev/null +++ b/allreduce-perf/mpi-cpp/env/recipe/environments.yaml @@ -0,0 +1,26 @@ +test_mpi-env: + compiler: + - toolchain: gcc + spec: gcc + mpi: + spec: cray-mpich@8.1.29 + gpu: cuda + unify: true + specs: + - gcc + - cray-mpich + - aws-ofi-nccl@master + - xpmem + - cmake + - cuda@12.3 + - ninja + - fmt + - osu-micro-benchmarks + variants: + - +mpi + - +cuda + - cuda_arch=90 + views: + default: + link: roots + diff --git a/allreduce-perf/mpi-cpp/run/run.sh b/allreduce-perf/mpi-cpp/run/run.sh new file mode 100755 index 0000000..fdce8b4 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/run.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +NODES=$1 +NTASKS_PER_NODE=$2 +TIME_LIMIT=${3:-"01:00:00"} + +CPUS_PER_TASK=$(( 288 / NTASKS_PER_NODE )) +NTASKS=$(( NODES * NTASKS_PER_NODE )) + +NODES_STR=$(printf "%04d" "$NODES") +NTASKS_STR=$(printf "%05d" "$NTASKS") + +sbatch \ + --uenv=/bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/mpi-cpp/env/store.squashfs < \$srunlist + + local unique_nodes=\$(sort \$srunlist | uniq | wc -l) + local total_tasks=\$(wc -l < \$srunlist) + + if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty. Exiting." + exit 1 + else + echo "All nodes and tasks are healthy." + fi +} + +check_health + +uenv view default +export LD_LIBRARY_PATH=/user-environment/env/default/lib64:\$LD_LIBRARY_PATH + +ITERATIONS=10 +SUB_ITERATIONS=5 +SIZE=$((1 << 30)) + +run_all() { + #srun \ + # --cpu-bind=rank_ldom \ + # ../../common/launch_wrapper \ + # ../src/build/allreduce_bench \$ITERATIONS \$SUB_ITERATIONS host \$SIZE \$1 \$2 + + #srun \ + # --cpu-bind=rank_ldom \ + # ../../common/launch_wrapper \ + # ../src/build/allreduce_bench \$ITERATIONS \$SUB_ITERATIONS pinned_host \$SIZE \$1 \$2 + + #srun \ + # --cpu-bind=rank_ldom \ + # ../../common/launch_wrapper \ + # ../src/build/allreduce_bench \$ITERATIONS \$SUB_ITERATIONS mpi_host \$SIZE \$1 \$2 + + srun \ + --cpu-bind=rank_ldom \ + ../../common/launch_wrapper \ + ../src/build/allreduce_bench \$ITERATIONS \$SUB_ITERATIONS device \$SIZE \$1 \$2 +} + +run_all 0 0 +run_all 1 0 +run_all 0 1 +run_all 1 1 + +EOT + diff --git a/allreduce-perf/mpi-cpp/run/run_all.sh b/allreduce-perf/mpi-cpp/run/run_all.sh new file mode 100755 index 0000000..497916f --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/run_all.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +./run.sh 1 4 01:00:00 +./run.sh 4 4 01:00:00 +./run.sh 8 4 01:00:00 +./run.sh 16 4 01:00:00 +./run.sh 32 4 03:00:00 +./run.sh 64 4 03:00:00 +./run.sh 128 4 04:00:00 +./run.sh 256 4 04:00:00 + +./run.sh 4 1 01:00:00 +./run.sh 16 1 01:00:00 +./run.sh 32 1 01:00:00 +./run.sh 64 1 01:00:00 +./run.sh 128 1 03:00:00 +./run.sh 256 1 03:00:00 diff --git a/allreduce-perf/mpi-cpp/src/CMakeLists.txt b/allreduce-perf/mpi-cpp/src/CMakeLists.txt new file mode 100644 index 0000000..5dfc47b --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.22) + +set(CMAKE_CUDA_ARCHITECTURES "90" CACHE STRING "CUDA architectures") + +project(allreduce_bench LANGUAGES CXX CUDA) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CUDA_STANDARD 20) +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_SEPARABLE_COMPILATION ON) + +include(CheckIPOSupported) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +check_ipo_supported(RESULT HAVE_LTO OUTPUT ERR_LTO) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + if(HAVE_LTO) + message (STATUS "LTO support found, enabling") + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) + else() + message(STATUS "No LTO: ${ERR_LTO}") + endif() +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +endif() + +add_executable(allreduce_bench) + +target_sources(allreduce_bench PRIVATE + bench.cpp + memory.cu + allreduce.cu +) + +target_compile_options(allreduce_bench PRIVATE + $<$:-Wall -Wextra -Wpedantic> + $<$:--compiler-options -Wall,-Wextra> +) + +find_package(MPI REQUIRED) +find_package(CUDAToolkit REQUIRED) + +target_link_libraries(allreduce_bench PRIVATE + MPI::MPI_CXX + CUDA::cudart +) diff --git a/allreduce-perf/mpi-cpp/src/allreduce.cu b/allreduce-perf/mpi-cpp/src/allreduce.cu new file mode 100644 index 0000000..7179c34 --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/allreduce.cu @@ -0,0 +1,24 @@ +#include "allreduce.hpp" + +namespace test { +namespace kernel { + +template +__global__ void sum(T* dest, T const * src, std::size_t size) { + auto idx = static_cast(blockIdx.x)*blockDim.x + threadIdx.x; + if (idx < size) { + dest[idx] += src[idx]; + } +} + +} // namespace kernel + +template<> +void sum_device(float* dest, float const * src, std::size_t size) { + int block_size = 256; + int grid_size = (size + block_size - 1)/block_size; + kernel::sum<<>>(dest, src, size); + CHECK_CUDA(cudaDeviceSynchronize()); +} + +} // namespace test diff --git a/allreduce-perf/mpi-cpp/src/allreduce.hpp b/allreduce-perf/mpi-cpp/src/allreduce.hpp new file mode 100644 index 0000000..2423460 --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/allreduce.hpp @@ -0,0 +1,87 @@ +#pragma once + +#include "memory.hpp" +#include + +namespace test { + +template +void all_reduce(buffer& buffer, MPI_Datatype mpi_type) { + CHECK_MPI(MPI_Allreduce(MPI_IN_PLACE, buffer.data(), buffer.size(), mpi_type, MPI_SUM, MPI_COMM_WORLD)); +} + +template +void sum_device(T* dest, T const * src, std::size_t size); + +template +void sum(buffer& dest, buffer const & src) { + switch (dest.type()) { + case mem_type::device: + sum_device(dest.data(), src.data(), dest.size()); + break; + + default: + for (std::size_t i=0; i +void copy(buffer& dest, buffer const & src) { + switch (dest.type()) { + case mem_type::device: + CHECK_CUDA(cudaMemcpy(dest.data(), src.data(), dest.size()*sizeof(T), cudaMemcpyDeviceToDevice)); + CHECK_CUDA(cudaDeviceSynchronize()); + break; + + default: + std::memcpy(dest.data(), src.data(), dest.size()*sizeof(T)); + } +} + +template +void all_reduce_naive_ring(buffer& b) { + static thread_local std::unique_ptr> recv_buffer_ptr; + if (!recv_buffer_ptr || + (recv_buffer_ptr->size() < b.size()) || + (recv_buffer_ptr->type() != b.type()) || + (recv_buffer_ptr->padding() != b.padding())) { + recv_buffer_ptr.reset(); + recv_buffer_ptr = std::make_unique>(b.type(), b.size(), b.padding()); + } + auto& recv_buffer = *recv_buffer_ptr; + + static thread_local std::unique_ptr> send_buffer_ptr; + if (!send_buffer_ptr || + (send_buffer_ptr->size() < b.size()) || + (send_buffer_ptr->type() != b.type()) || + (send_buffer_ptr->padding() != b.padding())) { + send_buffer_ptr.reset(); + send_buffer_ptr = std::make_unique>(b.type(), b.size(), b.padding()); + } + auto& send_buffer = *send_buffer_ptr; + + int comm_rank; + CHECK_MPI(MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank)); + int comm_size; + CHECK_MPI(MPI_Comm_size(MPI_COMM_WORLD, &comm_size)); + + copy(send_buffer, b); + auto const recv_rank = (comm_rank + 1) % comm_size; + auto const send_rank = ((comm_rank + comm_size) - 1) % comm_size; + + auto* s = &send_buffer; + auto* r = &recv_buffer; + + for (int i=0; idata(), s->size()*sizeof(T), MPI_BYTE, send_rank, 0, + r->data(), s->size()*sizeof(T), MPI_BYTE, recv_rank, 0, + MPI_COMM_WORLD, MPI_STATUS_IGNORE)); + sum(b, *r); + std::swap(s, r); + } +} + +} // namespace test diff --git a/allreduce-perf/mpi-cpp/src/bench.cpp b/allreduce-perf/mpi-cpp/src/bench.cpp new file mode 100644 index 0000000..ec82b25 --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/bench.cpp @@ -0,0 +1,166 @@ +#include "memory.hpp" +#include "allreduce.hpp" + +#include +#include +#include +#include +#include + +namespace test { + +inline mem_type parse_mem_type(std::string_view s) { + if (s == "host") { + return mem_type::host; + } else if (s == "pinned_host") { + return mem_type::pinned_host; + } else if (s == "mpi_host") { + return mem_type::mpi_host; + } else if (s == "device") { + return mem_type::device; + } else { + std::cout << "Memory type must be host, pinned_host, or device; got \"" + << s << "\"\n"; + std::exit(1); + } +} + +} // namespace test + +int main(int argc, char** argv) { + if (argc != 7) { + std::cout << "Usage: allreduce_bench " + " \n" + "got " << (argc-1) << " args\n"; + std::exit(1); + } + + CHECK_MPI(MPI_Init(&argc, &argv)); + + int comm_rank; + CHECK_MPI(MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank)); + int comm_size; + CHECK_MPI(MPI_Comm_size(MPI_COMM_WORLD, &comm_size)); + + std::size_t const num_iterations = std::stoul(argv[1]); + std::size_t const num_sub_iterations = std::stoul(argv[2]); + test::mem_type const t = test::parse_mem_type(argv[3]); + std::size_t const size = std::stoul(argv[4]); + std::size_t const padding = std::stoul(argv[5]); + bool const naive = std::stoul(argv[6]); + + CHECK_CUDA(cudaDeviceSynchronize()); + if (comm_rank == 0) { + std::cout << "mem_type: " << argv[3] << '\n'; + std::cout << "size: " << size << '\n'; + std::cout << "padding: " << padding << '\n'; + std::cout << "naive algo: " << naive << '\n'; + std::cout << std::endl; + } + + using type = float; + auto mpi_type = MPI_FLOAT; + + const auto num_data_transfers = 2*(comm_size - 1); + const double data_transfers_per_rank = (double)num_data_transfers/comm_size; + + std::vector> times(num_iterations); + std::vector start_times(num_iterations); + std::vector> bws(num_iterations); + std::vector start_bws(num_iterations); + for (auto& v : times) v.resize(num_sub_iterations-1); + for (auto& v : bws) v.resize(num_sub_iterations-1); + + // Warmup + { + const std::chrono::time_point start = + std::chrono::steady_clock::now(); + + { + auto buffer = test::buffer(t, size, padding); + test::fill(buffer, (type)1); + if (!naive) + test::all_reduce(buffer, mpi_type); + else + test::all_reduce_naive_ring(buffer); + + } + + CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); + + auto const stop = std::chrono::steady_clock::now(); + if (comm_rank == 0) { + std::cout << "[-1] time: " + << std::chrono::duration(stop - start).count() + << '\n'; + } + } + + for (std::size_t i = 0; i < num_iterations; ++i) { + auto buffer = test::buffer(t, size, padding); + test::fill(buffer, (type)1); + + for (std::size_t j = 0; j < num_sub_iterations; ++j) { + const std::chrono::time_point start = + std::chrono::steady_clock::now(); + + if (!naive) + test::all_reduce(buffer, mpi_type); + else + test::all_reduce_naive_ring(buffer); + + CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); + + auto const stop = std::chrono::steady_clock::now(); + auto const delta = std::chrono::duration(stop - start).count(); + + const double algbw = (size*sizeof(type))/delta*10e-9; //GB/s + const double bw = algbw * data_transfers_per_rank; + if (comm_rank == 0) { + std::cout << "[" << i << ":" << j << "] time: " << std::setw(12) << delta << " bw: " << std::setw(12) << bw << '\n'; + } + if (j==0) { + start_times[i] = delta; + start_bws[i] = bw; + } + else { + times[i][j-1] = delta; + bws[i][j-1] = bw; + } + } + } + + auto _mean = [](std::vector const & v, double& mu, std::size_t& n) { + for (auto x : v) { + double const delta = x - mu; + mu += delta/(++n); + } + }; + + auto mean = [&](std::vector> const & v) { + double mu = 0; + std::size_t n = 0; + for (auto& x : v) _mean(x, mu, n); + return mu; + }; + + if (comm_rank == 0) { + std::cout << "\n======================================="; + std::cout << "\ntype: " << std::setw(12) << argv[3]; + std::cout << "\nsize: " << std::setw(12) << size*sizeof(type); + std::cout << "\npad: " << std::setw(12) << padding*sizeof(type); + std::cout << "\nnaive: " << std::setw(12) << naive; + std::cout << "\ntime: " << std::setw(12) << mean(times); + std::cout << "\ntime0: " << std::setw(12) << mean({start_times}); + std::cout << "\nbw: " << std::setw(12) << mean(bws); + std::cout << "\nbw0: " << std::setw(12) << mean({start_bws}); + std::cout << "\n======================================="; + std::cout << "\n"; + std::cout << std::endl; + } + + CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); + CHECK_MPI(MPI_Finalize()); + + return EXIT_SUCCESS; +} diff --git a/allreduce-perf/mpi-cpp/src/memory.cu b/allreduce-perf/mpi-cpp/src/memory.cu new file mode 100644 index 0000000..79ca9b5 --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/memory.cu @@ -0,0 +1,25 @@ +#include "memory.hpp" + +namespace test { + +namespace kernel { + +template +__global__ void fill(T* ptr, std::size_t size, T def) { + auto idx = static_cast(blockIdx.x)*blockDim.x + threadIdx.x; + if (idx < size) { + ptr[idx] = def; + } +} + +} // namespace kernel + +template<> +void fill_device(float* ptr, std::size_t size, float def) { + int block_size = 256; + int grid_size = (size + block_size - 1)/block_size; + kernel::fill<<>>(ptr, size, def); + CHECK_CUDA(cudaDeviceSynchronize()); +} + +} // namespace test diff --git a/allreduce-perf/mpi-cpp/src/memory.hpp b/allreduce-perf/mpi-cpp/src/memory.hpp new file mode 100644 index 0000000..6ac5352 --- /dev/null +++ b/allreduce-perf/mpi-cpp/src/memory.hpp @@ -0,0 +1,131 @@ +#pragma once + +#include +#include +#include +#include +#include + +#define CHECK_MPI(x) \ + if (auto e = x; e != MPI_SUCCESS) { \ + std::cout << "MPI error: " << e << '\n'; \ + std::exit(1); \ + } +#define CHECK_CUDA(x) \ + if (auto e = x; e != cudaSuccess) { \ + std::cout << "CUDA error: " << e << '\n'; \ + std::exit(1); \ + } + +namespace test { + +enum class mem_type { + host, + pinned_host, + mpi_host, + device, +}; + +template +T* malloc(mem_type t, std::size_t n) { + switch (t) { + case mem_type::host: + return static_cast(::malloc(n)); + + case mem_type::pinned_host: { + void* p = nullptr; + CHECK_CUDA(cudaMallocHost(&p, n)); + return static_cast(p); + } + + case mem_type::mpi_host: { + void* p = nullptr; + CHECK_MPI(MPI_Alloc_mem(n, MPI_INFO_NULL, &p)); + return static_cast(p); + } + + case mem_type::device: { + void* p = nullptr; + CHECK_CUDA(cudaMalloc(&p, n)); + return static_cast(p); + } + + default: + std::terminate(); + } +} + +inline void free(mem_type t, void* p) { + switch (t) { + case mem_type::host: + ::free(p); + break; + + case mem_type::pinned_host: + CHECK_CUDA(cudaFreeHost(p)); + break; + + case mem_type::mpi_host: + CHECK_MPI(MPI_Free_mem(p)); + break; + + case mem_type::device: + CHECK_CUDA(cudaFree(p)); + break; + + default: + std::terminate(); + } +} + +template +struct buffer { + + mem_type _type = mem_type::host; + std::size_t _size = 0ul; + unsigned _padding = 0ul; + T* _data = nullptr; + + buffer(mem_type t, std::size_t size, unsigned padding = 0u) + : _type{t} + , _size{size} + , _padding{padding} + { + std::size_t space = (_size + _padding)*sizeof(T); + T* ptr = malloc(_type, space); + if (!ptr) { + std::cout << "allocation failed" << '\n'; + std::exit(1); + } + _data = ptr + _padding; + } + + ~buffer() { + if (_data) free(_type, _data - _padding); + } + + buffer(buffer const&) = delete; + buffer(buffer&& other) = delete; + + mem_type type() const noexcept { return _type; } + unsigned padding() const noexcept { return _padding; } + T* data() const noexcept { return _data; } + std::size_t size() const noexcept { return _size; } +}; + +template +void fill_device(T* ptr, std::size_t size, T def); + +template +void fill(buffer& b, T def = T{1}) { + switch (b._type) { + case mem_type::device: + fill_device(b.data(), b.size(), def); + break; + + default: + std::fill(b.data(), b.data()+b.size(), def); + } +} + +} // namespace test diff --git a/allreduce-perf/nccl-pytorch/env/Dockerfile b/allreduce-perf/nccl-pytorch/env/Dockerfile new file mode 100644 index 0000000..7599c12 --- /dev/null +++ b/allreduce-perf/nccl-pytorch/env/Dockerfile @@ -0,0 +1,13 @@ +ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:24.03-py3 +FROM ${FROM_IMAGE_NAME} + +RUN apt-get update \ + && apt-get install -yqq --no-install-recommends openssh-server \ + && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /run/sshd + +RUN pip install -vvv \ + git+https://github.com/mlcommons/logging.git@2.1.0-rc1 \ + git+https://github.com/NVIDIA/mlperf-common.git \ + zarr \ + git+https://github.com/google/tensorstore.git diff --git a/allreduce-perf/nccl-pytorch/env/podman_build.sh b/allreduce-perf/nccl-pytorch/env/podman_build.sh new file mode 100755 index 0000000..2fc0a54 --- /dev/null +++ b/allreduce-perf/nccl-pytorch/env/podman_build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +podman build --ulimit nofile=$(ulimit -n):$(ulimit -n) -t boeschf/ngc:24.03 . +enroot import -x mount -o /bret/scratch/cscs/lukasd/images/ngc-megatron+24.03.sqsh podman://lukasgd/ngc-megatron:24.03 diff --git a/allreduce-perf/nccl-pytorch/run/ngc.toml b/allreduce-perf/nccl-pytorch/run/ngc.toml new file mode 100644 index 0000000..c9d65a8 --- /dev/null +++ b/allreduce-perf/nccl-pytorch/run/ngc.toml @@ -0,0 +1,16 @@ +image = "/bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/nccl-pytorch/env/ngc.sqsh" + +mounts = [ + "/bret:/bret", + "/mchstor2:/mchstor2", + "/etc/slurm:/etc/slurmi:ro" +] + +writable = true + +workdir = "/bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/nccl-pytorch/run" + +[annotations] +com.hooks.aws_ofi_nccl.enabled = "true" +com.hooks.aws_ofi_nccl.variant = "cuda12" + diff --git a/allreduce-perf/nccl-pytorch/run/run.sh b/allreduce-perf/nccl-pytorch/run/run.sh new file mode 100755 index 0000000..19c93a7 --- /dev/null +++ b/allreduce-perf/nccl-pytorch/run/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +NODES=$1 +NTASKS_PER_NODE=$2 +TIME_LIMIT=${3:-"01:00:00"} + +CPUS_PER_TASK=$(( 288 / NTASKS_PER_NODE )) +NTASKS=$(( NODES * NTASKS_PER_NODE )) + +NODES_STR=$(printf "%04d" "$NODES") +NTASKS_STR=$(printf "%05d" "$NTASKS") + +sbatch < \$srunlist + + local unique_nodes=\$(sort \$srunlist | uniq | wc -l) + local total_tasks=\$(wc -l < \$srunlist) + + if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty. Exiting." + exit 1 + else + echo "All nodes and tasks are healthy." + fi +} + +check_health + +# Set torch.distributed variables +export MASTER_ADDR=\$(scontrol show hostname \$SLURM_NODELIST | head -n1) +export MASTER_PORT=29500 # default from torch launcher +export WORLD_SIZE=$NTASKS +export TORCH_CUDA_ARCH_LIST=9.0 +export MAX_JOBS=64 + +export NCCL_DEBUG=INFO + +run_cmd="CUDA_VISIBLE_DEVICES=0,1,2,3 python -u -m torch.distributed.run --nproc_per_node=$TASKS_PER_NODE --nnodes $NODES --rdzv_endpoint \$(scontrol show hostnames \$SLURM_JOB_NODELIST | head -n 1):6000 --rdzv_backend c10d ../src/all_reduce.py" + +srun -ul \ + --cpu-bind=verbose,rank_ldom \ + --environment="$(realpath ngc.toml)" \ + sh -c "\${run_cmd}" + +EOT diff --git a/allreduce-perf/nccl-pytorch/run/run_all.sh b/allreduce-perf/nccl-pytorch/run/run_all.sh new file mode 100755 index 0000000..c8c99f3 --- /dev/null +++ b/allreduce-perf/nccl-pytorch/run/run_all.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +./run.sh 1 4 01:00:00 +./run.sh 4 4 01:00:00 +./run.sh 8 4 01:00:00 +./run.sh 16 4 01:00:00 +./run.sh 32 4 03:00:00 +./run.sh 64 4 03:00:00 +./run.sh 128 4 04:00:00 +./run.sh 256 4 04:00:00 diff --git a/allreduce-perf/nccl-pytorch/src/all_reduce.py b/allreduce-perf/nccl-pytorch/src/all_reduce.py new file mode 100644 index 0000000..7653a2f --- /dev/null +++ b/allreduce-perf/nccl-pytorch/src/all_reduce.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python + +""" + +The latest version of this program can be found at https://github.com/stas00/ml-engineering + +This benchmark is very similar to https://github.com/NVIDIA/nccl-tests but it's much easier to set +up as it only requires PyTorch to be installed + +This version: +- has been derived from @jeffra's gist: https://gist.github.com/jeffra/b5e80466b4c86be00ea3b6f130fb7a36 +- which in turn is derived from the logic in https://github.com/NVIDIA/nccl-tests +- with contributions from: + * Indu Thangakrishnan https://github.com/indhub to handle timing correctly using cuda events + + +Important notes: + +- when you finished running this benchmark you want to pay attention to the busbw result (not + algbw) as explained here https://github.com/NVIDIA/nccl-tests/blob/master/doc/PERFORMANCE.md#bandwidth + +- similar to NVIDIA/nccl-tests this benchmark measures a unidirectional bandwidth - so compare the + outcome against the advertised unidirectional peak throughput and not bi-directional (duplex) + +- currently this benchmark tests a payload of 4GB (M * N * 4). If your target application uses a + much smaller payload you want to modify M*N*4 to match the target payload. To calculate the + payload use the number of parameters sent in each reduction multiplied by 2 (bf16/fp16) or 4 + (fp32). e.g., if a reduction is of a single layer of 1B params, and you use bf16 grads it'd be + 2GB of payload. depending on the framework you use (DDP, FSDP, DeepSpeed ZeRO) they all use + different logic to how much of a message size they send. + +- if you are wondering whether you need to also run https://github.com/NVIDIA/nccl-tests - I + already validated that I got very similar results with ./build/all_reduce_perf -b 4G -e 4G + (tested with mpirun on 4 nodes). It should be either on par or slightly slower because it uses a + blocking approach - that is it wait for each new all_reduce to finish before firing the next + one, whereas nccl-tests fires them all in an async fashion (you can add `-z` to nccl-tests to + emulate blocking) + +- to benchmark other collectives use nccl-tests. It's also useful if you want to test a range of + payloads, e.g. there you'd set -b 8 -e 4G -f 2 and it will test many sizes automatically. + +To run on 4 nodes: + +GPUS_PER_NODE=8 +NNODES=4 +MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) +MASTER_PORT=6000 +python -u -m torch.distributed.run \ + --nproc_per_node $GPUS_PER_NODE \ + --nnodes $NNODES \ + --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ + --rdzv_backend c10d \ + --max_restarts 0 \ + --role `hostname -s`: \ + --tee 3 \ + all_reduce_bench.py + +note: adapt MASTER_ADDR to node rank 0's hostname if it's not a SLURM environment where it's derived automatically + +e.g. example to run with salloc+srun: + +salloc --partition=mypartition --nodes=4 --ntasks-per-node=1 --cpus-per-task=48 --gres=gpu:8 --time=1:00:00 bash + +srun --gres=gpu:8 --nodes=4 --tasks-per-node=1 python -u -m torch.distributed.run --nproc_per_node=8 \ +--nnodes 4 --rdzv_endpoint $(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1):6000 --rdzv_backend \ +c10d all_reduce_bench.py + +To do a quick test on 2 gpus: + +python -u -m torch.distributed.run --nproc_per_node=2 --rdzv_endpoint localhost:6000 --rdzv_backend c10d \ +all_reduce_bench.py + +""" + +import os +import socket +import torch +import torch.distributed as dist + +TRIALS = 5 + +# these emulate the payload which will become a M * N * 4-sized tensor below +#N = 500000 +#M = 2000 +N = 524288 +M = 2048 + +def timed_allreduce(mat, start_event, end_event): + dist.barrier() + start_event.record() + dist.all_reduce(mat) + end_event.record() + + torch.cuda.synchronize() + duration = start_event.elapsed_time(end_event) / 1000 + + n = dist.get_world_size() + size = M * N * 4 # 4 is 4 bytes in fp32 + # note that this is following the same math as NVIDIA/nccl-tests + algbw = torch.tensor([size / duration]).cuda(local_rank) + + # calculate mean across all ranks + dist.reduce(algbw, dst=0, op=dist.ReduceOp.SUM) + algbw /= n + + return algbw + +def run(local_rank): + hostname = socket.gethostname() + is_global_rank_0 = dist.get_rank() == 0 + + mat = torch.rand(N, M, dtype=torch.float32).cuda(local_rank) + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + # do a few warm up iterations + for i in range(2): + timed_allreduce(mat, start_event, end_event) + + # real benchmark + algbw_gather = [] + for i in range(TRIALS): + if is_global_rank_0: + print(i+1) + algbw_gather += timed_allreduce(mat, start_event, end_event) + + algbw = torch.mean(torch.stack(algbw_gather)) + + # the 2*(n-1)/n busbw correction factor specific to all-reduce is explained here: + # https://github.com/NVIDIA/nccl-tests/blob/master/doc/PERFORMANCE.md#allreduce + # busbw reflects how optimally the hardware is used + n = dist.get_world_size() + busbw = algbw * (2*(n - 1) / n) + + if is_global_rank_0: + print(f"The average bandwidth of all_reduce with a {M*N*4/1e9}GB payload ({TRIALS} trials, {n} ranks):\n", + f"algbw: {algbw/1e9:.3f} GBps ({algbw*8/1e9:.1f} Gbps)\n", + f"busbw: {busbw/1e9:.3f} GBps ({busbw*8/1e9:.1f} Gbps)\n", + ) + +def init_processes(local_rank, fn, backend='nccl'): + torch.cuda.set_device(local_rank) + dist.init_process_group(backend) + fn(local_rank) + + +if __name__ == "__main__": + local_rank = int(os.environ["LOCAL_RANK"]) + init_processes(local_rank=local_rank, fn=run) diff --git a/allreduce-perf/nccl-tests-cpp/env/import.sh b/allreduce-perf/nccl-tests-cpp/env/import.sh new file mode 100755 index 0000000..f841516 --- /dev/null +++ b/allreduce-perf/nccl-tests-cpp/env/import.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +enroot import -x mount -o /bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/nccl-tests-cpp/env/nccl-test.sqsh docker://jfrog.svc.cscs.ch#reframe-oci/nccl-tests:cuda12.3 diff --git a/allreduce-perf/nccl-tests-cpp/run/nccl-test.toml b/allreduce-perf/nccl-tests-cpp/run/nccl-test.toml new file mode 100644 index 0000000..e466f48 --- /dev/null +++ b/allreduce-perf/nccl-tests-cpp/run/nccl-test.toml @@ -0,0 +1,22 @@ +image = "/bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/nccl-tests-cpp/env/nccl-test.sqsh" + +mounts = [ + "/bret:/bret", + "/mchstor2:/mchstor2", + "/etc/slurm:/etc/slurmi:ro" +] + + +writable = true + +workdir = "/bret/scratch/cscs/boeschf/alps-gh200-reproducers/allreduce-perf/nccl-tests-cpp/run" + +[annotations.com.hooks] +aws_ofi_nccl.enabled = "true" +aws_ofi_nccl.variant = "cuda12" + +[env] +NVIDIA_DISABLE_REQUIRE = "1" +#ENROOT_CXI_HOOK = "1" +#NCCL_NET_PLUGIN = "ofi" +#NCCL_DEBUG = "INFO" diff --git a/allreduce-perf/nccl-tests-cpp/run/run.sh b/allreduce-perf/nccl-tests-cpp/run/run.sh new file mode 100755 index 0000000..7957839 --- /dev/null +++ b/allreduce-perf/nccl-tests-cpp/run/run.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +NODES=$1 +NTASKS_PER_NODE=$2 +TIME_LIMIT=${3:-"01:00:00"} + +CPUS_PER_TASK=$(( 288 / NTASKS_PER_NODE )) +NTASKS=$(( NODES * NTASKS_PER_NODE )) + +NODES_STR=$(printf "%04d" "$NODES") +NTASKS_STR=$(printf "%05d" "$NTASKS") + +sbatch < \$srunlist + + local unique_nodes=\$(sort \$srunlist | uniq | wc -l) + local total_tasks=\$(wc -l < \$srunlist) + + if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty. Exiting." + exit 1 + else + echo "All nodes and tasks are healthy." + fi +} + +check_health + +srun \ + --cpu-bind=verbose,rank_ldom \ + --environment="$(realpath nccl-test.toml)" \ + --mpi=pmi2 \ + all_reduce_perf -b 4294967296 -e 4294967296 -g 1 + +EOT diff --git a/allreduce-perf/nccl-tests-cpp/run/run_all.sh b/allreduce-perf/nccl-tests-cpp/run/run_all.sh new file mode 100755 index 0000000..497916f --- /dev/null +++ b/allreduce-perf/nccl-tests-cpp/run/run_all.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +./run.sh 1 4 01:00:00 +./run.sh 4 4 01:00:00 +./run.sh 8 4 01:00:00 +./run.sh 16 4 01:00:00 +./run.sh 32 4 03:00:00 +./run.sh 64 4 03:00:00 +./run.sh 128 4 04:00:00 +./run.sh 256 4 04:00:00 + +./run.sh 4 1 01:00:00 +./run.sh 16 1 01:00:00 +./run.sh 32 1 01:00:00 +./run.sh 64 1 01:00:00 +./run.sh 128 1 03:00:00 +./run.sh 256 1 03:00:00 From 4572ad5eab958a123ff4584797a4cb6a6115d5f6 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 30 May 2024 17:36:47 +0200 Subject: [PATCH 2/8] added plotting file --- allreduce-perf/mpi-cpp/run/plot.py | 218 +++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 allreduce-perf/mpi-cpp/run/plot.py diff --git a/allreduce-perf/mpi-cpp/run/plot.py b/allreduce-perf/mpi-cpp/run/plot.py new file mode 100644 index 0000000..989ceeb --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plot.py @@ -0,0 +1,218 @@ +import os +import re +import matplotlib.pyplot as plt +import seaborn as sns + +# List of files to process +file_paths = [ + 'data/job_n_00004_N_0001_TPN_4.out', + 'data/job_n_00004_N_0004_TPN_1.out', + 'data/job_n_00016_N_0004_TPN_4.out', + 'data/job_n_00016_N_0016_TPN_1.out', + 'data/job_n_00032_N_0008_TPN_4.out', + 'data/job_n_00032_N_0032_TPN_1.out', + 'data/job_n_00064_N_0016_TPN_4.out', + 'data/job_n_00064_N_0064_TPN_1.out', + 'data/job_n_00128_N_0032_TPN_4.out', + 'data/job_n_00128_N_0128_TPN_1.out', + 'data/job_n_00256_N_0064_TPN_4.out', + 'data/job_n_00256_N_0256_TPN_1.out', +] + +# Output directory for the plots +output_dir = 'plots' + +# Function to parse integers from the filename +def parse_filename(file_path): + filename = os.path.basename(file_path) + pattern = re.compile(r'job_n_(\d+)_N_(\d+)_TPN_(\d+)\.out') + match = pattern.match(filename) + if match: + n = int(match.group(1)) + N = int(match.group(2)) + TPN = int(match.group(3)) + return n, N, TPN + else: + raise ValueError(f"Filename {filename} does not match the expected pattern") + +# Function to parse a single file and extract the blocks +def parse_file(file_path): + with open(file_path, 'r') as file: + content = file.read() + + # Regex pattern to match the blocks + pattern = re.compile(r'={39}\ntype:\s*(\w+)\nsize:\s*([0-9.]+)\npad:\s*([0-9.]+)\nnaive:\s*([0-9.]+)\ntime:\s*([0-9.]+)\ntime0:\s*([0-9.]+)\nbw:\s*([0-9.]+)\nbw0:\s*([0-9.]+)\n={39}') + blocks = pattern.findall(content) + + # Extracting data into a list of dictionaries + data = [] + for block in blocks: + data.append({ + 'type': block[0], + 'size': float(block[1]), + 'pad': int(block[2]), + 'naive': int(block[3]), + 'time': float(block[4]), + 'time0': float(block[5]), + 'bw': float(block[6]), + 'bw0': float(block[7]) + }) + + return data + +# Function to generate block names based on type, naive, and pad +def create_names(block): + base_name = f"{block['type']}_memory" + block_name = base_name + if block['naive'] == 1: + block_name += "_naive" + small_block_name = "naive" + else: + block_name += "_mpi" + small_block_name = "mpi" + if block['pad'] != 0: + block_name += "_pad" + small_block_name += "_pad" + return base_name, block_name, small_block_name + +# Function to plot the data blocks and save to files +def plot_data(blocks, ns, tpns, output_dir): + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # Create a dictionary to group blocks by their base names + grouped_blocks = {} + for file_blocks in blocks: + for block in file_blocks: + base_name, block_name, small_block_name = create_names(block) + print(base_name, block_name, small_block_name) + if base_name not in grouped_blocks: + grouped_blocks[base_name] = [] + grouped_blocks[base_name].append((small_block_name, block, ns[blocks.index(file_blocks)], tpns[blocks.index(file_blocks)])) + + for base_name, block_data in grouped_blocks.items(): + print(f"plotting {base_name}") + + grouped_lines = {} + for idx, (small_block_name, block, n, tpn) in enumerate(block_data): + #color = colors[idx % len(colors)] + small_block_name = f"{small_block_name} (ranks/node={tpn})" + print(small_block_name) + print("") + if small_block_name not in grouped_lines: + grouped_lines[small_block_name] = { 'tpn' : tpn, 'n' : [], 'bw' : [], 'bw0' : [] } + grouped_lines[small_block_name]['n'].append(n) + grouped_lines[small_block_name]['bw'].append(block['bw']) + grouped_lines[small_block_name]['bw0'].append(block['bw0']) + + print(grouped_lines) + + colors = sns.color_palette("husl", len(grouped_lines)+4) + #colors = sns.color_palette("tab10") + markers = ['o', 's', 'D', '^', 'v', 'p', '*', 'X'] + markers = ['o', 'p', 's', 'D', '^', 'v', '>', '<', 'd', 'P', 'h', 'x', '|', 'H', '+', '_'] + line_styles = ['-', '--', '-.', ':', '-', '--', '-.', ':'] + alpha=0.7 + markersize=10 + #line_styles = ['-', '--', '-.', ':', (0, (3, 5, 1, 5)), (0, (1, 10)), (0, (3, 10, 1, 10)), (0, (5, 10)), (0, (3, 1, 1, 1)), (0, (5, 1, 1, 1))] + + plt.figure(figsize=(10, 6)) + plt.title(base_name) + plt.ylabel('bandwidth [GB/s]') + plt.xlabel('number of ranks') + plt.xticks(sorted(set(ns))) + plt.grid(True) # Enable grid lines + plt.yscale('log') + plt.xscale('log') + + idx = 0 + l = 0 + for small_block_name, data in grouped_lines.items(): + #if idx==0 or idx==2: + #if idx==0 or idx==1 or idx==2 or idx==3: + if True: + #if idx==4 or idx==6: + #if idx==4 or idx==5 or idx==6 or idx==7: + + color = colors[idx % len(colors)] + marker = markers[idx % len(markers)] + line_style = line_styles[idx % len(line_styles)] + plt.plot(data['n'], data['bw'], color=color, marker=marker, linestyle=line_style, label=f'{small_block_name}', alpha=alpha, markersize=markersize) + l = l+1 + plt.scatter(data['n'], data['bw0'], facecolors ='none', edgecolors=color, marker=marker, label=f'{small_block_name}_0', alpha=alpha, s=markersize*10) + l = l+1 + idx = idx + 1 + + if base_name == "device_memory": + + plt.ylim([4,5000]) + + n_nccl = [4, 16, 32, 64] + bw_nccl = [343.360, 91.740, 91.673, 91.646] + plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], + label=f'NCCL/container (ranks/node=4)', alpha=alpha, markersize=markersize) + idx = idx + 1 + + n_nccl = [4, 16, 32, 64, 128, 256] + bw_nccl = [342.326, 91.305, 91.311, 91.331, 91.437, 91.462] + plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], + label=f'NCCL/uenv (ranks/node=4)', alpha=alpha, markersize=markersize) + idx = idx + 1 + + n_nccl = [4, 16, 32, 64] + bw_nccl = [22.846, 22.869, 22.889, 22.896] + plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], + label=f'NCCL/uenv (ranks/node=1)', alpha=alpha, markersize=markersize) + idx = idx + 1 + + n_nccl = [4, 16, 32, 64, 128, 256] + bw_nccl = [343.613, 91.2704, 91.2404, 91.339, 91.3899, 91.4304] + print(n_nccl) + print(bw_nccl) + plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], + label=f'NCCL-Tests (ranks/node=4)', alpha=alpha, markersize=markersize) + idx = idx + 1 + + # + ### Add custom legend entries for TPN + ##handles, labels = plt.gca().get_legend_handles_labels() + ##unique_labels = list(dict.fromkeys(labels)) # Remove duplicates while preserving order + ##plt.legend(handles[:len(unique_labels)], unique_labels, loc='best') + # Add custom legend entries + handles, labels = plt.gca().get_legend_handles_labels() + unique_labels = list(dict.fromkeys(labels)) # Remove duplicates while preserving order + print(unique_labels) + unique_handles = [handles[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >=l] # Keep every second handle + unique_labels = [unique_labels[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >=l] # Keep every second label + #unique_handles = [handles[i] for i in range(len(unique_labels))] # Keep every second handle + #unique_labels = [unique_labels[i] for i in range(len(unique_labels))] # Keep every second label + plt.legend(unique_handles, unique_labels, fontsize='small', + loc='center left', bbox_to_anchor=(1, 0.5)) + + + # Use tight layout + plt.tight_layout(rect=[0, 0, 1, 1]) + + # Save the plot to a file + output_file = os.path.join(output_dir, f'{base_name}.png') + plt.savefig(output_file, dpi=300) + plt.close() + + +# Main function to process multiple files +def main(file_paths, output_dir): + all_blocks = [] + ns = [] + tpns = [] + + for file_path in file_paths: + n, _, tpn = parse_filename(file_path) + blocks = parse_file(file_path) + ns.append(n) + tpns.append(tpn) + all_blocks.append(blocks) + + plot_data(all_blocks, ns, tpns, output_dir) + +# Run the main function +main(file_paths, output_dir) From bc1ea3b0b573f7c7371baf3f782fbb82ac62d0f7 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 31 May 2024 16:53:39 +0200 Subject: [PATCH 3/8] fix prefactor --- allreduce-perf/mpi-cpp/src/bench.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/allreduce-perf/mpi-cpp/src/bench.cpp b/allreduce-perf/mpi-cpp/src/bench.cpp index ec82b25..d9feede 100644 --- a/allreduce-perf/mpi-cpp/src/bench.cpp +++ b/allreduce-perf/mpi-cpp/src/bench.cpp @@ -114,7 +114,7 @@ int main(int argc, char** argv) { auto const stop = std::chrono::steady_clock::now(); auto const delta = std::chrono::duration(stop - start).count(); - const double algbw = (size*sizeof(type))/delta*10e-9; //GB/s + const double algbw = (size*sizeof(type))/delta*1e-9; //GB/s const double bw = algbw * data_transfers_per_rank; if (comm_rank == 0) { std::cout << "[" << i << ":" << j << "] time: " << std::setw(12) << delta << " bw: " << std::setw(12) << bw << '\n'; From e7262fc2d698e68441d0deaf303bcfa84d867061 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:03:39 +0200 Subject: [PATCH 4/8] plot script and data --- .../run/data/job_n_00004_N_0001_TPN_4.out | 295 ++ .../run/data/job_n_00004_N_0004_TPN_1.out | 295 ++ .../run/data/job_n_00016_N_0004_TPN_4.out | 343 ++ .../run/data/job_n_00016_N_0016_TPN_1.out | 343 ++ .../run/data/job_n_00032_N_0008_TPN_4.out | 407 ++ .../run/data/job_n_00032_N_0032_TPN_1.out | 407 ++ .../run/data/job_n_00064_N_0016_TPN_4.out | 535 ++ .../run/data/job_n_00064_N_0064_TPN_1.out | 535 ++ .../run/data/job_n_00128_N_0032_TPN_4.out | 791 +++ .../run/data/job_n_00128_N_0128_TPN_1.out | 791 +++ .../run/data/job_n_00256_N_0064_TPN_4.out | 1303 +++++ .../run/data/job_n_00256_N_0256_TPN_1.out | 1303 +++++ .../run/data/job_n_00512_N_0128_TPN_4.out | 2327 +++++++++ .../run/data/job_n_01024_N_0256_TPN_4.out | 4375 +++++++++++++++++ allreduce-perf/mpi-cpp/run/plot.py | 401 +- .../mpi-cpp/run/plots/device_memory.svg | 2199 +++++++++ .../mpi-cpp/run/plots/device_memory_1.svg | 1631 ++++++ .../mpi-cpp/run/plots/device_memory_1_pad.svg | 1701 +++++++ .../run/plots/device_memory_1_pad_startup.svg | 1801 +++++++ .../run/plots/device_memory_1_startup.svg | 1731 +++++++ .../mpi-cpp/run/plots/device_memory_4.svg | 1782 +++++++ .../mpi-cpp/run/plots/device_memory_4_pad.svg | 1848 +++++++ .../run/plots/device_memory_4_pad_startup.svg | 1965 ++++++++ .../run/plots/device_memory_4_startup.svg | 1903 +++++++ .../run/plots/device_memory_startup.svg | 2574 ++++++++++ 25 files changed, 33453 insertions(+), 133 deletions(-) create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg create mode 100644 allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out new file mode 100644 index 0000000..3631ea0 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out @@ -0,0 +1,295 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid005332 +Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.348798 +[0:0] time: 0.103512 bw: 62.2384 +[0:1] time: 0.0975796 bw: 66.0225 +[0:2] time: 0.0976147 bw: 65.9988 +[0:3] time: 0.0974713 bw: 66.0959 +[0:4] time: 0.097489 bw: 66.0839 +[1:0] time: 0.103568 bw: 62.2051 +[1:1] time: 0.0976669 bw: 65.9635 +[1:2] time: 0.0975045 bw: 66.0734 +[1:3] time: 0.0975854 bw: 66.0186 +[1:4] time: 0.0976583 bw: 65.9693 +[2:0] time: 0.103796 bw: 62.0685 +[2:1] time: 0.0977194 bw: 65.928 +[2:2] time: 0.0977674 bw: 65.8957 +[2:3] time: 0.0973601 bw: 66.1714 +[2:4] time: 0.097463 bw: 66.1015 +[3:0] time: 0.105657 bw: 60.9749 +[3:1] time: 0.0974052 bw: 66.1407 +[3:2] time: 0.0975839 bw: 66.0196 +[3:3] time: 0.0975276 bw: 66.0577 +[3:4] time: 0.0976227 bw: 65.9934 +[4:0] time: 0.10359 bw: 62.1919 +[4:1] time: 0.0975199 bw: 66.0629 +[4:2] time: 0.0975719 bw: 66.0277 +[4:3] time: 0.0975666 bw: 66.0313 +[4:4] time: 0.0976924 bw: 65.9463 +[5:0] time: 0.107909 bw: 59.7028 +[5:1] time: 0.0975342 bw: 66.0533 +[5:2] time: 0.0976476 bw: 65.9766 +[5:3] time: 0.0975732 bw: 66.0269 +[5:4] time: 0.0974735 bw: 66.0944 +[6:0] time: 0.103833 bw: 62.0464 +[6:1] time: 0.0978823 bw: 65.8183 +[6:2] time: 0.0979781 bw: 65.754 +[6:3] time: 0.097787 bw: 65.8825 +[6:4] time: 0.0977631 bw: 65.8986 +[7:0] time: 0.10384 bw: 62.0418 +[7:1] time: 0.0977532 bw: 65.9053 +[7:2] time: 0.0977929 bw: 65.8785 +[7:3] time: 0.0979134 bw: 65.7975 +[7:4] time: 0.0978088 bw: 65.8678 +[8:0] time: 0.108783 bw: 59.2231 +[8:1] time: 0.097989 bw: 65.7466 +[8:2] time: 0.097793 bw: 65.8785 +[8:3] time: 0.0979342 bw: 65.7834 +[8:4] time: 0.0977989 bw: 65.8744 +[9:0] time: 0.1039 bw: 62.0065 +[9:1] time: 0.0979841 bw: 65.75 +[9:2] time: 0.0979718 bw: 65.7582 +[9:3] time: 0.0981458 bw: 65.6417 +[9:4] time: 0.0978096 bw: 65.8672 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.0976926 +time0: 0.104839 +bw: 65.9464 +bw0: 61.4699 +======================================= + +Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid005332 +Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 14.952 +[0:0] time: 14.9227 bw: 0.431722 +[0:1] time: 0.0976961 bw: 65.9438 +[0:2] time: 0.0975225 bw: 66.0612 +[0:3] time: 0.0977217 bw: 65.9265 +[0:4] time: 0.0976766 bw: 65.9569 +[1:0] time: 14.9305 bw: 0.431495 +[1:1] time: 0.0975571 bw: 66.0377 +[1:2] time: 0.0976218 bw: 65.994 +[1:3] time: 0.0977372 bw: 65.9161 +[1:4] time: 0.0975366 bw: 66.0517 +[2:0] time: 14.9384 bw: 0.431268 +[2:1] time: 0.0972738 bw: 66.23 +[2:2] time: 0.097445 bw: 66.1137 +[2:3] time: 0.0974536 bw: 66.1079 +[2:4] time: 0.0974845 bw: 66.0869 +[3:0] time: 14.9404 bw: 0.431211 +[3:1] time: 0.0973405 bw: 66.1847 +[3:2] time: 0.0974357 bw: 66.1201 +[3:3] time: 0.0974774 bw: 66.0917 +[3:4] time: 0.0974584 bw: 66.1046 +[4:0] time: 14.968 bw: 0.430415 +[4:1] time: 0.0975701 bw: 66.0289 +[4:2] time: 0.0974869 bw: 66.0853 +[4:3] time: 0.0976788 bw: 65.9555 +[4:4] time: 0.0976532 bw: 65.9728 +[5:0] time: 14.9486 bw: 0.430973 +[5:1] time: 0.0972953 bw: 66.2154 +[5:2] time: 0.0973075 bw: 66.2071 +[5:3] time: 0.0971813 bw: 66.2931 +[5:4] time: 0.0972392 bw: 66.2537 +[6:0] time: 14.9752 bw: 0.430208 +[6:1] time: 0.0976714 bw: 65.9605 +[6:2] time: 0.0976596 bw: 65.9684 +[6:3] time: 0.0976185 bw: 65.9962 +[6:4] time: 0.0977148 bw: 65.9312 +[7:0] time: 14.9511 bw: 0.430901 +[7:1] time: 0.0975293 bw: 66.0566 +[7:2] time: 0.0976393 bw: 65.9822 +[7:3] time: 0.0975174 bw: 66.0646 +[7:4] time: 0.0977363 bw: 65.9167 +[8:0] time: 14.9418 bw: 0.431169 +[8:1] time: 0.0974611 bw: 66.1028 +[8:2] time: 0.0976367 bw: 65.9839 +[8:3] time: 0.0974381 bw: 66.1184 +[8:4] time: 0.0975926 bw: 66.0137 +[9:0] time: 14.9554 bw: 0.430777 +[9:1] time: 0.0976213 bw: 65.9943 +[9:2] time: 0.0976296 bw: 65.9887 +[9:3] time: 0.0975525 bw: 66.0409 +[9:4] time: 0.0975892 bw: 66.016 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.0975365 +time0: 14.9472 +bw: 66.0519 +bw0: 0.431014 +======================================= + +Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid005332 +Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.0550143 +[0:0] time: 0.015113 bw: 426.287 +[0:1] time: 0.0150717 bw: 427.454 +[0:2] time: 0.0150779 bw: 427.276 +[0:3] time: 0.0150775 bw: 427.29 +[0:4] time: 0.0150858 bw: 427.055 +[1:0] time: 0.0152073 bw: 423.641 +[1:1] time: 0.0150675 bw: 427.573 +[1:2] time: 0.0150675 bw: 427.572 +[1:3] time: 0.0150641 bw: 427.669 +[1:4] time: 0.0150662 bw: 427.61 +[2:0] time: 0.0152628 bw: 422.102 +[2:1] time: 0.0150829 bw: 427.137 +[2:2] time: 0.0150693 bw: 427.522 +[2:3] time: 0.0150759 bw: 427.334 +[2:4] time: 0.0150794 bw: 427.235 +[3:0] time: 0.0152469 bw: 422.543 +[3:1] time: 0.0150704 bw: 427.491 +[3:2] time: 0.0150789 bw: 427.25 +[3:3] time: 0.0150714 bw: 427.461 +[3:4] time: 0.0150602 bw: 427.78 +[4:0] time: 0.0152731 bw: 421.817 +[4:1] time: 0.0150689 bw: 427.533 +[4:2] time: 0.0150675 bw: 427.572 +[4:3] time: 0.0150681 bw: 427.555 +[4:4] time: 0.0150679 bw: 427.561 +[5:0] time: 0.0152159 bw: 423.402 +[5:1] time: 0.0150591 bw: 427.81 +[5:2] time: 0.0150599 bw: 427.788 +[5:3] time: 0.0150616 bw: 427.74 +[5:4] time: 0.0150604 bw: 427.773 +[6:0] time: 0.0151478 bw: 425.305 +[6:1] time: 0.0150695 bw: 427.517 +[6:2] time: 0.015073 bw: 427.418 +[6:3] time: 0.015065 bw: 427.645 +[6:4] time: 0.0150675 bw: 427.571 +[7:0] time: 0.0152497 bw: 422.465 +[7:1] time: 0.0150627 bw: 427.709 +[7:2] time: 0.0150674 bw: 427.577 +[7:3] time: 0.0150634 bw: 427.689 +[7:4] time: 0.0150681 bw: 427.555 +[8:0] time: 0.0237532 bw: 271.225 +[8:1] time: 0.0150711 bw: 427.471 +[8:2] time: 0.0150714 bw: 427.462 +[8:3] time: 0.0150699 bw: 427.503 +[8:4] time: 0.0150675 bw: 427.573 +[9:0] time: 0.015248 bw: 422.512 +[9:1] time: 0.0150674 bw: 427.575 +[9:2] time: 0.0150603 bw: 427.777 +[9:3] time: 0.0150606 bw: 427.769 +[9:4] time: 0.0150619 bw: 427.732 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.0150687 +time0: 0.0160718 +bw: 427.54 +bw0: 408.13 +======================================= + +Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid005332 +Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.119388 +[0:0] time: 0.0155304 bw: 414.828 +[0:1] time: 0.0154832 bw: 416.093 +[0:2] time: 0.015483 bw: 416.098 +[0:3] time: 0.015481 bw: 416.152 +[0:4] time: 0.0154971 bw: 415.72 +[1:0] time: 0.0156326 bw: 412.116 +[1:1] time: 0.0154866 bw: 416.001 +[1:2] time: 0.0154855 bw: 416.032 +[1:3] time: 0.0154784 bw: 416.221 +[1:4] time: 0.0154824 bw: 416.115 +[2:0] time: 0.0156866 bw: 410.697 +[2:1] time: 0.0154877 bw: 415.972 +[2:2] time: 0.0154809 bw: 416.156 +[2:3] time: 0.0154839 bw: 416.073 +[2:4] time: 0.0154835 bw: 416.085 +[3:0] time: 0.0155252 bw: 414.968 +[3:1] time: 0.0154827 bw: 416.106 +[3:2] time: 0.0155011 bw: 415.612 +[3:3] time: 0.0154918 bw: 415.862 +[3:4] time: 0.0154768 bw: 416.264 +[4:0] time: 0.0156413 bw: 411.888 +[4:1] time: 0.0154795 bw: 416.192 +[4:2] time: 0.0154801 bw: 416.176 +[4:3] time: 0.0154804 bw: 416.169 +[4:4] time: 0.0154793 bw: 416.198 +[5:0] time: 0.0156836 bw: 410.777 +[5:1] time: 0.0154719 bw: 416.396 +[5:2] time: 0.0154743 bw: 416.334 +[5:3] time: 0.0154743 bw: 416.331 +[5:4] time: 0.015474 bw: 416.341 +[6:0] time: 0.0156298 bw: 412.19 +[6:1] time: 0.0154789 bw: 416.21 +[6:2] time: 0.0154836 bw: 416.082 +[6:3] time: 0.0154804 bw: 416.168 +[6:4] time: 0.0154785 bw: 416.218 +[7:0] time: 0.0157185 bw: 409.863 +[7:1] time: 0.0154786 bw: 416.216 +[7:2] time: 0.015481 bw: 416.152 +[7:3] time: 0.0154747 bw: 416.322 +[7:4] time: 0.0154832 bw: 416.093 +[8:0] time: 0.0155806 bw: 413.492 +[8:1] time: 0.015477 bw: 416.259 +[8:2] time: 0.0154789 bw: 416.208 +[8:3] time: 0.0154796 bw: 416.189 +[8:4] time: 0.0154765 bw: 416.274 +[9:0] time: 0.021907 bw: 294.082 +[9:1] time: 0.0154852 bw: 416.039 +[9:2] time: 0.0154728 bw: 416.372 +[9:3] time: 0.0154778 bw: 416.239 +[9:4] time: 0.0154743 bw: 416.334 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.015481 +time0: 0.0162536 +bw: 416.152 +bw0: 400.49 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out new file mode 100644 index 0000000..6150814 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out @@ -0,0 +1,295 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[005352,005356,005360,005362] +Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.381335 +[0:0] time: 0.344453 bw: 18.7034 +[0:1] time: 0.323566 bw: 19.9108 +[0:2] time: 0.323661 bw: 19.9049 +[0:3] time: 0.32359 bw: 19.9093 +[0:4] time: 0.32369 bw: 19.9031 +[1:0] time: 0.344244 bw: 18.7148 +[1:1] time: 0.324131 bw: 19.8761 +[1:2] time: 0.323547 bw: 19.9119 +[1:3] time: 0.323755 bw: 19.8992 +[1:4] time: 0.323589 bw: 19.9094 +[2:0] time: 0.344449 bw: 18.7036 +[2:1] time: 0.323609 bw: 19.9081 +[2:2] time: 0.323543 bw: 19.9122 +[2:3] time: 0.323941 bw: 19.8877 +[2:4] time: 0.32369 bw: 19.9032 +[3:0] time: 0.344098 bw: 18.7227 +[3:1] time: 0.324104 bw: 19.8777 +[3:2] time: 0.324189 bw: 19.8725 +[3:3] time: 0.324084 bw: 19.8789 +[3:4] time: 0.32463 bw: 19.8455 +[4:0] time: 0.344621 bw: 18.6943 +[4:1] time: 0.324155 bw: 19.8746 +[4:2] time: 0.324157 bw: 19.8745 +[4:3] time: 0.323858 bw: 19.8928 +[4:4] time: 0.323788 bw: 19.8972 +[5:0] time: 0.343632 bw: 18.7481 +[5:1] time: 0.323413 bw: 19.9202 +[5:2] time: 0.323192 bw: 19.9338 +[5:3] time: 0.323099 bw: 19.9396 +[5:4] time: 0.323184 bw: 19.9343 +[6:0] time: 0.344311 bw: 18.7111 +[6:1] time: 0.32341 bw: 19.9204 +[6:2] time: 0.32358 bw: 19.9099 +[6:3] time: 0.323432 bw: 19.919 +[6:4] time: 0.32353 bw: 19.913 +[7:0] time: 0.344217 bw: 18.7163 +[7:1] time: 0.323508 bw: 19.9144 +[7:2] time: 0.32385 bw: 19.8933 +[7:3] time: 0.323944 bw: 19.8876 +[7:4] time: 0.323675 bw: 19.904 +[8:0] time: 0.344494 bw: 18.7012 +[8:1] time: 0.323187 bw: 19.9341 +[8:2] time: 0.323111 bw: 19.9388 +[8:3] time: 0.323283 bw: 19.9282 +[8:4] time: 0.323258 bw: 19.9298 +[9:0] time: 0.343387 bw: 18.7615 +[9:1] time: 0.323274 bw: 19.9288 +[9:2] time: 0.323297 bw: 19.9273 +[9:3] time: 0.323303 bw: 19.927 +[9:4] time: 0.323448 bw: 19.9181 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.323631 +time0: 0.344191 +bw: 19.9068 +bw0: 18.7177 +======================================= + +Slurm Job Hostlist: nid[005352,005356,005360,005362] +Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.27841 +[0:0] time: 1.26066 bw: 5.11036 +[0:1] time: 1.25103 bw: 5.14974 +[0:2] time: 1.24951 bw: 5.15599 +[0:3] time: 1.25101 bw: 5.14981 +[0:4] time: 1.25126 bw: 5.14876 +[1:0] time: 1.27675 bw: 5.04596 +[1:1] time: 1.25492 bw: 5.13374 +[1:2] time: 1.25404 bw: 5.13737 +[1:3] time: 1.25667 bw: 5.1266 +[1:4] time: 1.25445 bw: 5.13569 +[2:0] time: 1.27783 bw: 5.04173 +[2:1] time: 1.25564 bw: 5.13079 +[2:2] time: 1.2561 bw: 5.12894 +[2:3] time: 1.25683 bw: 5.12596 +[2:4] time: 1.25325 bw: 5.14061 +[3:0] time: 1.27916 bw: 5.03646 +[3:1] time: 1.25797 bw: 5.12129 +[3:2] time: 1.25733 bw: 5.12391 +[3:3] time: 1.2562 bw: 5.12851 +[3:4] time: 1.25754 bw: 5.12307 +[4:0] time: 1.27943 bw: 5.03542 +[4:1] time: 1.25601 bw: 5.1293 +[4:2] time: 1.25592 bw: 5.12966 +[4:3] time: 1.25932 bw: 5.11583 +[4:4] time: 1.25727 bw: 5.12414 +[5:0] time: 1.28368 bw: 5.01874 +[5:1] time: 1.26102 bw: 5.10893 +[5:2] time: 1.2602 bw: 5.11226 +[5:3] time: 1.26148 bw: 5.10704 +[5:4] time: 1.26043 bw: 5.11132 +[6:0] time: 1.28472 bw: 5.01467 +[6:1] time: 1.26123 bw: 5.10807 +[6:2] time: 1.2631 bw: 5.10049 +[6:3] time: 1.26235 bw: 5.10355 +[6:4] time: 1.26164 bw: 5.10643 +[7:0] time: 1.28281 bw: 5.02215 +[7:1] time: 1.26126 bw: 5.10796 +[7:2] time: 1.26156 bw: 5.10673 +[7:3] time: 1.26152 bw: 5.1069 +[7:4] time: 1.26188 bw: 5.10542 +[8:0] time: 1.28496 bw: 5.01374 +[8:1] time: 1.26111 bw: 5.10854 +[8:2] time: 1.26141 bw: 5.10735 +[8:3] time: 1.26178 bw: 5.10585 +[8:4] time: 1.26287 bw: 5.10143 +[9:0] time: 1.28522 bw: 5.01274 +[9:1] time: 1.26394 bw: 5.09713 +[9:2] time: 1.26422 bw: 5.096 +[9:3] time: 1.26386 bw: 5.09745 +[9:4] time: 1.26429 bw: 5.09572 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.25858 +time0: 1.27952 +bw: 5.11886 +bw0: 5.0352 +======================================= + +Slurm Job Hostlist: nid[005352,005356,005360,005362] +Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.0489155 +[0:0] time: 0.0156881 bw: 410.658 +[0:1] time: 0.0150684 bw: 427.548 +[0:2] time: 0.0177612 bw: 362.726 +[0:3] time: 0.0166887 bw: 386.036 +[0:4] time: 0.0644491 bw: 99.9618 +[1:0] time: 0.0157591 bw: 408.807 +[1:1] time: 0.0150642 bw: 427.668 +[1:2] time: 0.015068 bw: 427.559 +[1:3] time: 0.0150626 bw: 427.711 +[1:4] time: 0.0150714 bw: 427.462 +[2:0] time: 0.0159035 bw: 405.098 +[2:1] time: 0.015067 bw: 427.588 +[2:2] time: 0.0150681 bw: 427.557 +[2:3] time: 0.023429 bw: 274.978 +[2:4] time: 0.0150763 bw: 427.324 +[3:0] time: 0.0156805 bw: 410.857 +[3:1] time: 0.0150795 bw: 427.233 +[3:2] time: 0.0150731 bw: 427.414 +[3:3] time: 0.0150706 bw: 427.486 +[3:4] time: 0.01507 bw: 427.501 +[4:0] time: 0.0158737 bw: 405.857 +[4:1] time: 0.0150664 bw: 427.604 +[4:2] time: 0.0150659 bw: 427.617 +[4:3] time: 0.0150626 bw: 427.712 +[4:4] time: 0.0150635 bw: 427.685 +[5:0] time: 0.0158658 bw: 406.058 +[5:1] time: 0.0150823 bw: 427.153 +[5:2] time: 0.0150787 bw: 427.255 +[5:3] time: 0.0150716 bw: 427.457 +[5:4] time: 0.0150721 bw: 427.441 +[6:0] time: 0.0158779 bw: 405.749 +[6:1] time: 0.0150651 bw: 427.639 +[6:2] time: 0.0150648 bw: 427.648 +[6:3] time: 0.0150615 bw: 427.743 +[6:4] time: 0.0150632 bw: 427.695 +[7:0] time: 0.0158243 bw: 407.123 +[7:1] time: 0.0150658 bw: 427.622 +[7:2] time: 0.0150722 bw: 427.441 +[7:3] time: 0.0150726 bw: 427.429 +[7:4] time: 0.015066 bw: 427.616 +[8:0] time: 0.0159407 bw: 404.151 +[8:1] time: 0.0150754 bw: 427.348 +[8:2] time: 0.0150726 bw: 427.428 +[8:3] time: 0.0150671 bw: 427.583 +[8:4] time: 0.0150684 bw: 427.547 +[9:0] time: 0.0161478 bw: 398.968 +[9:1] time: 0.0150664 bw: 427.605 +[9:2] time: 0.0150707 bw: 427.482 +[9:3] time: 0.01507 bw: 427.501 +[9:4] time: 0.0150696 bw: 427.514 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.0166205 +time0: 0.0158562 +bw: 412.863 +bw0: 406.333 +======================================= + +Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[005352,005356,005360,005362] +Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.048395 +[0:0] time: 0.0160341 bw: 401.796 +[0:1] time: 0.0170961 bw: 376.838 +[0:2] time: 0.0160611 bw: 401.123 +[0:3] time: 0.0154836 bw: 416.082 +[0:4] time: 0.0154749 bw: 416.315 +[1:0] time: 0.0162892 bw: 395.504 +[1:1] time: 0.0154839 bw: 416.074 +[1:2] time: 0.0154867 bw: 415.998 +[1:3] time: 0.015489 bw: 415.936 +[1:4] time: 0.0154798 bw: 416.184 +[2:0] time: 0.0161949 bw: 397.807 +[2:1] time: 0.0154822 bw: 416.119 +[2:2] time: 0.0154785 bw: 416.22 +[2:3] time: 0.0154741 bw: 416.339 +[2:4] time: 0.0154698 bw: 416.452 +[3:0] time: 0.0162204 bw: 397.181 +[3:1] time: 0.0154798 bw: 416.186 +[3:2] time: 0.0154841 bw: 416.069 +[3:3] time: 0.015489 bw: 415.938 +[3:4] time: 0.0154909 bw: 415.887 +[4:0] time: 0.0161888 bw: 397.958 +[4:1] time: 0.0154772 bw: 416.255 +[4:2] time: 0.0154811 bw: 416.15 +[4:3] time: 0.0154731 bw: 416.364 +[4:4] time: 0.015482 bw: 416.125 +[5:0] time: 0.0162015 bw: 397.645 +[5:1] time: 0.0154887 bw: 415.946 +[5:2] time: 0.0154923 bw: 415.849 +[5:3] time: 0.0154873 bw: 415.983 +[5:4] time: 0.0154927 bw: 415.837 +[6:0] time: 0.0161586 bw: 398.702 +[6:1] time: 0.0154876 bw: 415.976 +[6:2] time: 0.0154866 bw: 416.001 +[6:3] time: 0.0154885 bw: 415.952 +[6:4] time: 0.0154811 bw: 416.149 +[7:0] time: 0.0161795 bw: 398.187 +[7:1] time: 0.0154785 bw: 416.219 +[7:2] time: 0.0154822 bw: 416.12 +[7:3] time: 0.0154799 bw: 416.181 +[7:4] time: 0.01548 bw: 416.178 +[8:0] time: 0.0161812 bw: 398.145 +[8:1] time: 0.0154757 bw: 416.295 +[8:2] time: 0.0154813 bw: 416.143 +[8:3] time: 0.0154779 bw: 416.235 +[8:4] time: 0.0154783 bw: 416.226 +[9:0] time: 0.0161852 bw: 398.045 +[9:1] time: 0.0154781 bw: 416.23 +[9:2] time: 0.015479 bw: 416.205 +[9:3] time: 0.015484 bw: 416.07 +[9:4] time: 0.0154837 bw: 416.081 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.015537 +time0: 0.0161833 +bw: 414.763 +bw0: 398.097 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out new file mode 100644 index 0000000..e748bd8 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out @@ -0,0 +1,343 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004328,004330,004350,004352] +Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.504726 +[0:0] time: 0.241972 bw: 33.2809 +[0:1] time: 0.208666 bw: 38.593 +[0:2] time: 0.209733 bw: 38.3968 +[0:3] time: 0.208332 bw: 38.6549 +[0:4] time: 0.208196 bw: 38.6803 +[1:0] time: 0.241162 bw: 33.3928 +[1:1] time: 0.208529 bw: 38.6185 +[1:2] time: 0.208481 bw: 38.6274 +[1:3] time: 0.208321 bw: 38.6571 +[1:4] time: 0.208221 bw: 38.6756 +[2:0] time: 0.260553 bw: 30.9075 +[2:1] time: 0.208279 bw: 38.6648 +[2:2] time: 0.208373 bw: 38.6474 +[2:3] time: 0.208177 bw: 38.6838 +[2:4] time: 0.208313 bw: 38.6586 +[3:0] time: 0.258582 bw: 31.1431 +[3:1] time: 0.209427 bw: 38.4529 +[3:2] time: 0.208713 bw: 38.5844 +[3:3] time: 0.208182 bw: 38.6828 +[3:4] time: 0.20828 bw: 38.6646 +[4:0] time: 0.258232 bw: 31.1853 +[4:1] time: 0.209434 bw: 38.4516 +[4:2] time: 0.208277 bw: 38.6652 +[4:3] time: 0.210214 bw: 38.3089 +[4:4] time: 0.210498 bw: 38.2573 +[5:0] time: 0.258393 bw: 31.166 +[5:1] time: 0.20952 bw: 38.4358 +[5:2] time: 0.208331 bw: 38.6551 +[5:3] time: 0.208451 bw: 38.633 +[5:4] time: 0.208236 bw: 38.6727 +[6:0] time: 0.258274 bw: 31.1803 +[6:1] time: 0.209245 bw: 38.4864 +[6:2] time: 0.209108 bw: 38.5115 +[6:3] time: 0.208826 bw: 38.5635 +[6:4] time: 0.208938 bw: 38.5428 +[7:0] time: 0.25848 bw: 31.1555 +[7:1] time: 0.209007 bw: 38.5301 +[7:2] time: 0.210524 bw: 38.2525 +[7:3] time: 0.208789 bw: 38.5704 +[7:4] time: 0.208905 bw: 38.549 +[8:0] time: 0.260194 bw: 30.9502 +[8:1] time: 0.208724 bw: 38.5823 +[8:2] time: 0.208796 bw: 38.569 +[8:3] time: 0.208742 bw: 38.5789 +[8:4] time: 0.209112 bw: 38.5108 +[9:0] time: 0.261271 bw: 30.8226 +[9:1] time: 0.208765 bw: 38.5748 +[9:2] time: 0.208802 bw: 38.5679 +[9:3] time: 0.209494 bw: 38.4406 +[9:4] time: 0.209783 bw: 38.3877 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.208868 +time0: 0.255711 +bw: 38.556 +bw0: 31.5184 +======================================= + +Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004328,004330,004350,004352] +Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.3198 +[0:0] time: 15.2332 bw: 0.528653 +[0:1] time: 0.410208 bw: 19.6317 +[0:2] time: 0.409929 bw: 19.645 +[0:3] time: 0.409678 bw: 19.657 +[0:4] time: 0.409146 bw: 19.6826 +[1:0] time: 15.2707 bw: 0.527355 +[1:1] time: 0.409097 bw: 19.685 +[1:2] time: 0.412953 bw: 19.5011 +[1:3] time: 0.413536 bw: 19.4737 +[1:4] time: 0.410126 bw: 19.6356 +[2:0] time: 15.2757 bw: 0.527182 +[2:1] time: 0.409817 bw: 19.6504 +[2:2] time: 0.40926 bw: 19.6771 +[2:3] time: 0.408736 bw: 19.7023 +[2:4] time: 0.409518 bw: 19.6647 +[3:0] time: 15.2834 bw: 0.526916 +[3:1] time: 0.409126 bw: 19.6836 +[3:2] time: 0.409713 bw: 19.6554 +[3:3] time: 0.410242 bw: 19.63 +[3:4] time: 0.409805 bw: 19.6509 +[4:0] time: 15.2516 bw: 0.528013 +[4:1] time: 0.409713 bw: 19.6554 +[4:2] time: 0.409568 bw: 19.6623 +[4:3] time: 0.409725 bw: 19.6548 +[4:4] time: 0.410053 bw: 19.6391 +[5:0] time: 15.3014 bw: 0.526295 +[5:1] time: 0.409516 bw: 19.6649 +[5:2] time: 0.409567 bw: 19.6624 +[5:3] time: 0.409492 bw: 19.666 +[5:4] time: 0.409715 bw: 19.6553 +[6:0] time: 15.2714 bw: 0.52733 +[6:1] time: 0.41017 bw: 19.6335 +[6:2] time: 0.410395 bw: 19.6227 +[6:3] time: 0.410084 bw: 19.6376 +[6:4] time: 0.409973 bw: 19.6429 +[7:0] time: 15.344 bw: 0.524834 +[7:1] time: 0.410583 bw: 19.6137 +[7:2] time: 0.410555 bw: 19.6151 +[7:3] time: 0.410706 bw: 19.6079 +[7:4] time: 0.410923 bw: 19.5975 +[8:0] time: 15.2429 bw: 0.528317 +[8:1] time: 0.410859 bw: 19.6005 +[8:2] time: 0.410402 bw: 19.6224 +[8:3] time: 0.41054 bw: 19.6158 +[8:4] time: 0.410281 bw: 19.6282 +[9:0] time: 15.2713 bw: 0.527332 +[9:1] time: 0.409985 bw: 19.6423 +[9:2] time: 0.410118 bw: 19.636 +[9:3] time: 0.410345 bw: 19.6251 +[9:4] time: 0.409929 bw: 19.645 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.410102 +time0: 15.2746 +bw: 19.6368 +bw0: 0.527223 +======================================= + +Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004328,004330,004350,004352] +Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.169212 +[0:0] time: 0.0638838 bw: 126.058 +[0:1] time: 0.0638435 bw: 126.138 +[0:2] time: 0.0638601 bw: 126.105 +[0:3] time: 0.0638798 bw: 126.066 +[0:4] time: 0.0638359 bw: 126.153 +[1:0] time: 0.0639934 bw: 125.842 +[1:1] time: 0.0638425 bw: 126.14 +[1:2] time: 0.0638436 bw: 126.137 +[1:3] time: 0.0638445 bw: 126.136 +[1:4] time: 0.0638368 bw: 126.151 +[2:0] time: 0.0639946 bw: 125.84 +[2:1] time: 0.0638369 bw: 126.151 +[2:2] time: 0.0638402 bw: 126.144 +[2:3] time: 0.0638374 bw: 126.15 +[2:4] time: 0.0638435 bw: 126.138 +[3:0] time: 0.0639794 bw: 125.87 +[3:1] time: 0.0638491 bw: 126.126 +[3:2] time: 0.0638452 bw: 126.134 +[3:3] time: 0.0638336 bw: 126.157 +[3:4] time: 0.0638349 bw: 126.155 +[4:0] time: 0.0640293 bw: 125.772 +[4:1] time: 0.0638316 bw: 126.161 +[4:2] time: 0.0638163 bw: 126.191 +[4:3] time: 0.0638206 bw: 126.183 +[4:4] time: 0.0638374 bw: 126.15 +[5:0] time: 0.0639385 bw: 125.95 +[5:1] time: 0.0638342 bw: 126.156 +[5:2] time: 0.0638322 bw: 126.16 +[5:3] time: 0.0638383 bw: 126.148 +[5:4] time: 0.0638489 bw: 126.127 +[6:0] time: 0.0639692 bw: 125.89 +[6:1] time: 0.0638361 bw: 126.152 +[6:2] time: 0.0638218 bw: 126.18 +[6:3] time: 0.0638154 bw: 126.193 +[6:4] time: 0.0638323 bw: 126.16 +[7:0] time: 0.0711399 bw: 113.2 +[7:1] time: 0.0638318 bw: 126.161 +[7:2] time: 0.0638399 bw: 126.145 +[7:3] time: 0.0638385 bw: 126.147 +[7:4] time: 0.0638398 bw: 126.145 +[8:0] time: 0.0640239 bw: 125.782 +[8:1] time: 0.0638172 bw: 126.189 +[8:2] time: 0.0638161 bw: 126.192 +[8:3] time: 0.0638202 bw: 126.184 +[8:4] time: 0.0638288 bw: 126.167 +[9:0] time: 0.0639901 bw: 125.849 +[9:1] time: 0.0638294 bw: 126.165 +[9:2] time: 0.0638373 bw: 126.15 +[9:3] time: 0.0638898 bw: 126.046 +[9:4] time: 0.0667015 bw: 120.733 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.0639091 +time0: 0.0646942 +bw: 126.014 +bw0: 124.605 +======================================= + +Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004328,004330,004350,004352] +Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.172451 +[0:0] time: 0.0659321 bw: 122.142 +[0:1] time: 0.0658436 bw: 122.306 +[0:2] time: 0.0658629 bw: 122.27 +[0:3] time: 0.0658461 bw: 122.301 +[0:4] time: 0.0658377 bw: 122.317 +[1:0] time: 0.066035 bw: 121.951 +[1:1] time: 0.0658166 bw: 122.356 +[1:2] time: 0.0658196 bw: 122.351 +[1:3] time: 0.0658155 bw: 122.358 +[1:4] time: 0.065819 bw: 122.352 +[2:0] time: 0.0660486 bw: 121.926 +[2:1] time: 0.0658185 bw: 122.353 +[2:2] time: 0.0658158 bw: 122.358 +[2:3] time: 0.0658287 bw: 122.334 +[2:4] time: 0.0658313 bw: 122.329 +[3:0] time: 0.0660506 bw: 121.923 +[3:1] time: 0.065818 bw: 122.354 +[3:2] time: 0.0658214 bw: 122.347 +[3:3] time: 0.0658232 bw: 122.344 +[3:4] time: 0.0658267 bw: 122.337 +[4:0] time: 0.0660649 bw: 121.896 +[4:1] time: 0.0658102 bw: 122.368 +[4:2] time: 0.0658166 bw: 122.356 +[4:3] time: 0.0658205 bw: 122.349 +[4:4] time: 0.0658206 bw: 122.349 +[5:0] time: 0.066022 bw: 121.975 +[5:1] time: 0.0658203 bw: 122.349 +[5:2] time: 0.0658119 bw: 122.365 +[5:3] time: 0.0658274 bw: 122.336 +[5:4] time: 0.0658182 bw: 122.353 +[6:0] time: 0.0666281 bw: 120.866 +[6:1] time: 0.0658188 bw: 122.352 +[6:2] time: 0.0658227 bw: 122.345 +[6:3] time: 0.0658179 bw: 122.354 +[6:4] time: 0.065816 bw: 122.357 +[7:0] time: 0.0660048 bw: 122.007 +[7:1] time: 0.0658132 bw: 122.363 +[7:2] time: 0.0658203 bw: 122.349 +[7:3] time: 0.0658167 bw: 122.356 +[7:4] time: 0.065806 bw: 122.376 +[8:0] time: 0.066024 bw: 121.972 +[8:1] time: 0.065812 bw: 122.365 +[8:2] time: 0.0658108 bw: 122.367 +[8:3] time: 0.0658188 bw: 122.352 +[8:4] time: 0.0658144 bw: 122.36 +[9:0] time: 0.0660467 bw: 121.93 +[9:1] time: 0.0658218 bw: 122.346 +[9:2] time: 0.0658211 bw: 122.348 +[9:3] time: 0.065812 bw: 122.365 +[9:4] time: 0.0658237 bw: 122.343 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.0658214 +time0: 0.0660857 +bw: 122.347 +bw0: 121.859 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out new file mode 100644 index 0000000..4e10319 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out @@ -0,0 +1,343 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] +Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.524098 +[0:0] time: 0.45217 bw: 17.8098 +[0:1] time: 0.431128 bw: 18.6791 +[0:2] time: 0.431328 bw: 18.6704 +[0:3] time: 0.431319 bw: 18.6708 +[0:4] time: 0.432948 bw: 18.6005 +[1:0] time: 0.452207 bw: 17.8084 +[1:1] time: 0.431044 bw: 18.6827 +[1:2] time: 0.431515 bw: 18.6623 +[1:3] time: 0.430952 bw: 18.6867 +[1:4] time: 0.432727 bw: 18.61 +[2:0] time: 0.45263 bw: 17.7917 +[2:1] time: 0.431193 bw: 18.6762 +[2:2] time: 0.431659 bw: 18.6561 +[2:3] time: 0.432598 bw: 18.6156 +[2:4] time: 0.431165 bw: 18.6775 +[3:0] time: 0.452535 bw: 17.7954 +[3:1] time: 0.430977 bw: 18.6856 +[3:2] time: 0.43096 bw: 18.6863 +[3:3] time: 0.43086 bw: 18.6907 +[3:4] time: 0.430864 bw: 18.6905 +[4:0] time: 0.45257 bw: 17.7941 +[4:1] time: 0.430953 bw: 18.6867 +[4:2] time: 0.430958 bw: 18.6864 +[4:3] time: 0.43102 bw: 18.6837 +[4:4] time: 0.430785 bw: 18.6939 +[5:0] time: 0.452741 bw: 17.7873 +[5:1] time: 0.431577 bw: 18.6596 +[5:2] time: 0.431073 bw: 18.6814 +[5:3] time: 0.430894 bw: 18.6892 +[5:4] time: 0.430903 bw: 18.6888 +[6:0] time: 0.452656 bw: 17.7907 +[6:1] time: 0.431624 bw: 18.6576 +[6:2] time: 0.431565 bw: 18.6602 +[6:3] time: 0.432193 bw: 18.633 +[6:4] time: 0.431477 bw: 18.6639 +[7:0] time: 0.452481 bw: 17.7976 +[7:1] time: 0.431624 bw: 18.6576 +[7:2] time: 0.431581 bw: 18.6594 +[7:3] time: 0.431617 bw: 18.6579 +[7:4] time: 0.431547 bw: 18.6609 +[8:0] time: 0.453368 bw: 17.7628 +[8:1] time: 0.431407 bw: 18.667 +[8:2] time: 0.431571 bw: 18.6599 +[8:3] time: 0.431661 bw: 18.656 +[8:4] time: 0.431576 bw: 18.6597 +[9:0] time: 0.453671 bw: 17.7509 +[9:1] time: 0.43157 bw: 18.6599 +[9:2] time: 0.431371 bw: 18.6685 +[9:3] time: 0.431705 bw: 18.6541 +[9:4] time: 0.431542 bw: 18.6611 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.431426 +time0: 0.452703 +bw: 18.6662 +bw0: 17.7889 +======================================= + +Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] +Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.53478 +[0:0] time: 1.51548 bw: 5.31388 +[0:1] time: 1.49732 bw: 5.37831 +[0:2] time: 1.50292 bw: 5.35828 +[0:3] time: 1.49913 bw: 5.37183 +[0:4] time: 1.49535 bw: 5.38541 +[1:0] time: 1.5503 bw: 5.19453 +[1:1] time: 1.4977 bw: 5.37694 +[1:2] time: 1.49808 bw: 5.37558 +[1:3] time: 1.49939 bw: 5.37089 +[1:4] time: 1.49805 bw: 5.37568 +[2:0] time: 1.556 bw: 5.17551 +[2:1] time: 1.5002 bw: 5.36798 +[2:2] time: 1.49921 bw: 5.37152 +[2:3] time: 1.4992 bw: 5.37158 +[2:4] time: 1.49764 bw: 5.37715 +[3:0] time: 1.55047 bw: 5.19396 +[3:1] time: 1.49872 bw: 5.3733 +[3:2] time: 1.49969 bw: 5.36981 +[3:3] time: 1.50234 bw: 5.36033 +[3:4] time: 1.50162 bw: 5.36293 +[4:0] time: 1.56192 bw: 5.15589 +[4:1] time: 1.50549 bw: 5.34915 +[4:2] time: 1.50628 bw: 5.34634 +[4:3] time: 1.50297 bw: 5.35811 +[4:4] time: 1.50391 bw: 5.35475 +[5:0] time: 1.55689 bw: 5.17255 +[5:1] time: 1.50599 bw: 5.34737 +[5:2] time: 1.50475 bw: 5.35176 +[5:3] time: 1.50298 bw: 5.35808 +[5:4] time: 1.50195 bw: 5.36173 +[6:0] time: 1.5636 bw: 5.15035 +[6:1] time: 1.50561 bw: 5.3487 +[6:2] time: 1.50612 bw: 5.34689 +[6:3] time: 1.50444 bw: 5.35286 +[6:4] time: 1.50356 bw: 5.356 +[7:0] time: 1.55807 bw: 5.16862 +[7:1] time: 1.50621 bw: 5.34657 +[7:2] time: 1.51077 bw: 5.33043 +[7:3] time: 1.51147 bw: 5.32798 +[7:4] time: 1.50719 bw: 5.34312 +[8:0] time: 1.56196 bw: 5.15575 +[8:1] time: 1.50816 bw: 5.33967 +[8:2] time: 1.50822 bw: 5.33944 +[8:3] time: 1.50661 bw: 5.34515 +[8:4] time: 1.50987 bw: 5.33361 +[9:0] time: 1.56375 bw: 5.14983 +[9:1] time: 1.51397 bw: 5.31917 +[9:2] time: 1.51274 bw: 5.32349 +[9:3] time: 1.5127 bw: 5.32364 +[9:4] time: 1.50884 bw: 5.33725 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.50393 +time0: 1.55384 +bw: 5.35472 +bw0: 5.18309 +======================================= + +Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] +Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.114812 +[0:0] time: 0.0814363 bw: 98.8879 +[0:1] time: 0.0637716 bw: 126.28 +[0:2] time: 0.0637575 bw: 126.308 +[0:3] time: 0.0637743 bw: 126.274 +[0:4] time: 0.063765 bw: 126.293 +[1:0] time: 0.0638736 bw: 126.078 +[1:1] time: 0.0637744 bw: 126.274 +[1:2] time: 0.0637744 bw: 126.274 +[1:3] time: 0.0637857 bw: 126.252 +[1:4] time: 0.0637804 bw: 126.262 +[2:0] time: 0.0639457 bw: 125.936 +[2:1] time: 0.0637546 bw: 126.314 +[2:2] time: 0.0637755 bw: 126.272 +[2:3] time: 0.063776 bw: 126.271 +[2:4] time: 0.0637746 bw: 126.274 +[3:0] time: 0.0640146 bw: 125.8 +[3:1] time: 0.0637866 bw: 126.25 +[3:2] time: 0.0637704 bw: 126.282 +[3:3] time: 0.0637741 bw: 126.275 +[3:4] time: 0.0637741 bw: 126.275 +[4:0] time: 0.0639243 bw: 125.978 +[4:1] time: 0.0637621 bw: 126.299 +[4:2] time: 0.0639204 bw: 125.986 +[4:3] time: 0.0637927 bw: 126.238 +[4:4] time: 0.0637901 bw: 126.243 +[5:0] time: 0.0638428 bw: 126.139 +[5:1] time: 0.0637724 bw: 126.278 +[5:2] time: 0.0637812 bw: 126.261 +[5:3] time: 0.0637823 bw: 126.258 +[5:4] time: 0.0637861 bw: 126.251 +[6:0] time: 0.0638597 bw: 126.106 +[6:1] time: 0.0637637 bw: 126.295 +[6:2] time: 0.063777 bw: 126.269 +[6:3] time: 0.0639618 bw: 125.904 +[6:4] time: 0.063799 bw: 126.226 +[7:0] time: 0.0638307 bw: 126.163 +[7:1] time: 0.0637793 bw: 126.264 +[7:2] time: 0.0637712 bw: 126.28 +[7:3] time: 0.0637853 bw: 126.253 +[7:4] time: 0.0637796 bw: 126.264 +[8:0] time: 0.0638549 bw: 126.115 +[8:1] time: 0.0637794 bw: 126.264 +[8:2] time: 0.0637726 bw: 126.278 +[8:3] time: 0.0637647 bw: 126.293 +[8:4] time: 0.0637743 bw: 126.274 +[9:0] time: 0.0638479 bw: 126.129 +[9:1] time: 0.0637783 bw: 126.266 +[9:2] time: 0.0637822 bw: 126.259 +[9:3] time: 0.0637631 bw: 126.297 +[9:4] time: 0.0637697 bw: 126.284 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.0637839 +time0: 0.065643 +bw: 126.255 +bw0: 123.333 +======================================= + +Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] +Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.107586 +[0:0] time: 0.0729256 bw: 110.429 +[0:1] time: 0.0706225 bw: 114.03 +[0:2] time: 0.0658227 bw: 122.345 +[0:3] time: 0.0657996 bw: 122.388 +[0:4] time: 0.0657915 bw: 122.403 +[1:0] time: 0.0659209 bw: 122.162 +[1:1] time: 0.065774 bw: 122.435 +[1:2] time: 0.0657776 bw: 122.429 +[1:3] time: 0.0657843 bw: 122.416 +[1:4] time: 0.065796 bw: 122.394 +[2:0] time: 0.0660119 bw: 121.994 +[2:1] time: 0.0657994 bw: 122.388 +[2:2] time: 0.0658116 bw: 122.365 +[2:3] time: 0.0658104 bw: 122.368 +[2:4] time: 0.0657992 bw: 122.389 +[3:0] time: 0.0659366 bw: 122.133 +[3:1] time: 0.0657833 bw: 122.418 +[3:2] time: 0.0657837 bw: 122.417 +[3:3] time: 0.0657796 bw: 122.425 +[3:4] time: 0.0657807 bw: 122.423 +[4:0] time: 0.0659165 bw: 122.171 +[4:1] time: 0.0658036 bw: 122.38 +[4:2] time: 0.0658015 bw: 122.384 +[4:3] time: 0.0658102 bw: 122.368 +[4:4] time: 0.0658293 bw: 122.332 +[5:0] time: 0.0660176 bw: 121.984 +[5:1] time: 0.0657672 bw: 122.448 +[5:2] time: 0.0657704 bw: 122.442 +[5:3] time: 0.0657876 bw: 122.41 +[5:4] time: 0.0657723 bw: 122.438 +[6:0] time: 0.065951 bw: 122.107 +[6:1] time: 0.0658052 bw: 122.377 +[6:2] time: 0.0658152 bw: 122.359 +[6:3] time: 0.0658016 bw: 122.384 +[6:4] time: 0.0658126 bw: 122.364 +[7:0] time: 0.0659276 bw: 122.15 +[7:1] time: 0.0657712 bw: 122.441 +[7:2] time: 0.065771 bw: 122.441 +[7:3] time: 0.0657692 bw: 122.444 +[7:4] time: 0.0657852 bw: 122.415 +[8:0] time: 0.0659151 bw: 122.173 +[8:1] time: 0.065794 bw: 122.398 +[8:2] time: 0.0658168 bw: 122.356 +[8:3] time: 0.0658174 bw: 122.355 +[8:4] time: 0.0658179 bw: 122.354 +[9:0] time: 0.0658831 bw: 122.233 +[9:1] time: 0.0657896 bw: 122.406 +[9:2] time: 0.0657703 bw: 122.442 +[9:3] time: 0.0657817 bw: 122.421 +[9:4] time: 0.0657768 bw: 122.43 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.0659138 +time0: 0.0666406 +bw: 122.191 +bw0: 120.954 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out new file mode 100644 index 0000000..762d97e --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out @@ -0,0 +1,407 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] +Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.5358 +[0:0] time: 0.272119 bw: 30.5803 +[0:1] time: 0.237648 bw: 35.0161 +[0:2] time: 0.238383 bw: 34.9082 +[0:3] time: 0.238066 bw: 34.9546 +[0:4] time: 0.238775 bw: 34.8508 +[1:0] time: 0.274917 bw: 30.2691 +[1:1] time: 0.240515 bw: 34.5987 +[1:2] time: 0.240849 bw: 34.5506 +[1:3] time: 0.238045 bw: 34.9577 +[1:4] time: 0.239615 bw: 34.7286 +[2:0] time: 0.287758 bw: 28.9184 +[2:1] time: 0.239333 bw: 34.7696 +[2:2] time: 0.238508 bw: 34.8898 +[2:3] time: 0.237994 bw: 34.9652 +[2:4] time: 0.237478 bw: 35.0411 +[3:0] time: 0.286365 bw: 29.0591 +[3:1] time: 0.23751 bw: 35.0364 +[3:2] time: 0.237439 bw: 35.0469 +[3:3] time: 0.237622 bw: 35.0199 +[3:4] time: 0.237657 bw: 35.0148 +[4:0] time: 0.289036 bw: 28.7906 +[4:1] time: 0.237409 bw: 35.0513 +[4:2] time: 0.237221 bw: 35.0791 +[4:3] time: 0.237719 bw: 35.0056 +[4:4] time: 0.23746 bw: 35.0438 +[5:0] time: 0.288283 bw: 28.8657 +[5:1] time: 0.237349 bw: 35.0602 +[5:2] time: 0.23738 bw: 35.0557 +[5:3] time: 0.237856 bw: 34.9854 +[5:4] time: 0.237546 bw: 35.031 +[6:0] time: 0.289813 bw: 28.7133 +[6:1] time: 0.238364 bw: 34.9109 +[6:2] time: 0.238187 bw: 34.9369 +[6:3] time: 0.23819 bw: 34.9363 +[6:4] time: 0.238636 bw: 34.871 +[7:0] time: 0.28947 bw: 28.7474 +[7:1] time: 0.238051 bw: 34.9567 +[7:2] time: 0.238143 bw: 34.9433 +[7:3] time: 0.237987 bw: 34.9662 +[7:4] time: 0.238179 bw: 34.938 +[8:0] time: 0.292047 bw: 28.4937 +[8:1] time: 0.23808 bw: 34.9526 +[8:2] time: 0.237888 bw: 34.9807 +[8:3] time: 0.238048 bw: 34.9573 +[8:4] time: 0.237946 bw: 34.9722 +[9:0] time: 0.294056 bw: 28.299 +[9:1] time: 0.237718 bw: 35.0057 +[9:2] time: 0.2376 bw: 35.0232 +[9:3] time: 0.23795 bw: 34.9716 +[9:4] time: 0.239134 bw: 34.7985 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.238137 +time0: 0.286386 +bw: 34.9446 +bw0: 29.0737 +======================================= + +Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] +Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.3317 +[0:0] time: 15.3146 bw: 0.543371 +[0:1] time: 0.488911 bw: 17.0205 +[0:2] time: 0.488674 bw: 17.0287 +[0:3] time: 0.495813 bw: 16.7835 +[0:4] time: 0.494635 bw: 16.8235 +[1:0] time: 15.3212 bw: 0.543136 +[1:1] time: 0.491453 bw: 16.9325 +[1:2] time: 0.489134 bw: 17.0127 +[1:3] time: 0.488371 bw: 17.0393 +[1:4] time: 0.489111 bw: 17.0135 +[2:0] time: 15.3396 bw: 0.542484 +[2:1] time: 0.489068 bw: 17.015 +[2:2] time: 0.488835 bw: 17.0231 +[2:3] time: 0.489616 bw: 16.996 +[2:4] time: 0.489063 bw: 17.0152 +[3:0] time: 15.3209 bw: 0.543147 +[3:1] time: 0.487861 bw: 17.0571 +[3:2] time: 0.488828 bw: 17.0234 +[3:3] time: 0.488735 bw: 17.0266 +[3:4] time: 0.489019 bw: 17.0167 +[4:0] time: 15.3538 bw: 0.541982 +[4:1] time: 0.488568 bw: 17.0324 +[4:2] time: 0.48805 bw: 17.0505 +[4:3] time: 0.489084 bw: 17.0145 +[4:4] time: 0.489108 bw: 17.0136 +[5:0] time: 15.3286 bw: 0.542875 +[5:1] time: 0.487746 bw: 17.0611 +[5:2] time: 0.488908 bw: 17.0206 +[5:3] time: 0.488416 bw: 17.0377 +[5:4] time: 0.489023 bw: 17.0166 +[6:0] time: 15.3282 bw: 0.542889 +[6:1] time: 0.489106 bw: 17.0137 +[6:2] time: 0.489054 bw: 17.0155 +[6:3] time: 0.489433 bw: 17.0023 +[6:4] time: 0.489527 bw: 16.9991 +[7:0] time: 15.3116 bw: 0.543478 +[7:1] time: 0.489192 bw: 17.0107 +[7:2] time: 0.48956 bw: 16.9979 +[7:3] time: 0.489072 bw: 17.0149 +[7:4] time: 0.489586 bw: 16.997 +[8:0] time: 15.3319 bw: 0.542759 +[8:1] time: 0.489171 bw: 17.0114 +[8:2] time: 0.488715 bw: 17.0273 +[8:3] time: 0.489309 bw: 17.0066 +[8:4] time: 0.489263 bw: 17.0082 +[9:0] time: 15.333 bw: 0.542719 +[9:1] time: 0.489545 bw: 16.9984 +[9:2] time: 0.489924 bw: 16.9853 +[9:3] time: 0.489367 bw: 17.0046 +[9:4] time: 0.489619 bw: 16.9959 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.489387 +time0: 15.3283 +bw: 17.0041 +bw0: 0.542884 +======================================= + +Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] +Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.23378 +[0:0] time: 0.129277 bw: 64.3697 +[0:1] time: 0.128748 bw: 64.6338 +[0:2] time: 0.129379 bw: 64.3187 +[0:3] time: 0.130838 bw: 63.6015 +[0:4] time: 0.133852 bw: 62.1694 +[1:0] time: 0.138144 bw: 60.2379 +[1:1] time: 0.135822 bw: 61.2676 +[1:2] time: 0.140804 bw: 59.0999 +[1:3] time: 0.159225 bw: 52.2625 +[1:4] time: 0.218732 bw: 38.0443 +[2:0] time: 0.219306 bw: 37.9447 +[2:1] time: 0.232804 bw: 35.7447 +[2:2] time: 0.235519 bw: 35.3326 +[2:3] time: 0.237181 bw: 35.085 +[2:4] time: 0.238294 bw: 34.9212 +[3:0] time: 0.199769 bw: 41.6556 +[3:1] time: 0.230894 bw: 36.0403 +[3:2] time: 0.242589 bw: 34.3029 +[3:3] time: 0.241189 bw: 34.502 +[3:4] time: 0.243914 bw: 34.1165 +[4:0] time: 0.224221 bw: 37.1129 +[4:1] time: 0.240922 bw: 34.5403 +[4:2] time: 0.243774 bw: 34.1361 +[4:3] time: 0.244955 bw: 33.9715 +[4:4] time: 0.244927 bw: 33.9754 +[5:0] time: 0.2026 bw: 41.0735 +[5:1] time: 0.219568 bw: 37.8994 +[5:2] time: 0.246739 bw: 33.7259 +[5:3] time: 0.247412 bw: 33.6341 +[5:4] time: 0.249365 bw: 33.3708 +[6:0] time: 0.226711 bw: 36.7053 +[6:1] time: 0.243836 bw: 34.1275 +[6:2] time: 0.244916 bw: 33.9769 +[6:3] time: 0.244891 bw: 33.9805 +[6:4] time: 0.245022 bw: 33.9622 +[7:0] time: 0.208145 bw: 39.9793 +[7:1] time: 0.218671 bw: 38.0549 +[7:2] time: 0.247365 bw: 33.6406 +[7:3] time: 0.245194 bw: 33.9384 +[7:4] time: 0.246479 bw: 33.7615 +[8:0] time: 0.211283 bw: 39.3856 +[8:1] time: 0.227064 bw: 36.6483 +[8:2] time: 0.262478 bw: 31.7036 +[8:3] time: 0.261062 bw: 31.8756 +[8:4] time: 0.261084 bw: 31.8729 +[9:0] time: 0.21635 bw: 38.4632 +[9:1] time: 0.230681 bw: 36.0736 +[9:2] time: 0.232946 bw: 35.7229 +[9:3] time: 0.245782 bw: 33.8573 +[9:4] time: 0.262406 bw: 31.7123 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.223433 +time0: 0.197581 +bw: 39.1401 +bw0: 43.6928 +======================================= + +Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] +Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.241767 +[0:0] time: 0.136765 bw: 60.8453 +[0:1] time: 0.133156 bw: 62.4946 +[0:2] time: 0.138198 bw: 60.2142 +[0:3] time: 0.135602 bw: 61.3672 +[0:4] time: 0.139155 bw: 59.8001 +[1:0] time: 0.139563 bw: 59.6254 +[1:1] time: 0.147225 bw: 56.5223 +[1:2] time: 0.162121 bw: 51.329 +[1:3] time: 0.203298 bw: 40.9325 +[1:4] time: 0.206365 bw: 40.3242 +[2:0] time: 0.191855 bw: 43.3739 +[2:1] time: 0.20539 bw: 40.5156 +[2:2] time: 0.221643 bw: 37.5445 +[2:3] time: 0.229658 bw: 36.2342 +[2:4] time: 0.229823 bw: 36.2084 +[3:0] time: 0.214055 bw: 38.8755 +[3:1] time: 0.241419 bw: 34.4691 +[3:2] time: 0.246477 bw: 33.7617 +[3:3] time: 0.249312 bw: 33.3778 +[3:4] time: 0.249366 bw: 33.3706 +[4:0] time: 0.232967 bw: 35.7196 +[4:1] time: 0.247684 bw: 33.5973 +[4:2] time: 0.246973 bw: 33.694 +[4:3] time: 0.248154 bw: 33.5336 +[4:4] time: 0.250923 bw: 33.1635 +[5:0] time: 0.235223 bw: 35.3771 +[5:1] time: 0.246523 bw: 33.7555 +[5:2] time: 0.245224 bw: 33.9343 +[5:3] time: 0.251414 bw: 33.0988 +[5:4] time: 0.27345 bw: 30.4315 +[6:0] time: 0.256066 bw: 32.4975 +[6:1] time: 0.245473 bw: 33.8998 +[6:2] time: 0.248569 bw: 33.4776 +[6:3] time: 0.247796 bw: 33.582 +[6:4] time: 0.248193 bw: 33.5284 +[7:0] time: 0.212248 bw: 39.2065 +[7:1] time: 0.248331 bw: 33.5097 +[7:2] time: 0.257127 bw: 32.3634 +[7:3] time: 0.258463 bw: 32.1961 +[7:4] time: 0.260985 bw: 31.8849 +[8:0] time: 0.244194 bw: 34.0774 +[8:1] time: 0.254596 bw: 32.6851 +[8:2] time: 0.257055 bw: 32.3724 +[8:3] time: 0.291329 bw: 28.5639 +[8:4] time: 0.294063 bw: 28.2983 +[9:0] time: 0.250627 bw: 33.2027 +[9:1] time: 0.259761 bw: 32.0353 +[9:2] time: 0.262148 bw: 31.7435 +[9:3] time: 0.264955 bw: 31.4072 +[9:4] time: 0.263812 bw: 31.5433 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.23278 +time0: 0.211356 +bw: 37.4191 +bw0: 41.2801 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out new file mode 100644 index 0000000..759750f --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out @@ -0,0 +1,407 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.633378 +[0:0] time: 0.494195 bw: 16.8385 +[0:1] time: 0.460126 bw: 18.0853 +[0:2] time: 0.461276 bw: 18.0402 +[0:3] time: 0.460579 bw: 18.0675 +[0:4] time: 0.46246 bw: 17.994 +[1:0] time: 0.496649 bw: 16.7553 +[1:1] time: 0.461198 bw: 18.0432 +[1:2] time: 0.461104 bw: 18.0469 +[1:3] time: 0.462001 bw: 18.0119 +[1:4] time: 0.46027 bw: 18.0796 +[2:0] time: 0.498402 bw: 16.6964 +[2:1] time: 0.460073 bw: 18.0874 +[2:2] time: 0.460819 bw: 18.0581 +[2:3] time: 0.460536 bw: 18.0692 +[2:4] time: 0.460496 bw: 18.0707 +[3:0] time: 0.496646 bw: 16.7554 +[3:1] time: 0.460274 bw: 18.0794 +[3:2] time: 0.46012 bw: 18.0855 +[3:3] time: 0.45978 bw: 18.0989 +[3:4] time: 0.459854 bw: 18.0959 +[4:0] time: 0.495814 bw: 16.7835 +[4:1] time: 0.460021 bw: 18.0894 +[4:2] time: 0.459773 bw: 18.0992 +[4:3] time: 0.459722 bw: 18.1011 +[4:4] time: 0.459713 bw: 18.1015 +[5:0] time: 0.496767 bw: 16.7513 +[5:1] time: 0.459741 bw: 18.1004 +[5:2] time: 0.459751 bw: 18.1 +[5:3] time: 0.459853 bw: 18.096 +[5:4] time: 0.459769 bw: 18.0993 +[6:0] time: 0.50076 bw: 16.6177 +[6:1] time: 0.46046 bw: 18.0722 +[6:2] time: 0.460443 bw: 18.0728 +[6:3] time: 0.460526 bw: 18.0696 +[6:4] time: 0.460566 bw: 18.068 +[7:0] time: 0.500478 bw: 16.6271 +[7:1] time: 0.460434 bw: 18.0732 +[7:2] time: 0.460552 bw: 18.0685 +[7:3] time: 0.460665 bw: 18.0641 +[7:4] time: 0.460517 bw: 18.0699 +[8:0] time: 0.500088 bw: 16.6401 +[8:1] time: 0.460416 bw: 18.0739 +[8:2] time: 0.460721 bw: 18.0619 +[8:3] time: 0.460516 bw: 18.07 +[8:4] time: 0.460538 bw: 18.0691 +[9:0] time: 0.500873 bw: 16.614 +[9:1] time: 0.460452 bw: 18.0725 +[9:2] time: 0.46073 bw: 18.0616 +[9:3] time: 0.460643 bw: 18.065 +[9:4] time: 0.460624 bw: 18.0657 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.460453 +time0: 0.498067 +bw: 18.0725 +bw0: 16.7079 +======================================= + +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.61589 +[0:0] time: 1.59151 bw: 5.22868 +[0:1] time: 1.5849 bw: 5.25048 +[0:2] time: 1.58426 bw: 5.25262 +[0:3] time: 1.58139 bw: 5.26215 +[0:4] time: 1.57734 bw: 5.27564 +[1:0] time: 1.62241 bw: 5.12911 +[1:1] time: 1.57625 bw: 5.2793 +[1:2] time: 1.57924 bw: 5.26932 +[1:3] time: 1.5798 bw: 5.26744 +[1:4] time: 1.58211 bw: 5.25974 +[2:0] time: 1.61847 bw: 5.1416 +[2:1] time: 1.58337 bw: 5.25557 +[2:2] time: 1.57872 bw: 5.27104 +[2:3] time: 1.57994 bw: 5.26698 +[2:4] time: 1.57869 bw: 5.27115 +[3:0] time: 1.62336 bw: 5.12611 +[3:1] time: 1.57834 bw: 5.27231 +[3:2] time: 1.58255 bw: 5.2583 +[3:3] time: 1.57513 bw: 5.28305 +[3:4] time: 1.58089 bw: 5.2638 +[4:0] time: 1.62349 bw: 5.12568 +[4:1] time: 1.57971 bw: 5.26775 +[4:2] time: 1.57898 bw: 5.27016 +[4:3] time: 1.58272 bw: 5.25774 +[4:4] time: 1.5777 bw: 5.27444 +[5:0] time: 1.63172 bw: 5.09982 +[5:1] time: 1.57912 bw: 5.26969 +[5:2] time: 1.58129 bw: 5.26247 +[5:3] time: 1.58291 bw: 5.2571 +[5:4] time: 1.58223 bw: 5.25935 +[6:0] time: 1.62536 bw: 5.1198 +[6:1] time: 1.58651 bw: 5.24515 +[6:2] time: 1.57969 bw: 5.2678 +[6:3] time: 1.5824 bw: 5.25879 +[6:4] time: 1.58236 bw: 5.25892 +[7:0] time: 1.63464 bw: 5.09071 +[7:1] time: 1.58165 bw: 5.26129 +[7:2] time: 1.58741 bw: 5.24219 +[7:3] time: 1.57853 bw: 5.27167 +[7:4] time: 1.58302 bw: 5.25671 +[8:0] time: 1.63303 bw: 5.09575 +[8:1] time: 1.58233 bw: 5.25901 +[8:2] time: 1.58062 bw: 5.26471 +[8:3] time: 1.58599 bw: 5.24687 +[8:4] time: 1.57939 bw: 5.26881 +[9:0] time: 1.63824 bw: 5.07954 +[9:1] time: 1.58286 bw: 5.25727 +[9:2] time: 1.58338 bw: 5.25553 +[9:3] time: 1.58319 bw: 5.25615 +[9:4] time: 1.58604 bw: 5.24672 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.58132 +time0: 1.62422 +bw: 5.26238 +bw0: 5.12368 +======================================= + +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.225496 +[0:0] time: 0.136698 bw: 60.8751 +[0:1] time: 0.134958 bw: 61.6599 +[0:2] time: 0.128691 bw: 64.6626 +[0:3] time: 0.128683 bw: 64.6667 +[0:4] time: 0.128685 bw: 64.6655 +[1:0] time: 0.129058 bw: 64.4789 +[1:1] time: 0.128637 bw: 64.69 +[1:2] time: 0.128651 bw: 64.6828 +[1:3] time: 0.128656 bw: 64.6803 +[1:4] time: 0.128647 bw: 64.6847 +[2:0] time: 0.12905 bw: 64.4827 +[2:1] time: 0.128671 bw: 64.6726 +[2:2] time: 0.128657 bw: 64.68 +[2:3] time: 0.128668 bw: 64.6741 +[2:4] time: 0.128664 bw: 64.6761 +[3:0] time: 0.129074 bw: 64.4707 +[3:1] time: 0.128654 bw: 64.681 +[3:2] time: 0.128672 bw: 64.6723 +[3:3] time: 0.128647 bw: 64.6846 +[3:4] time: 0.128664 bw: 64.6761 +[4:0] time: 0.129034 bw: 64.4908 +[4:1] time: 0.128678 bw: 64.6691 +[4:2] time: 0.128691 bw: 64.6626 +[4:3] time: 0.128672 bw: 64.6724 +[4:4] time: 0.128663 bw: 64.6766 +[5:0] time: 0.129052 bw: 64.482 +[5:1] time: 0.128657 bw: 64.6797 +[5:2] time: 0.128769 bw: 64.6234 +[5:3] time: 0.128678 bw: 64.6694 +[5:4] time: 0.128679 bw: 64.6689 +[6:0] time: 0.128968 bw: 64.5238 +[6:1] time: 0.128831 bw: 64.5924 +[6:2] time: 0.128688 bw: 64.664 +[6:3] time: 0.128681 bw: 64.6677 +[6:4] time: 0.128666 bw: 64.6752 +[7:0] time: 0.129101 bw: 64.4575 +[7:1] time: 0.128662 bw: 64.6772 +[7:2] time: 0.128643 bw: 64.6868 +[7:3] time: 0.128637 bw: 64.69 +[7:4] time: 0.128644 bw: 64.686 +[8:0] time: 0.129218 bw: 64.3989 +[8:1] time: 0.128695 bw: 64.6608 +[8:2] time: 0.128658 bw: 64.6791 +[8:3] time: 0.128656 bw: 64.68 +[8:4] time: 0.128653 bw: 64.6815 +[9:0] time: 0.129183 bw: 64.4164 +[9:1] time: 0.12868 bw: 64.668 +[9:2] time: 0.128645 bw: 64.6857 +[9:3] time: 0.128871 bw: 64.5721 +[9:4] time: 0.128674 bw: 64.6714 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.128834 +time0: 0.129843 +bw: 64.5942 +bw0: 64.1077 +======================================= + +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.203091 +[0:0] time: 0.138902 bw: 59.9091 +[0:1] time: 0.138449 bw: 60.1052 +[0:2] time: 0.136795 bw: 60.832 +[0:3] time: 0.135388 bw: 61.464 +[0:4] time: 0.135144 bw: 61.5749 +[1:0] time: 0.137294 bw: 60.6107 +[1:1] time: 0.135363 bw: 61.4753 +[1:2] time: 0.135182 bw: 61.5578 +[1:3] time: 0.135137 bw: 61.5784 +[1:4] time: 0.135177 bw: 61.5599 +[2:0] time: 0.135791 bw: 61.2815 +[2:1] time: 0.132879 bw: 62.6244 +[2:2] time: 0.132895 bw: 62.6172 +[2:3] time: 0.132892 bw: 62.6186 +[2:4] time: 0.132904 bw: 62.6128 +[3:0] time: 0.132944 bw: 62.5942 +[3:1] time: 0.132761 bw: 62.6804 +[3:2] time: 0.132782 bw: 62.6705 +[3:3] time: 0.132793 bw: 62.6651 +[3:4] time: 0.132774 bw: 62.674 +[4:0] time: 0.132916 bw: 62.6073 +[4:1] time: 0.132888 bw: 62.6205 +[4:2] time: 0.132924 bw: 62.6036 +[4:3] time: 0.132898 bw: 62.6155 +[4:4] time: 0.132919 bw: 62.6056 +[5:0] time: 0.13291 bw: 62.6102 +[5:1] time: 0.132767 bw: 62.6775 +[5:2] time: 0.132756 bw: 62.6826 +[5:3] time: 0.132929 bw: 62.6012 +[5:4] time: 0.132815 bw: 62.655 +[6:0] time: 0.133057 bw: 62.541 +[6:1] time: 0.132893 bw: 62.6179 +[6:2] time: 0.132943 bw: 62.5947 +[6:3] time: 0.132916 bw: 62.6073 +[6:4] time: 0.132914 bw: 62.6082 +[7:0] time: 0.133092 bw: 62.5245 +[7:1] time: 0.132775 bw: 62.6737 +[7:2] time: 0.132753 bw: 62.6842 +[7:3] time: 0.132761 bw: 62.6801 +[7:4] time: 0.132743 bw: 62.6886 +[8:0] time: 0.132998 bw: 62.5685 +[8:1] time: 0.132878 bw: 62.6249 +[8:2] time: 0.132914 bw: 62.6083 +[8:3] time: 0.132899 bw: 62.6154 +[8:4] time: 0.132902 bw: 62.6139 +[9:0] time: 0.132978 bw: 62.5778 +[9:1] time: 0.132747 bw: 62.6869 +[9:2] time: 0.13273 bw: 62.6951 +[9:3] time: 0.132761 bw: 62.6803 +[9:4] time: 0.132771 bw: 62.6755 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.133438 +time0: 0.134288 +bw: 62.3682 +bw0: 61.9825 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out new file mode 100644 index 0000000..c2d9994 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out @@ -0,0 +1,535 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] +Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.577848 +[0:0] time: 0.296514 bw: 28.5171 +[0:1] time: 0.260698 bw: 32.435 +[0:2] time: 0.26355 bw: 32.0839 +[0:3] time: 0.263462 bw: 32.0946 +[0:4] time: 0.262884 bw: 32.1652 +[1:0] time: 0.301235 bw: 28.0702 +[1:1] time: 0.265641 bw: 31.8314 +[1:2] time: 0.266208 bw: 31.7636 +[1:3] time: 0.260905 bw: 32.4092 +[1:4] time: 0.264152 bw: 32.0108 +[2:0] time: 0.325059 bw: 26.0128 +[2:1] time: 0.261902 bw: 32.2858 +[2:2] time: 0.263345 bw: 32.1089 +[2:3] time: 0.260814 bw: 32.4204 +[2:4] time: 0.260924 bw: 32.4068 +[3:0] time: 0.320153 bw: 26.4115 +[3:1] time: 0.260319 bw: 32.4821 +[3:2] time: 0.260399 bw: 32.4722 +[3:3] time: 0.26014 bw: 32.5045 +[3:4] time: 0.260093 bw: 32.5104 +[4:0] time: 0.326936 bw: 25.8635 +[4:1] time: 0.260172 bw: 32.5005 +[4:2] time: 0.260607 bw: 32.4462 +[4:3] time: 0.260016 bw: 32.52 +[4:4] time: 0.260194 bw: 32.4977 +[5:0] time: 0.322766 bw: 26.1977 +[5:1] time: 0.26018 bw: 32.4995 +[5:2] time: 0.260792 bw: 32.4233 +[5:3] time: 0.261101 bw: 32.3848 +[5:4] time: 0.261049 bw: 32.3914 +[6:0] time: 0.325079 bw: 26.0112 +[6:1] time: 0.260946 bw: 32.4041 +[6:2] time: 0.260621 bw: 32.4445 +[6:3] time: 0.260808 bw: 32.4212 +[6:4] time: 0.26134 bw: 32.3552 +[7:0] time: 0.326505 bw: 25.8977 +[7:1] time: 0.260917 bw: 32.4077 +[7:2] time: 0.261139 bw: 32.3801 +[7:3] time: 0.261779 bw: 32.301 +[7:4] time: 0.260806 bw: 32.4215 +[8:0] time: 0.323372 bw: 26.1486 +[8:1] time: 0.261384 bw: 32.3497 +[8:2] time: 0.260498 bw: 32.4598 +[8:3] time: 0.260793 bw: 32.4231 +[8:4] time: 0.261004 bw: 32.3969 +[9:0] time: 0.325747 bw: 25.9579 +[9:1] time: 0.261037 bw: 32.3929 +[9:2] time: 0.261697 bw: 32.3112 +[9:3] time: 0.260858 bw: 32.4151 +[9:4] time: 0.260898 bw: 32.4101 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.261402 +time0: 0.319337 +bw: 32.3486 +bw0: 26.5088 +======================================= + +Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] +Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.4604 +[0:0] time: 15.3975 bw: 0.549161 +[0:1] time: 0.533471 bw: 15.8504 +[0:2] time: 0.552953 bw: 15.2919 +[0:3] time: 0.544049 bw: 15.5422 +[0:4] time: 0.534672 bw: 15.8148 +[1:0] time: 15.4652 bw: 0.546756 +[1:1] time: 0.532915 bw: 15.8669 +[1:2] time: 0.532406 bw: 15.8821 +[1:3] time: 0.533373 bw: 15.8533 +[1:4] time: 0.533062 bw: 15.8625 +[2:0] time: 15.4293 bw: 0.54803 +[2:1] time: 0.532551 bw: 15.8777 +[2:2] time: 0.533304 bw: 15.8553 +[2:3] time: 0.533145 bw: 15.8601 +[2:4] time: 0.533563 bw: 15.8476 +[3:0] time: 15.4175 bw: 0.548449 +[3:1] time: 0.533184 bw: 15.8589 +[3:2] time: 0.532332 bw: 15.8843 +[3:3] time: 0.533368 bw: 15.8534 +[3:4] time: 0.532978 bw: 15.865 +[4:0] time: 15.397 bw: 0.549179 +[4:1] time: 0.532439 bw: 15.8811 +[4:2] time: 0.532761 bw: 15.8715 +[4:3] time: 0.532601 bw: 15.8763 +[4:4] time: 0.533335 bw: 15.8544 +[5:0] time: 15.4129 bw: 0.548612 +[5:1] time: 0.532273 bw: 15.8861 +[5:2] time: 0.5323 bw: 15.8852 +[5:3] time: 0.533085 bw: 15.8618 +[5:4] time: 0.533529 bw: 15.8487 +[6:0] time: 15.4066 bw: 0.548837 +[6:1] time: 0.533409 bw: 15.8522 +[6:2] time: 0.533869 bw: 15.8386 +[6:3] time: 0.533509 bw: 15.8492 +[6:4] time: 0.533393 bw: 15.8527 +[7:0] time: 15.4442 bw: 0.547502 +[7:1] time: 0.533528 bw: 15.8487 +[7:2] time: 0.532995 bw: 15.8645 +[7:3] time: 0.534086 bw: 15.8321 +[7:4] time: 0.533587 bw: 15.8469 +[8:0] time: 15.4427 bw: 0.547554 +[8:1] time: 0.533102 bw: 15.8613 +[8:2] time: 0.533552 bw: 15.848 +[8:3] time: 0.533553 bw: 15.8479 +[8:4] time: 0.53382 bw: 15.84 +[9:0] time: 15.4224 bw: 0.548275 +[9:1] time: 0.533314 bw: 15.855 +[9:2] time: 0.533487 bw: 15.8499 +[9:3] time: 0.534329 bw: 15.8249 +[9:4] time: 0.533597 bw: 15.8466 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.534019 +time0: 15.4235 +bw: 15.8348 +bw0: 0.548236 +======================================= + +Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] +Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.371044 +[0:0] time: 0.264918 bw: 31.9183 +[0:1] time: 0.258789 bw: 32.6741 +[0:2] time: 0.258964 bw: 32.6521 +[0:3] time: 0.2588 bw: 32.6728 +[0:4] time: 0.258787 bw: 32.6745 +[1:0] time: 0.261152 bw: 32.3786 +[1:1] time: 0.258654 bw: 32.6913 +[1:2] time: 0.258672 bw: 32.689 +[1:3] time: 0.258702 bw: 32.6852 +[1:4] time: 0.258711 bw: 32.684 +[2:0] time: 0.259632 bw: 32.5681 +[2:1] time: 0.25873 bw: 32.6817 +[2:2] time: 0.258757 bw: 32.6782 +[2:3] time: 0.258763 bw: 32.6774 +[2:4] time: 0.258781 bw: 32.6751 +[3:0] time: 0.259519 bw: 32.5822 +[3:1] time: 0.258671 bw: 32.6891 +[3:2] time: 0.258675 bw: 32.6886 +[3:3] time: 0.258644 bw: 32.6925 +[3:4] time: 0.258669 bw: 32.6893 +[4:0] time: 0.260168 bw: 32.501 +[4:1] time: 0.258733 bw: 32.6812 +[4:2] time: 0.258725 bw: 32.6823 +[4:3] time: 0.258742 bw: 32.6801 +[4:4] time: 0.25876 bw: 32.6779 +[5:0] time: 0.259661 bw: 32.5644 +[5:1] time: 0.258665 bw: 32.6898 +[5:2] time: 0.258638 bw: 32.6933 +[5:3] time: 0.260174 bw: 32.5003 +[5:4] time: 0.260529 bw: 32.456 +[6:0] time: 0.259625 bw: 32.569 +[6:1] time: 0.25875 bw: 32.6792 +[6:2] time: 0.258739 bw: 32.6804 +[6:3] time: 0.258753 bw: 32.6788 +[6:4] time: 0.258742 bw: 32.6802 +[7:0] time: 0.25956 bw: 32.5771 +[7:1] time: 0.258662 bw: 32.6902 +[7:2] time: 0.258621 bw: 32.6954 +[7:3] time: 0.258645 bw: 32.6923 +[7:4] time: 0.258651 bw: 32.6917 +[8:0] time: 0.26263 bw: 32.1963 +[8:1] time: 0.258792 bw: 32.6737 +[8:2] time: 0.258731 bw: 32.6814 +[8:3] time: 0.258724 bw: 32.6823 +[8:4] time: 0.258772 bw: 32.6763 +[9:0] time: 0.259719 bw: 32.5572 +[9:1] time: 0.25863 bw: 32.6943 +[9:2] time: 0.258637 bw: 32.6933 +[9:3] time: 0.258607 bw: 32.6971 +[9:4] time: 0.258654 bw: 32.6912 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.258796 +time0: 0.260658 +bw: 32.6733 +bw0: 32.4412 +======================================= + +Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] +Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.404822 +[0:0] time: 0.267391 bw: 31.623 +[0:1] time: 0.267059 bw: 31.6623 +[0:2] time: 0.26707 bw: 31.661 +[0:3] time: 0.267079 bw: 31.66 +[0:4] time: 0.267049 bw: 31.6635 +[1:0] time: 0.267457 bw: 31.6152 +[1:1] time: 0.267035 bw: 31.6652 +[1:2] time: 0.267037 bw: 31.665 +[1:3] time: 0.267048 bw: 31.6637 +[1:4] time: 0.267045 bw: 31.6641 +[2:0] time: 0.267568 bw: 31.6022 +[2:1] time: 0.267068 bw: 31.6613 +[2:2] time: 0.267054 bw: 31.6629 +[2:3] time: 0.267046 bw: 31.6638 +[2:4] time: 0.267073 bw: 31.6607 +[3:0] time: 0.26771 bw: 31.5854 +[3:1] time: 0.267008 bw: 31.6684 +[3:2] time: 0.26706 bw: 31.6622 +[3:3] time: 0.267027 bw: 31.6662 +[3:4] time: 0.267058 bw: 31.6624 +[4:0] time: 0.268078 bw: 31.542 +[4:1] time: 0.267089 bw: 31.6587 +[4:2] time: 0.267068 bw: 31.6613 +[4:3] time: 0.267066 bw: 31.6616 +[4:4] time: 0.267034 bw: 31.6653 +[5:0] time: 0.267991 bw: 31.5523 +[5:1] time: 0.267019 bw: 31.6671 +[5:2] time: 0.267046 bw: 31.6639 +[5:3] time: 0.267005 bw: 31.6687 +[5:4] time: 0.267036 bw: 31.6651 +[6:0] time: 0.268042 bw: 31.5462 +[6:1] time: 0.267038 bw: 31.6648 +[6:2] time: 0.267056 bw: 31.6627 +[6:3] time: 0.267031 bw: 31.6656 +[6:4] time: 0.271913 bw: 31.0971 +[7:0] time: 0.268058 bw: 31.5443 +[7:1] time: 0.267011 bw: 31.6681 +[7:2] time: 0.267023 bw: 31.6666 +[7:3] time: 0.267008 bw: 31.6684 +[7:4] time: 0.267011 bw: 31.6681 +[8:0] time: 0.270079 bw: 31.3084 +[8:1] time: 0.267047 bw: 31.6638 +[8:2] time: 0.267059 bw: 31.6624 +[8:3] time: 0.267095 bw: 31.658 +[8:4] time: 0.267085 bw: 31.6592 +[9:0] time: 0.26802 bw: 31.5488 +[9:1] time: 0.267045 bw: 31.664 +[9:2] time: 0.267024 bw: 31.6666 +[9:3] time: 0.266986 bw: 31.671 +[9:4] time: 0.267004 bw: 31.6689 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.267165 +time0: 0.268039 +bw: 31.65 +bw0: 31.5468 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out new file mode 100644 index 0000000..b20d5b7 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out @@ -0,0 +1,535 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] +Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.661234 +[0:0] time: 0.553014 bw: 15.2902 +[0:1] time: 0.512511 bw: 16.4986 +[0:2] time: 0.509409 bw: 16.5991 +[0:3] time: 0.510615 bw: 16.5599 +[0:4] time: 0.510136 bw: 16.5754 +[1:0] time: 0.543105 bw: 15.5692 +[1:1] time: 0.506487 bw: 16.6948 +[1:2] time: 0.505329 bw: 16.7331 +[1:3] time: 0.505525 bw: 16.7266 +[1:4] time: 0.505777 bw: 16.7183 +[2:0] time: 0.542354 bw: 15.5908 +[2:1] time: 0.505128 bw: 16.7397 +[2:2] time: 0.504478 bw: 16.7613 +[2:3] time: 0.504188 bw: 16.771 +[2:4] time: 0.503884 bw: 16.7811 +[3:0] time: 0.541636 bw: 15.6114 +[3:1] time: 0.504279 bw: 16.7679 +[3:2] time: 0.503556 bw: 16.792 +[3:3] time: 0.504248 bw: 16.769 +[3:4] time: 0.5042 bw: 16.7706 +[4:0] time: 0.544191 bw: 15.5381 +[4:1] time: 0.503954 bw: 16.7787 +[4:2] time: 0.503594 bw: 16.7907 +[4:3] time: 0.502879 bw: 16.8146 +[4:4] time: 0.503713 bw: 16.7868 +[5:0] time: 0.543364 bw: 15.5618 +[5:1] time: 0.504404 bw: 16.7638 +[5:2] time: 0.503679 bw: 16.7879 +[5:3] time: 0.504213 bw: 16.7701 +[5:4] time: 0.504002 bw: 16.7771 +[6:0] time: 0.543806 bw: 15.5491 +[6:1] time: 0.50485 bw: 16.749 +[6:2] time: 0.505183 bw: 16.7379 +[6:3] time: 0.504502 bw: 16.7605 +[6:4] time: 0.505087 bw: 16.7411 +[7:0] time: 0.545689 bw: 15.4955 +[7:1] time: 0.504497 bw: 16.7607 +[7:2] time: 0.504201 bw: 16.7705 +[7:3] time: 0.504589 bw: 16.7576 +[7:4] time: 0.508141 bw: 16.6405 +[8:0] time: 0.545215 bw: 15.509 +[8:1] time: 0.50496 bw: 16.7453 +[8:2] time: 0.504247 bw: 16.769 +[8:3] time: 0.504246 bw: 16.769 +[8:4] time: 0.504552 bw: 16.7589 +[9:0] time: 0.545114 bw: 15.5118 +[9:1] time: 0.504756 bw: 16.7521 +[9:2] time: 0.504522 bw: 16.7599 +[9:3] time: 0.505344 bw: 16.7326 +[9:4] time: 0.50697 bw: 16.6789 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.505271 +time0: 0.544749 +bw: 16.7353 +bw0: 15.5227 +======================================= + +Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] +Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.66756 +[0:0] time: 1.65 bw: 5.12468 +[0:1] time: 1.65037 bw: 5.12352 +[0:2] time: 1.63637 bw: 5.16736 +[0:3] time: 1.63381 bw: 5.17546 +[0:4] time: 1.6333 bw: 5.17707 +[1:0] time: 1.67583 bw: 5.04569 +[1:1] time: 1.63422 bw: 5.17415 +[1:2] time: 1.63415 bw: 5.17439 +[1:3] time: 1.63126 bw: 5.18354 +[1:4] time: 1.63254 bw: 5.17949 +[2:0] time: 1.67461 bw: 5.04937 +[2:1] time: 1.63353 bw: 5.17635 +[2:2] time: 1.63351 bw: 5.1764 +[2:3] time: 1.63255 bw: 5.17945 +[2:4] time: 1.63238 bw: 5.17998 +[3:0] time: 1.67806 bw: 5.03899 +[3:1] time: 1.63336 bw: 5.17687 +[3:2] time: 1.63386 bw: 5.1753 +[3:3] time: 1.63344 bw: 5.17663 +[3:4] time: 1.63266 bw: 5.17909 +[4:0] time: 1.67582 bw: 5.04572 +[4:1] time: 1.63484 bw: 5.17219 +[4:2] time: 1.63459 bw: 5.17299 +[4:3] time: 1.63348 bw: 5.17652 +[4:4] time: 1.63232 bw: 5.18017 +[5:0] time: 1.68209 bw: 5.02692 +[5:1] time: 1.63368 bw: 5.17588 +[5:2] time: 1.63293 bw: 5.17825 +[5:3] time: 1.63366 bw: 5.17593 +[5:4] time: 1.63276 bw: 5.17878 +[6:0] time: 1.67957 bw: 5.03446 +[6:1] time: 1.63476 bw: 5.17245 +[6:2] time: 1.63506 bw: 5.17149 +[6:3] time: 1.63559 bw: 5.16983 +[6:4] time: 1.63509 bw: 5.1714 +[7:0] time: 1.68332 bw: 5.02323 +[7:1] time: 1.63503 bw: 5.1716 +[7:2] time: 1.63607 bw: 5.16831 +[7:3] time: 1.63486 bw: 5.17212 +[7:4] time: 1.63246 bw: 5.17973 +[8:0] time: 1.68721 bw: 5.01165 +[8:1] time: 1.63491 bw: 5.17196 +[8:2] time: 1.63552 bw: 5.17006 +[8:3] time: 1.63514 bw: 5.17124 +[8:4] time: 1.6335 bw: 5.17645 +[9:0] time: 1.68562 bw: 5.01637 +[9:1] time: 1.63633 bw: 5.16748 +[9:2] time: 1.6358 bw: 5.16917 +[9:3] time: 1.63311 bw: 5.17768 +[9:4] time: 1.63399 bw: 5.17488 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.63442 +time0: 1.67721 +bw: 5.17354 +bw0: 5.04171 +======================================= + +Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] +Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.348451 +[0:0] time: 0.265063 bw: 31.9008 +[0:1] time: 0.259007 bw: 32.6467 +[0:2] time: 0.258838 bw: 32.668 +[0:3] time: 0.258738 bw: 32.6806 +[0:4] time: 0.258746 bw: 32.6796 +[1:0] time: 0.259437 bw: 32.5925 +[1:1] time: 0.258729 bw: 32.6818 +[1:2] time: 0.258714 bw: 32.6836 +[1:3] time: 0.258726 bw: 32.6821 +[1:4] time: 0.258714 bw: 32.6836 +[2:0] time: 0.259239 bw: 32.6174 +[2:1] time: 0.258773 bw: 32.6762 +[2:2] time: 0.258755 bw: 32.6785 +[2:3] time: 0.258763 bw: 32.6774 +[2:4] time: 0.258837 bw: 32.6681 +[3:0] time: 0.259261 bw: 32.6147 +[3:1] time: 0.258712 bw: 32.6838 +[3:2] time: 0.258719 bw: 32.683 +[3:3] time: 0.258746 bw: 32.6796 +[3:4] time: 0.258733 bw: 32.6812 +[4:0] time: 0.259452 bw: 32.5907 +[4:1] time: 0.258776 bw: 32.6759 +[4:2] time: 0.258747 bw: 32.6795 +[4:3] time: 0.258762 bw: 32.6776 +[4:4] time: 0.258885 bw: 32.6621 +[5:0] time: 0.259056 bw: 32.6405 +[5:1] time: 0.258695 bw: 32.6861 +[5:2] time: 0.258728 bw: 32.6819 +[5:3] time: 0.258711 bw: 32.684 +[5:4] time: 0.258708 bw: 32.6844 +[6:0] time: 0.258975 bw: 32.6506 +[6:1] time: 0.258741 bw: 32.6803 +[6:2] time: 0.258764 bw: 32.6773 +[6:3] time: 0.258812 bw: 32.6712 +[6:4] time: 0.258767 bw: 32.677 +[7:0] time: 0.259106 bw: 32.6342 +[7:1] time: 0.258743 bw: 32.6799 +[7:2] time: 0.258765 bw: 32.6773 +[7:3] time: 0.258771 bw: 32.6765 +[7:4] time: 0.258684 bw: 32.6875 +[8:0] time: 0.259206 bw: 32.6217 +[8:1] time: 0.258739 bw: 32.6804 +[8:2] time: 0.258745 bw: 32.6797 +[8:3] time: 0.258746 bw: 32.6796 +[8:4] time: 0.258748 bw: 32.6794 +[9:0] time: 0.259128 bw: 32.6314 +[9:1] time: 0.258723 bw: 32.6825 +[9:2] time: 0.258742 bw: 32.6801 +[9:3] time: 0.258746 bw: 32.6796 +[9:4] time: 0.258759 bw: 32.678 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.258756 +time0: 0.259792 +bw: 32.6783 +bw0: 32.5495 +======================================= + +Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] +Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.374189 +[0:0] time: 0.302522 bw: 27.9507 +[0:1] time: 0.26733 bw: 31.6302 +[0:2] time: 0.26711 bw: 31.6563 +[0:3] time: 0.267154 bw: 31.6511 +[0:4] time: 0.267168 bw: 31.6494 +[1:0] time: 0.267367 bw: 31.6259 +[1:1] time: 0.266981 bw: 31.6716 +[1:2] time: 0.266991 bw: 31.6704 +[1:3] time: 0.266983 bw: 31.6714 +[1:4] time: 0.266985 bw: 31.6711 +[2:0] time: 0.267516 bw: 31.6083 +[2:1] time: 0.267168 bw: 31.6494 +[2:2] time: 0.267161 bw: 31.6502 +[2:3] time: 0.267157 bw: 31.6508 +[2:4] time: 0.267202 bw: 31.6454 +[3:0] time: 0.267328 bw: 31.6305 +[3:1] time: 0.266962 bw: 31.6738 +[3:2] time: 0.266987 bw: 31.6709 +[3:3] time: 0.266998 bw: 31.6696 +[3:4] time: 0.266974 bw: 31.6725 +[4:0] time: 0.267575 bw: 31.6013 +[4:1] time: 0.267145 bw: 31.6521 +[4:2] time: 0.267163 bw: 31.65 +[4:3] time: 0.267159 bw: 31.6505 +[4:4] time: 0.267189 bw: 31.6469 +[5:0] time: 0.267419 bw: 31.6197 +[5:1] time: 0.267003 bw: 31.669 +[5:2] time: 0.266964 bw: 31.6737 +[5:3] time: 0.266965 bw: 31.6735 +[5:4] time: 0.266995 bw: 31.67 +[6:0] time: 0.267626 bw: 31.5953 +[6:1] time: 0.267199 bw: 31.6458 +[6:2] time: 0.267202 bw: 31.6454 +[6:3] time: 0.267137 bw: 31.6531 +[6:4] time: 0.267165 bw: 31.6498 +[7:0] time: 0.267331 bw: 31.6301 +[7:1] time: 0.266989 bw: 31.6706 +[7:2] time: 0.266989 bw: 31.6707 +[7:3] time: 0.267005 bw: 31.6687 +[7:4] time: 0.267034 bw: 31.6654 +[8:0] time: 0.267876 bw: 31.5658 +[8:1] time: 0.267225 bw: 31.6427 +[8:2] time: 0.267156 bw: 31.6509 +[8:3] time: 0.267182 bw: 31.6478 +[8:4] time: 0.267177 bw: 31.6484 +[9:0] time: 0.267372 bw: 31.6253 +[9:1] time: 0.266982 bw: 31.6715 +[9:2] time: 0.266988 bw: 31.6708 +[9:3] time: 0.267005 bw: 31.6687 +[9:4] time: 0.267011 bw: 31.6681 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.267084 +time0: 0.270993 +bw: 31.6595 +bw0: 31.2453 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out new file mode 100644 index 0000000..629cdac --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out @@ -0,0 +1,791 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] +Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.591198 +[0:0] time: 0.326292 bw: 26.1202 +[0:1] time: 0.283811 bw: 30.0299 +[0:2] time: 0.287136 bw: 29.6822 +[0:3] time: 0.29624 bw: 28.77 +[0:4] time: 0.300375 bw: 28.374 +[1:0] time: 0.338962 bw: 25.1439 +[1:1] time: 0.293263 bw: 29.0621 +[1:2] time: 0.28744 bw: 29.6508 +[1:3] time: 0.2853 bw: 29.8732 +[1:4] time: 0.28324 bw: 30.0904 +[2:0] time: 0.338344 bw: 25.1898 +[2:1] time: 0.283691 bw: 30.0426 +[2:2] time: 0.283325 bw: 30.0814 +[2:3] time: 0.283101 bw: 30.1052 +[2:4] time: 0.283151 bw: 30.1 +[3:0] time: 0.340839 bw: 25.0054 +[3:1] time: 0.282444 bw: 30.1753 +[3:2] time: 0.282981 bw: 30.118 +[3:3] time: 0.283065 bw: 30.1091 +[3:4] time: 0.282786 bw: 30.1388 +[4:0] time: 0.338792 bw: 25.1565 +[4:1] time: 0.283255 bw: 30.0889 +[4:2] time: 0.282958 bw: 30.1205 +[4:3] time: 0.283124 bw: 30.1028 +[4:4] time: 0.283498 bw: 30.0631 +[5:0] time: 0.339911 bw: 25.0737 +[5:1] time: 0.283083 bw: 30.1072 +[5:2] time: 0.28267 bw: 30.1512 +[5:3] time: 0.28261 bw: 30.1576 +[5:4] time: 0.282612 bw: 30.1574 +[6:0] time: 0.341024 bw: 24.9919 +[6:1] time: 0.283905 bw: 30.0199 +[6:2] time: 0.283659 bw: 30.046 +[6:3] time: 0.283697 bw: 30.042 +[6:4] time: 0.283859 bw: 30.0248 +[7:0] time: 0.343417 bw: 24.8177 +[7:1] time: 0.283913 bw: 30.0192 +[7:2] time: 0.283966 bw: 30.0135 +[7:3] time: 0.284001 bw: 30.0099 +[7:4] time: 0.283763 bw: 30.035 +[8:0] time: 0.341885 bw: 24.929 +[8:1] time: 0.28372 bw: 30.0396 +[8:2] time: 0.283598 bw: 30.0525 +[8:3] time: 0.284167 bw: 29.9924 +[8:4] time: 0.283792 bw: 30.0319 +[9:0] time: 0.345817 bw: 24.6455 +[9:1] time: 0.283857 bw: 30.0251 +[9:2] time: 0.284069 bw: 30.0027 +[9:3] time: 0.293016 bw: 29.0865 +[9:4] time: 0.284317 bw: 29.9765 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.284911 +time0: 0.339528 +bw: 29.9192 +bw0: 25.1074 +======================================= + +Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] +Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.5125 +[0:0] time: 15.457 bw: 0.551391 +[0:1] time: 0.585132 bw: 14.5656 +[0:2] time: 0.596263 bw: 14.2937 +[0:3] time: 0.570848 bw: 14.9301 +[0:4] time: 0.565644 bw: 15.0675 +[1:0] time: 15.515 bw: 0.549328 +[1:1] time: 0.564512 bw: 15.0977 +[1:2] time: 0.56464 bw: 15.0943 +[1:3] time: 0.564064 bw: 15.1097 +[1:4] time: 0.567994 bw: 15.0051 +[2:0] time: 15.591 bw: 0.546649 +[2:1] time: 0.565183 bw: 15.0798 +[2:2] time: 0.563374 bw: 15.1282 +[2:3] time: 0.564642 bw: 15.0942 +[2:4] time: 0.565448 bw: 15.0727 +[3:0] time: 15.6007 bw: 0.546312 +[3:1] time: 0.564545 bw: 15.0968 +[3:2] time: 0.564509 bw: 15.0978 +[3:3] time: 0.563897 bw: 15.1141 +[3:4] time: 0.565939 bw: 15.0596 +[4:0] time: 15.5648 bw: 0.54757 +[4:1] time: 0.566016 bw: 15.0576 +[4:2] time: 0.563935 bw: 15.1131 +[4:3] time: 0.564616 bw: 15.0949 +[4:4] time: 0.564514 bw: 15.0976 +[5:0] time: 15.5723 bw: 0.547305 +[5:1] time: 0.564027 bw: 15.1107 +[5:2] time: 0.56434 bw: 15.1023 +[5:3] time: 0.563738 bw: 15.1184 +[5:4] time: 0.565801 bw: 15.0633 +[6:0] time: 15.5523 bw: 0.548009 +[6:1] time: 0.566283 bw: 15.0505 +[6:2] time: 0.565632 bw: 15.0678 +[6:3] time: 0.566279 bw: 15.0506 +[6:4] time: 0.565684 bw: 15.0664 +[7:0] time: 15.5316 bw: 0.54874 +[7:1] time: 0.564823 bw: 15.0894 +[7:2] time: 0.565513 bw: 15.071 +[7:3] time: 0.565421 bw: 15.0734 +[7:4] time: 0.567205 bw: 15.026 +[8:0] time: 15.5539 bw: 0.547954 +[8:1] time: 0.566359 bw: 15.0485 +[8:2] time: 0.565391 bw: 15.0742 +[8:3] time: 0.565871 bw: 15.0614 +[8:4] time: 0.564878 bw: 15.0879 +[9:0] time: 15.5067 bw: 0.549624 +[9:1] time: 0.565169 bw: 15.0801 +[9:2] time: 0.566242 bw: 15.0516 +[9:3] time: 0.565887 bw: 15.061 +[9:4] time: 0.566875 bw: 15.0348 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.566678 +time0: 15.5445 +bw: 15.0415 +bw0: 0.548288 +======================================= + +Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] +Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.667972 +[0:0] time: 0.5193 bw: 16.4121 +[0:1] time: 0.518693 bw: 16.4313 +[0:2] time: 0.51868 bw: 16.4318 +[0:3] time: 0.518733 bw: 16.4301 +[0:4] time: 0.518752 bw: 16.4295 +[1:0] time: 0.520381 bw: 16.3781 +[1:1] time: 0.518597 bw: 16.4344 +[1:2] time: 0.518634 bw: 16.4332 +[1:3] time: 0.518553 bw: 16.4358 +[1:4] time: 0.518552 bw: 16.4358 +[2:0] time: 0.520946 bw: 16.3603 +[2:1] time: 0.518697 bw: 16.4312 +[2:2] time: 0.518714 bw: 16.4307 +[2:3] time: 0.518654 bw: 16.4326 +[2:4] time: 0.518661 bw: 16.4324 +[3:0] time: 0.519394 bw: 16.4092 +[3:1] time: 0.51852 bw: 16.4368 +[3:2] time: 0.518582 bw: 16.4349 +[3:3] time: 0.518563 bw: 16.4355 +[3:4] time: 0.518599 bw: 16.4343 +[4:0] time: 0.519963 bw: 16.3912 +[4:1] time: 0.518634 bw: 16.4332 +[4:2] time: 0.518633 bw: 16.4333 +[4:3] time: 0.518577 bw: 16.435 +[4:4] time: 0.518585 bw: 16.4348 +[5:0] time: 0.520673 bw: 16.3689 +[5:1] time: 0.518647 bw: 16.4328 +[5:2] time: 0.518862 bw: 16.426 +[5:3] time: 0.518672 bw: 16.432 +[5:4] time: 0.518515 bw: 16.437 +[6:0] time: 0.520594 bw: 16.3713 +[6:1] time: 0.51861 bw: 16.434 +[6:2] time: 0.518593 bw: 16.4345 +[6:3] time: 0.518637 bw: 16.4331 +[6:4] time: 0.518613 bw: 16.4339 +[7:0] time: 0.519122 bw: 16.4178 +[7:1] time: 0.518617 bw: 16.4338 +[7:2] time: 0.518567 bw: 16.4353 +[7:3] time: 0.51856 bw: 16.4356 +[7:4] time: 0.518517 bw: 16.4369 +[8:0] time: 0.521128 bw: 16.3546 +[8:1] time: 0.518648 bw: 16.4328 +[8:2] time: 0.518637 bw: 16.4331 +[8:3] time: 0.518654 bw: 16.4326 +[8:4] time: 0.518699 bw: 16.4312 +[9:0] time: 0.520828 bw: 16.364 +[9:1] time: 0.518558 bw: 16.4356 +[9:2] time: 0.518527 bw: 16.4366 +[9:3] time: 0.518639 bw: 16.4331 +[9:4] time: 0.518567 bw: 16.4353 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.518624 +time0: 0.520233 +bw: 16.4335 +bw0: 16.3827 +======================================= + +Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] +Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.653662 +[0:0] time: 0.536277 bw: 15.8926 +[0:1] time: 0.535427 bw: 15.9178 +[0:2] time: 0.535349 bw: 15.9201 +[0:3] time: 0.535339 bw: 15.9204 +[0:4] time: 0.535387 bw: 15.919 +[1:0] time: 0.538748 bw: 15.8197 +[1:1] time: 0.536134 bw: 15.8968 +[1:2] time: 0.536122 bw: 15.8972 +[1:3] time: 0.536122 bw: 15.8972 +[1:4] time: 0.536121 bw: 15.8972 +[2:0] time: 0.538528 bw: 15.8262 +[2:1] time: 0.535429 bw: 15.9178 +[2:2] time: 0.535365 bw: 15.9196 +[2:3] time: 0.535403 bw: 15.9185 +[2:4] time: 0.535437 bw: 15.9175 +[3:0] time: 0.537047 bw: 15.8698 +[3:1] time: 0.536155 bw: 15.8962 +[3:2] time: 0.536211 bw: 15.8945 +[3:3] time: 0.536123 bw: 15.8971 +[3:4] time: 0.536134 bw: 15.8968 +[4:0] time: 0.536608 bw: 15.8828 +[4:1] time: 0.535337 bw: 15.9205 +[4:2] time: 0.535375 bw: 15.9194 +[4:3] time: 0.535415 bw: 15.9182 +[4:4] time: 0.535426 bw: 15.9178 +[5:0] time: 0.540036 bw: 15.782 +[5:1] time: 0.536098 bw: 15.8979 +[5:2] time: 0.536208 bw: 15.8946 +[5:3] time: 0.536152 bw: 15.8963 +[5:4] time: 0.536112 bw: 15.8975 +[6:0] time: 0.536353 bw: 15.8903 +[6:1] time: 0.535434 bw: 15.9176 +[6:2] time: 0.53537 bw: 15.9195 +[6:3] time: 0.535415 bw: 15.9182 +[6:4] time: 0.535406 bw: 15.9184 +[7:0] time: 0.536922 bw: 15.8735 +[7:1] time: 0.53615 bw: 15.8964 +[7:2] time: 0.536102 bw: 15.8978 +[7:3] time: 0.536164 bw: 15.8959 +[7:4] time: 0.536171 bw: 15.8957 +[8:0] time: 0.538298 bw: 15.8329 +[8:1] time: 0.535419 bw: 15.9181 +[8:2] time: 0.535359 bw: 15.9198 +[8:3] time: 0.53539 bw: 15.9189 +[8:4] time: 0.535357 bw: 15.9199 +[9:0] time: 0.538671 bw: 15.822 +[9:1] time: 0.536115 bw: 15.8974 +[9:2] time: 0.53615 bw: 15.8964 +[9:3] time: 0.536071 bw: 15.8987 +[9:4] time: 0.536244 bw: 15.8935 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.535767 +time0: 0.537749 +bw: 15.9077 +bw0: 15.8492 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out new file mode 100644 index 0000000..47fe185 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out @@ -0,0 +1,791 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.716111 +[0:0] time: 0.572177 bw: 14.8954 +[0:1] time: 0.55515 bw: 15.3523 +[0:2] time: 0.546648 bw: 15.5911 +[0:3] time: 0.539229 bw: 15.8056 +[0:4] time: 0.531681 bw: 16.03 +[1:0] time: 0.564211 bw: 15.1057 +[1:1] time: 0.53434 bw: 15.9502 +[1:2] time: 0.531578 bw: 16.0331 +[1:3] time: 0.530334 bw: 16.0707 +[1:4] time: 0.530595 bw: 16.0628 +[2:0] time: 0.562702 bw: 15.1463 +[2:1] time: 0.530771 bw: 16.0574 +[2:2] time: 0.530011 bw: 16.0805 +[2:3] time: 0.530063 bw: 16.0789 +[2:4] time: 0.530208 bw: 16.0745 +[3:0] time: 0.56185 bw: 15.1692 +[3:1] time: 0.529129 bw: 16.1073 +[3:2] time: 0.529142 bw: 16.1069 +[3:3] time: 0.529218 bw: 16.1046 +[3:4] time: 0.528903 bw: 16.1141 +[4:0] time: 0.563964 bw: 15.1124 +[4:1] time: 0.529316 bw: 16.1016 +[4:2] time: 0.529431 bw: 16.0981 +[4:3] time: 0.528864 bw: 16.1153 +[4:4] time: 0.528617 bw: 16.1229 +[5:0] time: 0.563973 bw: 15.1121 +[5:1] time: 0.528717 bw: 16.1198 +[5:2] time: 0.528605 bw: 16.1232 +[5:3] time: 0.528723 bw: 16.1197 +[5:4] time: 0.528914 bw: 16.1138 +[6:0] time: 0.56483 bw: 15.0892 +[6:1] time: 0.530213 bw: 16.0744 +[6:2] time: 0.529582 bw: 16.0935 +[6:3] time: 0.529813 bw: 16.0865 +[6:4] time: 0.529612 bw: 16.0926 +[7:0] time: 0.565838 bw: 15.0623 +[7:1] time: 0.529902 bw: 16.0838 +[7:2] time: 0.531852 bw: 16.0248 +[7:3] time: 0.530277 bw: 16.0724 +[7:4] time: 0.530147 bw: 16.0764 +[8:0] time: 0.566229 bw: 15.0519 +[8:1] time: 0.530155 bw: 16.0761 +[8:2] time: 0.529988 bw: 16.0812 +[8:3] time: 0.529909 bw: 16.0836 +[8:4] time: 0.529843 bw: 16.0856 +[9:0] time: 0.568095 bw: 15.0025 +[9:1] time: 0.530555 bw: 16.064 +[9:2] time: 0.530341 bw: 16.0705 +[9:3] time: 0.530028 bw: 16.08 +[9:4] time: 0.530318 bw: 16.0712 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.531268 +time0: 0.565387 +bw: 16.0438 +bw0: 15.0747 +======================================= + +Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.71178 +[0:0] time: 1.70396 bw: 5.00177 +[0:1] time: 1.70293 bw: 5.00479 +[0:2] time: 1.6719 bw: 5.0977 +[0:3] time: 1.67351 bw: 5.09279 +[0:4] time: 1.67165 bw: 5.09846 +[1:0] time: 1.72297 bw: 4.94659 +[1:1] time: 1.67043 bw: 5.10218 +[1:2] time: 1.67457 bw: 5.08957 +[1:3] time: 1.67374 bw: 5.0921 +[1:4] time: 1.675 bw: 5.08826 +[2:0] time: 1.71821 bw: 4.96028 +[2:1] time: 1.67485 bw: 5.0887 +[2:2] time: 1.67108 bw: 5.10018 +[2:3] time: 1.6739 bw: 5.0916 +[2:4] time: 1.67254 bw: 5.09573 +[3:0] time: 1.7311 bw: 4.92336 +[3:1] time: 1.67282 bw: 5.0949 +[3:2] time: 1.67472 bw: 5.0891 +[3:3] time: 1.67254 bw: 5.09575 +[3:4] time: 1.67546 bw: 5.08687 +[4:0] time: 1.72117 bw: 4.95177 +[4:1] time: 1.67535 bw: 5.08719 +[4:2] time: 1.67402 bw: 5.09123 +[4:3] time: 1.676 bw: 5.08521 +[4:4] time: 1.67294 bw: 5.09452 +[5:0] time: 1.73311 bw: 4.91766 +[5:1] time: 1.67263 bw: 5.09546 +[5:2] time: 1.67451 bw: 5.08974 +[5:3] time: 1.6728 bw: 5.09496 +[5:4] time: 1.6725 bw: 5.09585 +[6:0] time: 1.72353 bw: 4.94497 +[6:1] time: 1.67328 bw: 5.09349 +[6:2] time: 1.67098 bw: 5.1005 +[6:3] time: 1.67246 bw: 5.09598 +[6:4] time: 1.67077 bw: 5.10113 +[7:0] time: 1.7329 bw: 4.91823 +[7:1] time: 1.67098 bw: 5.10048 +[7:2] time: 1.67313 bw: 5.09393 +[7:3] time: 1.67094 bw: 5.10061 +[7:4] time: 1.67307 bw: 5.09412 +[8:0] time: 1.72269 bw: 4.94739 +[8:1] time: 1.67311 bw: 5.09401 +[8:2] time: 1.67077 bw: 5.10112 +[8:3] time: 1.67337 bw: 5.09321 +[8:4] time: 1.67086 bw: 5.10085 +[9:0] time: 1.73321 bw: 4.91737 +[9:1] time: 1.67033 bw: 5.10249 +[9:2] time: 1.67295 bw: 5.09448 +[9:3] time: 1.67019 bw: 5.10291 +[9:4] time: 1.67279 bw: 5.09497 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.67356 +time0: 1.72429 +bw: 5.09268 +bw0: 4.94294 +======================================= + +Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 0.621853 +[0:0] time: 0.527957 bw: 16.143 +[0:1] time: 0.518818 bw: 16.4274 +[0:2] time: 0.519047 bw: 16.4201 +[0:3] time: 0.51914 bw: 16.4172 +[0:4] time: 0.519105 bw: 16.4183 +[1:0] time: 0.525076 bw: 16.2316 +[1:1] time: 0.518668 bw: 16.4321 +[1:2] time: 0.518664 bw: 16.4323 +[1:3] time: 0.518614 bw: 16.4338 +[1:4] time: 0.518686 bw: 16.4316 +[2:0] time: 0.519386 bw: 16.4094 +[2:1] time: 0.518896 bw: 16.4249 +[2:2] time: 0.519053 bw: 16.42 +[2:3] time: 0.519038 bw: 16.4204 +[2:4] time: 0.51902 bw: 16.421 +[3:0] time: 0.519202 bw: 16.4153 +[3:1] time: 0.518679 bw: 16.4318 +[3:2] time: 0.518736 bw: 16.43 +[3:3] time: 0.51866 bw: 16.4324 +[3:4] time: 0.518643 bw: 16.4329 +[4:0] time: 0.519649 bw: 16.4011 +[4:1] time: 0.519031 bw: 16.4206 +[4:2] time: 0.519 bw: 16.4216 +[4:3] time: 0.519053 bw: 16.42 +[4:4] time: 0.519012 bw: 16.4212 +[5:0] time: 0.518999 bw: 16.4217 +[5:1] time: 0.518663 bw: 16.4323 +[5:2] time: 0.518764 bw: 16.4291 +[5:3] time: 0.518685 bw: 16.4316 +[5:4] time: 0.518749 bw: 16.4296 +[6:0] time: 0.519778 bw: 16.397 +[6:1] time: 0.519083 bw: 16.419 +[6:2] time: 0.518857 bw: 16.4262 +[6:3] time: 0.518874 bw: 16.4256 +[6:4] time: 0.518826 bw: 16.4271 +[7:0] time: 0.519398 bw: 16.4091 +[7:1] time: 0.518748 bw: 16.4296 +[7:2] time: 0.518624 bw: 16.4335 +[7:3] time: 0.518654 bw: 16.4326 +[7:4] time: 0.518643 bw: 16.4329 +[8:0] time: 0.519305 bw: 16.412 +[8:1] time: 0.518851 bw: 16.4263 +[8:2] time: 0.518835 bw: 16.4268 +[8:3] time: 0.518836 bw: 16.4268 +[8:4] time: 0.518805 bw: 16.4278 +[9:0] time: 0.519045 bw: 16.4202 +[9:1] time: 0.518683 bw: 16.4317 +[9:2] time: 0.518859 bw: 16.4261 +[9:3] time: 0.518642 bw: 16.433 +[9:4] time: 0.51867 bw: 16.4321 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 0.518823 +time0: 0.520779 +bw: 16.4272 +bw0: 16.366 +======================================= + +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 0.630766 +[0:0] time: 0.554362 bw: 15.3741 +[0:1] time: 0.535603 bw: 15.9126 +[0:2] time: 0.535613 bw: 15.9123 +[0:3] time: 0.535599 bw: 15.9127 +[0:4] time: 0.53562 bw: 15.9121 +[1:0] time: 0.535283 bw: 15.9221 +[1:1] time: 0.535252 bw: 15.923 +[1:2] time: 0.535348 bw: 15.9202 +[1:3] time: 0.535254 bw: 15.923 +[1:4] time: 0.535358 bw: 15.9199 +[2:0] time: 0.535795 bw: 15.9069 +[2:1] time: 0.535623 bw: 15.912 +[2:2] time: 0.535624 bw: 15.9119 +[2:3] time: 0.53564 bw: 15.9115 +[2:4] time: 0.535655 bw: 15.911 +[3:0] time: 0.53696 bw: 15.8724 +[3:1] time: 0.535262 bw: 15.9227 +[3:2] time: 0.535325 bw: 15.9209 +[3:3] time: 0.535266 bw: 15.9226 +[3:4] time: 0.535373 bw: 15.9194 +[4:0] time: 0.541276 bw: 15.7458 +[4:1] time: 0.535568 bw: 15.9136 +[4:2] time: 0.535632 bw: 15.9117 +[4:3] time: 0.535559 bw: 15.9139 +[4:4] time: 0.535605 bw: 15.9125 +[5:0] time: 0.544686 bw: 15.6472 +[5:1] time: 0.535263 bw: 15.9227 +[5:2] time: 0.535302 bw: 15.9215 +[5:3] time: 0.535404 bw: 15.9185 +[5:4] time: 0.535273 bw: 15.9224 +[6:0] time: 0.53593 bw: 15.9029 +[6:1] time: 0.535583 bw: 15.9132 +[6:2] time: 0.53559 bw: 15.913 +[6:3] time: 0.535631 bw: 15.9117 +[6:4] time: 0.535598 bw: 15.9127 +[7:0] time: 0.535719 bw: 15.9091 +[7:1] time: 0.535284 bw: 15.9221 +[7:2] time: 0.535338 bw: 15.9205 +[7:3] time: 0.535289 bw: 15.9219 +[7:4] time: 0.535391 bw: 15.9189 +[8:0] time: 0.536033 bw: 15.8998 +[8:1] time: 0.535615 bw: 15.9122 +[8:2] time: 0.535615 bw: 15.9122 +[8:3] time: 0.535629 bw: 15.9118 +[8:4] time: 0.535633 bw: 15.9117 +[9:0] time: 0.535771 bw: 15.9076 +[9:1] time: 0.535333 bw: 15.9206 +[9:2] time: 0.535343 bw: 15.9203 +[9:3] time: 0.535304 bw: 15.9215 +[9:4] time: 0.535462 bw: 15.9168 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 0.535466 +time0: 0.539181 +bw: 15.9166 +bw0: 15.8088 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out new file mode 100644 index 0000000..31915c9 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out @@ -0,0 +1,1303 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.647464 +[0:0] time: 0.372584 bw: 22.965 +[0:1] time: 0.353028 bw: 24.2371 +[0:2] time: 0.349702 bw: 24.4676 +[0:3] time: 0.339558 bw: 25.1986 +[0:4] time: 0.329432 bw: 25.9732 +[1:0] time: 0.380745 bw: 22.4728 +[1:1] time: 0.325236 bw: 26.3083 +[1:2] time: 0.322189 bw: 26.557 +[1:3] time: 0.322025 bw: 26.5705 +[1:4] time: 0.323599 bw: 26.4413 +[2:0] time: 0.380471 bw: 22.4889 +[2:1] time: 0.320246 bw: 26.7182 +[2:2] time: 0.321058 bw: 26.6506 +[2:3] time: 0.320511 bw: 26.6961 +[2:4] time: 0.320212 bw: 26.721 +[3:0] time: 0.385449 bw: 22.1985 +[3:1] time: 0.320728 bw: 26.678 +[3:2] time: 0.320348 bw: 26.7096 +[3:3] time: 0.320683 bw: 26.6817 +[3:4] time: 0.320752 bw: 26.676 +[4:0] time: 0.379422 bw: 22.5511 +[4:1] time: 0.321257 bw: 26.6341 +[4:2] time: 0.320537 bw: 26.6939 +[4:3] time: 0.320007 bw: 26.7381 +[4:4] time: 0.321417 bw: 26.6208 +[5:0] time: 0.390293 bw: 21.9229 +[5:1] time: 0.320165 bw: 26.7249 +[5:2] time: 0.320817 bw: 26.6706 +[5:3] time: 0.320363 bw: 26.7084 +[5:4] time: 0.32027 bw: 26.7161 +[6:0] time: 0.396585 bw: 21.5752 +[6:1] time: 0.321708 bw: 26.5967 +[6:2] time: 0.321434 bw: 26.6194 +[6:3] time: 0.322314 bw: 26.5467 +[6:4] time: 0.320983 bw: 26.6568 +[7:0] time: 0.389458 bw: 21.97 +[7:1] time: 0.320854 bw: 26.6675 +[7:2] time: 0.321167 bw: 26.6415 +[7:3] time: 0.321532 bw: 26.6113 +[7:4] time: 0.320994 bw: 26.6559 +[8:0] time: 0.40034 bw: 21.3728 +[8:1] time: 0.321736 bw: 26.5944 +[8:2] time: 0.321058 bw: 26.6506 +[8:3] time: 0.321299 bw: 26.6305 +[8:4] time: 0.321324 bw: 26.6285 +[9:0] time: 0.3882 bw: 22.0412 +[9:1] time: 0.321203 bw: 26.6385 +[9:2] time: 0.321865 bw: 26.5838 +[9:3] time: 0.321176 bw: 26.6408 +[9:4] time: 0.321038 bw: 26.6522 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.323396 +time0: 0.386355 +bw: 26.4702 +bw0: 22.1558 +======================================= + +Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.5765 +[0:0] time: 15.5273 bw: 0.551054 +[0:1] time: 0.677222 bw: 12.6345 +[0:2] time: 0.613842 bw: 13.9391 +[0:3] time: 0.596115 bw: 14.3536 +[0:4] time: 0.596758 bw: 14.3381 +[1:0] time: 15.5667 bw: 0.549658 +[1:1] time: 0.594328 bw: 14.3967 +[1:2] time: 0.595266 bw: 14.3741 +[1:3] time: 0.596687 bw: 14.3398 +[1:4] time: 0.59707 bw: 14.3306 +[2:0] time: 15.5455 bw: 0.550409 +[2:1] time: 0.595309 bw: 14.373 +[2:2] time: 0.595669 bw: 14.3643 +[2:3] time: 0.595673 bw: 14.3642 +[2:4] time: 0.594826 bw: 14.3847 +[3:0] time: 15.5411 bw: 0.550565 +[3:1] time: 0.595802 bw: 14.3611 +[3:2] time: 0.596905 bw: 14.3346 +[3:3] time: 0.59564 bw: 14.365 +[3:4] time: 0.595908 bw: 14.3586 +[4:0] time: 15.549 bw: 0.550284 +[4:1] time: 0.596075 bw: 14.3545 +[4:2] time: 0.595597 bw: 14.366 +[4:3] time: 0.595974 bw: 14.357 +[4:4] time: 0.595882 bw: 14.3592 +[5:0] time: 15.4998 bw: 0.552032 +[5:1] time: 0.595161 bw: 14.3766 +[5:2] time: 0.595141 bw: 14.3771 +[5:3] time: 0.595226 bw: 14.375 +[5:4] time: 0.59577 bw: 14.3619 +[6:0] time: 15.6076 bw: 0.548219 +[6:1] time: 0.596822 bw: 14.3366 +[6:2] time: 0.596615 bw: 14.3415 +[6:3] time: 0.597674 bw: 14.3161 +[6:4] time: 0.597416 bw: 14.3223 +[7:0] time: 15.562 bw: 0.549824 +[7:1] time: 0.596654 bw: 14.3406 +[7:2] time: 0.596942 bw: 14.3337 +[7:3] time: 0.597281 bw: 14.3256 +[7:4] time: 0.597715 bw: 14.3152 +[8:0] time: 15.5199 bw: 0.551316 +[8:1] time: 0.596257 bw: 14.3502 +[8:2] time: 0.596307 bw: 14.3489 +[8:3] time: 0.597144 bw: 14.3288 +[8:4] time: 0.597151 bw: 14.3287 +[9:0] time: 15.5165 bw: 0.551436 +[9:1] time: 0.595995 bw: 14.3565 +[9:2] time: 0.596341 bw: 14.3481 +[9:3] time: 0.596625 bw: 14.3413 +[9:4] time: 0.597202 bw: 14.3275 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.5987 +time0: 15.5436 +bw: 14.2975 +bw0: 0.55048 +======================================= + +Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 1.17638 +[0:0] time: 1.03872 bw: 8.23741 +[0:1] time: 1.03853 bw: 8.23894 +[0:2] time: 1.0385 bw: 8.23916 +[0:3] time: 1.03861 bw: 8.23833 +[0:4] time: 1.03853 bw: 8.23893 +[1:0] time: 1.0396 bw: 8.23044 +[1:1] time: 1.03876 bw: 8.2371 +[1:2] time: 1.03882 bw: 8.23666 +[1:3] time: 1.03875 bw: 8.2372 +[1:4] time: 1.03879 bw: 8.23691 +[2:0] time: 1.04504 bw: 8.18763 +[2:1] time: 1.0386 bw: 8.23838 +[2:2] time: 1.03858 bw: 8.23851 +[2:3] time: 1.03857 bw: 8.23864 +[2:4] time: 1.03844 bw: 8.23965 +[3:0] time: 1.04059 bw: 8.2226 +[3:1] time: 1.03876 bw: 8.23713 +[3:2] time: 1.03871 bw: 8.23748 +[3:3] time: 1.03871 bw: 8.23749 +[3:4] time: 1.03872 bw: 8.23745 +[4:0] time: 1.04096 bw: 8.21972 +[4:1] time: 1.03857 bw: 8.23862 +[4:2] time: 1.03851 bw: 8.23913 +[4:3] time: 1.03857 bw: 8.23862 +[4:4] time: 1.03855 bw: 8.23876 +[5:0] time: 1.03913 bw: 8.2342 +[5:1] time: 1.03876 bw: 8.23713 +[5:2] time: 1.03879 bw: 8.2369 +[5:3] time: 1.03878 bw: 8.23697 +[5:4] time: 1.03875 bw: 8.23722 +[6:0] time: 1.04029 bw: 8.225 +[6:1] time: 1.03852 bw: 8.23898 +[6:2] time: 1.03856 bw: 8.23868 +[6:3] time: 1.03856 bw: 8.23872 +[6:4] time: 1.0385 bw: 8.23921 +[7:0] time: 1.03946 bw: 8.23158 +[7:1] time: 1.03874 bw: 8.23726 +[7:2] time: 1.03872 bw: 8.23742 +[7:3] time: 1.03871 bw: 8.23747 +[7:4] time: 1.03871 bw: 8.23751 +[8:0] time: 1.03942 bw: 8.23188 +[8:1] time: 1.03853 bw: 8.2389 +[8:2] time: 1.03871 bw: 8.23754 +[8:3] time: 1.03858 bw: 8.23852 +[8:4] time: 1.03855 bw: 8.23879 +[9:0] time: 1.04101 bw: 8.21927 +[9:1] time: 1.03871 bw: 8.23753 +[9:2] time: 1.03874 bw: 8.23724 +[9:3] time: 1.03877 bw: 8.23701 +[9:4] time: 1.0387 bw: 8.2376 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 1.03865 +time0: 1.04042 +bw: 8.23799 +bw0: 8.22397 +======================================= + +Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 1.19437 +[0:0] time: 1.07402 bw: 7.96667 +[0:1] time: 1.07295 bw: 7.9746 +[0:2] time: 1.07289 bw: 7.97504 +[0:3] time: 1.07288 bw: 7.97513 +[0:4] time: 1.07288 bw: 7.97513 +[1:0] time: 1.07829 bw: 7.93513 +[1:1] time: 1.07365 bw: 7.96942 +[1:2] time: 1.07365 bw: 7.96943 +[1:3] time: 1.07349 bw: 7.97065 +[1:4] time: 1.07351 bw: 7.97049 +[2:0] time: 1.08201 bw: 7.90789 +[2:1] time: 1.07335 bw: 7.97165 +[2:2] time: 1.0735 bw: 7.97052 +[2:3] time: 1.07357 bw: 7.97 +[2:4] time: 1.07354 bw: 7.97023 +[3:0] time: 1.0841 bw: 7.8926 +[3:1] time: 1.07353 bw: 7.97035 +[3:2] time: 1.0736 bw: 7.9698 +[3:3] time: 1.07356 bw: 7.97008 +[3:4] time: 1.07355 bw: 7.97015 +[4:0] time: 1.07738 bw: 7.9418 +[4:1] time: 1.07298 bw: 7.9744 +[4:2] time: 1.07301 bw: 7.97422 +[4:3] time: 1.0729 bw: 7.97497 +[4:4] time: 1.07297 bw: 7.97452 +[5:0] time: 1.07988 bw: 7.92346 +[5:1] time: 1.07352 bw: 7.97037 +[5:2] time: 1.07358 bw: 7.96992 +[5:3] time: 1.07359 bw: 7.96991 +[5:4] time: 1.07364 bw: 7.9695 +[6:0] time: 1.07924 bw: 7.92814 +[6:1] time: 1.07295 bw: 7.97461 +[6:2] time: 1.07296 bw: 7.97458 +[6:3] time: 1.07295 bw: 7.97466 +[6:4] time: 1.073 bw: 7.97424 +[7:0] time: 1.07403 bw: 7.96665 +[7:1] time: 1.07363 bw: 7.96958 +[7:2] time: 1.07363 bw: 7.96956 +[7:3] time: 1.0737 bw: 7.96909 +[7:4] time: 1.07355 bw: 7.97019 +[8:0] time: 1.07536 bw: 7.95675 +[8:1] time: 1.07301 bw: 7.97421 +[8:2] time: 1.07305 bw: 7.9739 +[8:3] time: 1.073 bw: 7.97426 +[8:4] time: 1.07301 bw: 7.97422 +[9:0] time: 1.074 bw: 7.96687 +[9:1] time: 1.07362 bw: 7.96969 +[9:2] time: 1.07359 bw: 7.96988 +[9:3] time: 1.07352 bw: 7.97043 +[9:4] time: 1.07356 bw: 7.97006 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 1.07333 +time0: 1.07783 +bw: 7.97184 +bw0: 7.9386 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out new file mode 100644 index 0000000..eeee8d8 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out @@ -0,0 +1,1303 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.796547 +[0:0] time: 0.627596 bw: 13.6336 +[0:1] time: 0.613907 bw: 13.9376 +[0:2] time: 0.587998 bw: 14.5517 +[0:3] time: 0.576302 bw: 14.8471 +[0:4] time: 0.568177 bw: 15.0594 +[1:0] time: 0.600988 bw: 14.2372 +[1:1] time: 0.56367 bw: 15.1798 +[1:2] time: 0.562442 bw: 15.2129 +[1:3] time: 0.56351 bw: 15.1841 +[1:4] time: 0.562439 bw: 15.213 +[2:0] time: 0.600875 bw: 14.2399 +[2:1] time: 0.562034 bw: 15.2239 +[2:2] time: 0.561991 bw: 15.2251 +[2:3] time: 0.562089 bw: 15.2225 +[2:4] time: 0.561632 bw: 15.2348 +[3:0] time: 0.60018 bw: 14.2564 +[3:1] time: 0.561366 bw: 15.2421 +[3:2] time: 0.562622 bw: 15.208 +[3:3] time: 0.560694 bw: 15.2603 +[3:4] time: 0.561623 bw: 15.2351 +[4:0] time: 0.600334 bw: 14.2527 +[4:1] time: 0.560786 bw: 15.2578 +[4:2] time: 0.56182 bw: 15.2298 +[4:3] time: 0.561802 bw: 15.2302 +[4:4] time: 0.566383 bw: 15.1071 +[5:0] time: 0.603872 bw: 14.1692 +[5:1] time: 0.565397 bw: 15.1334 +[5:2] time: 0.562761 bw: 15.2043 +[5:3] time: 0.561596 bw: 15.2358 +[5:4] time: 0.56222 bw: 15.2189 +[6:0] time: 0.602409 bw: 14.2036 +[6:1] time: 0.563276 bw: 15.1904 +[6:2] time: 0.562935 bw: 15.1996 +[6:3] time: 0.562979 bw: 15.1984 +[6:4] time: 0.563318 bw: 15.1893 +[7:0] time: 0.600828 bw: 14.241 +[7:1] time: 0.562907 bw: 15.2003 +[7:2] time: 0.561572 bw: 15.2365 +[7:3] time: 0.563703 bw: 15.1789 +[7:4] time: 0.561663 bw: 15.234 +[8:0] time: 0.602416 bw: 14.2035 +[8:1] time: 0.562786 bw: 15.2036 +[8:2] time: 0.561753 bw: 15.2316 +[8:3] time: 0.566742 bw: 15.0975 +[8:4] time: 0.564751 bw: 15.1507 +[9:0] time: 0.605774 bw: 14.1247 +[9:1] time: 0.563437 bw: 15.186 +[9:2] time: 0.562667 bw: 15.2068 +[9:3] time: 0.562367 bw: 15.2149 +[9:4] time: 0.56169 bw: 15.2333 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.565095 +time0: 0.604527 +bw: 15.1452 +bw0: 14.1562 +======================================= + +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 1.74898 +[0:0] time: 1.7784 bw: 4.81128 +[0:1] time: 1.73091 bw: 4.94328 +[0:2] time: 1.70511 bw: 5.01809 +[0:3] time: 1.70569 bw: 5.01638 +[0:4] time: 1.70404 bw: 5.02123 +[1:0] time: 1.75407 bw: 4.87803 +[1:1] time: 1.70414 bw: 5.02092 +[1:2] time: 1.70442 bw: 5.02011 +[1:3] time: 1.7041 bw: 5.02104 +[1:4] time: 1.70319 bw: 5.02373 +[2:0] time: 1.7531 bw: 4.88071 +[2:1] time: 1.70602 bw: 5.01539 +[2:2] time: 1.70411 bw: 5.02104 +[2:3] time: 1.70437 bw: 5.02025 +[2:4] time: 1.70437 bw: 5.02026 +[3:0] time: 1.76096 bw: 4.85892 +[3:1] time: 1.70446 bw: 5.02001 +[3:2] time: 1.70352 bw: 5.02275 +[3:3] time: 1.70458 bw: 5.01965 +[3:4] time: 1.70351 bw: 5.02281 +[4:0] time: 1.759 bw: 4.86433 +[4:1] time: 1.70372 bw: 5.02218 +[4:2] time: 1.70359 bw: 5.02254 +[4:3] time: 1.70468 bw: 5.01935 +[4:4] time: 1.70464 bw: 5.01946 +[5:0] time: 1.7659 bw: 4.84535 +[5:1] time: 1.70327 bw: 5.02349 +[5:2] time: 1.70395 bw: 5.0215 +[5:3] time: 1.704 bw: 5.02134 +[5:4] time: 1.70362 bw: 5.02247 +[6:0] time: 1.76219 bw: 4.85554 +[6:1] time: 1.70664 bw: 5.01359 +[6:2] time: 1.70614 bw: 5.01506 +[6:3] time: 1.7062 bw: 5.01489 +[6:4] time: 1.70474 bw: 5.01916 +[7:0] time: 1.76915 bw: 4.83645 +[7:1] time: 1.70669 bw: 5.01344 +[7:2] time: 1.70496 bw: 5.01853 +[7:3] time: 1.70416 bw: 5.02087 +[7:4] time: 1.70457 bw: 5.01968 +[8:0] time: 1.76501 bw: 4.84779 +[8:1] time: 1.70435 bw: 5.02033 +[8:2] time: 1.70506 bw: 5.01824 +[8:3] time: 1.70395 bw: 5.02151 +[8:4] time: 1.70339 bw: 5.02314 +[9:0] time: 1.77053 bw: 4.83268 +[9:1] time: 1.70433 bw: 5.02039 +[9:2] time: 1.7042 bw: 5.02075 +[9:3] time: 1.70535 bw: 5.01738 +[9:4] time: 1.7048 bw: 5.01898 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 1.70519 +time0: 1.76383 +bw: 5.01788 +bw0: 4.85111 +======================================= + +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 1.13626 +[0:0] time: 1.04262 bw: 8.20661 +[0:1] time: 1.03877 bw: 8.23702 +[0:2] time: 1.03879 bw: 8.23684 +[0:3] time: 1.03879 bw: 8.23687 +[0:4] time: 1.03882 bw: 8.23665 +[1:0] time: 1.03984 bw: 8.22859 +[1:1] time: 1.03908 bw: 8.23461 +[1:2] time: 1.03875 bw: 8.23722 +[1:3] time: 1.03852 bw: 8.23905 +[1:4] time: 1.03843 bw: 8.2397 +[2:0] time: 1.04285 bw: 8.20478 +[2:1] time: 1.03876 bw: 8.2371 +[2:2] time: 1.03884 bw: 8.23648 +[2:3] time: 1.03876 bw: 8.23709 +[2:4] time: 1.03935 bw: 8.23242 +[3:0] time: 1.04611 bw: 8.17922 +[3:1] time: 1.0435 bw: 8.19969 +[3:2] time: 1.04532 bw: 8.18541 +[3:3] time: 1.03846 bw: 8.23952 +[3:4] time: 1.03845 bw: 8.23955 +[4:0] time: 1.03935 bw: 8.23242 +[4:1] time: 1.03872 bw: 8.23742 +[4:2] time: 1.03877 bw: 8.23702 +[4:3] time: 1.03876 bw: 8.23708 +[4:4] time: 1.03876 bw: 8.23715 +[5:0] time: 1.04024 bw: 8.22541 +[5:1] time: 1.03847 bw: 8.23942 +[5:2] time: 1.03879 bw: 8.23687 +[5:3] time: 1.03851 bw: 8.23907 +[5:4] time: 1.03849 bw: 8.23926 +[6:0] time: 1.03951 bw: 8.23114 +[6:1] time: 1.03877 bw: 8.237 +[6:2] time: 1.03874 bw: 8.2373 +[6:3] time: 1.03871 bw: 8.2375 +[6:4] time: 1.03873 bw: 8.23738 +[7:0] time: 1.03913 bw: 8.23421 +[7:1] time: 1.03849 bw: 8.23924 +[7:2] time: 1.03855 bw: 8.23881 +[7:3] time: 1.03853 bw: 8.23897 +[7:4] time: 1.03859 bw: 8.23847 +[8:0] time: 1.04172 bw: 8.21372 +[8:1] time: 1.03873 bw: 8.23733 +[8:2] time: 1.03884 bw: 8.23645 +[8:3] time: 1.03874 bw: 8.23724 +[8:4] time: 1.0387 bw: 8.23757 +[9:0] time: 1.03926 bw: 8.23315 +[9:1] time: 1.03867 bw: 8.23782 +[9:2] time: 1.03842 bw: 8.23979 +[9:3] time: 1.03854 bw: 8.23887 +[9:4] time: 1.03859 bw: 8.23846 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 1.03897 +time0: 1.04106 +bw: 8.23542 +bw0: 8.21893 +======================================= + +Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 1.19075 +[0:0] time: 1.08147 bw: 7.91177 +[0:1] time: 1.08162 bw: 7.91067 +[0:2] time: 1.07902 bw: 7.92979 +[0:3] time: 1.0727 bw: 7.97651 +[0:4] time: 1.0727 bw: 7.97648 +[1:0] time: 1.07312 bw: 7.97337 +[1:1] time: 1.07286 bw: 7.97531 +[1:2] time: 1.0729 bw: 7.97503 +[1:3] time: 1.07288 bw: 7.97512 +[1:4] time: 1.07288 bw: 7.97511 +[2:0] time: 1.07338 bw: 7.97144 +[2:1] time: 1.07269 bw: 7.97659 +[2:2] time: 1.07263 bw: 7.97701 +[2:3] time: 1.0727 bw: 7.9765 +[2:4] time: 1.07274 bw: 7.97618 +[3:0] time: 1.07323 bw: 7.97257 +[3:1] time: 1.07295 bw: 7.97466 +[3:2] time: 1.07295 bw: 7.97465 +[3:3] time: 1.07309 bw: 7.97356 +[3:4] time: 1.07296 bw: 7.97457 +[4:0] time: 1.07348 bw: 7.97068 +[4:1] time: 1.07279 bw: 7.97585 +[4:2] time: 1.07277 bw: 7.97598 +[4:3] time: 1.07289 bw: 7.97509 +[4:4] time: 1.07263 bw: 7.97704 +[5:0] time: 1.07332 bw: 7.97192 +[5:1] time: 1.07283 bw: 7.97551 +[5:2] time: 1.07319 bw: 7.97284 +[5:3] time: 1.07285 bw: 7.9754 +[5:4] time: 1.07294 bw: 7.97471 +[6:0] time: 1.07295 bw: 7.97461 +[6:1] time: 1.07268 bw: 7.97662 +[6:2] time: 1.07266 bw: 7.97682 +[6:3] time: 1.07273 bw: 7.97627 +[6:4] time: 1.07273 bw: 7.97627 +[7:0] time: 1.07338 bw: 7.97146 +[7:1] time: 1.07301 bw: 7.97415 +[7:2] time: 1.07294 bw: 7.97471 +[7:3] time: 1.07292 bw: 7.97482 +[7:4] time: 1.07289 bw: 7.97511 +[8:0] time: 1.07297 bw: 7.97451 +[8:1] time: 1.07272 bw: 7.97631 +[8:2] time: 1.07273 bw: 7.9763 +[8:3] time: 1.07271 bw: 7.97645 +[8:4] time: 1.07266 bw: 7.97682 +[9:0] time: 1.07311 bw: 7.97341 +[9:1] time: 1.07287 bw: 7.97521 +[9:2] time: 1.07291 bw: 7.97496 +[9:3] time: 1.07291 bw: 7.97495 +[9:4] time: 1.07285 bw: 7.97535 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 1.0732 +time0: 1.07404 +bw: 7.97278 +bw0: 7.96657 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out new file mode 100644 index 0000000..ee92f00 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out @@ -0,0 +1,2327 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.758111 +[0:0] time: 0.398167 bw: 21.5316 +[0:1] time: 0.395453 bw: 21.6794 +[0:2] time: 0.411099 bw: 20.8542 +[0:3] time: 0.374675 bw: 22.8816 +[0:4] time: 0.355259 bw: 24.1321 +[1:0] time: 0.404274 bw: 21.2063 +[1:1] time: 0.341317 bw: 25.1179 +[1:2] time: 0.340978 bw: 25.1428 +[1:3] time: 0.341165 bw: 25.1291 +[1:4] time: 0.340623 bw: 25.169 +[2:0] time: 0.409561 bw: 20.9325 +[2:1] time: 0.339334 bw: 25.2646 +[2:2] time: 0.339636 bw: 25.2422 +[2:3] time: 0.340022 bw: 25.2135 +[2:4] time: 0.339633 bw: 25.2424 +[3:0] time: 0.407087 bw: 21.0598 +[3:1] time: 0.339931 bw: 25.2203 +[3:2] time: 0.339699 bw: 25.2375 +[3:3] time: 0.339515 bw: 25.2512 +[3:4] time: 0.339692 bw: 25.238 +[4:0] time: 0.406124 bw: 21.1097 +[4:1] time: 0.340403 bw: 25.1853 +[4:2] time: 0.339713 bw: 25.2364 +[4:3] time: 0.339017 bw: 25.2883 +[4:4] time: 0.339863 bw: 25.2254 +[5:0] time: 0.406543 bw: 21.0879 +[5:1] time: 0.340094 bw: 25.2082 +[5:2] time: 0.340103 bw: 25.2076 +[5:3] time: 0.339335 bw: 25.2646 +[5:4] time: 0.33958 bw: 25.2464 +[6:0] time: 0.40838 bw: 20.9931 +[6:1] time: 0.340577 bw: 25.1725 +[6:2] time: 0.341103 bw: 25.1336 +[6:3] time: 0.340724 bw: 25.1616 +[6:4] time: 0.340851 bw: 25.1522 +[7:0] time: 0.413934 bw: 20.7114 +[7:1] time: 0.340765 bw: 25.1586 +[7:2] time: 0.340765 bw: 25.1585 +[7:3] time: 0.340834 bw: 25.1534 +[7:4] time: 0.340645 bw: 25.1675 +[8:0] time: 0.408367 bw: 20.9938 +[8:1] time: 0.340751 bw: 25.1596 +[8:2] time: 0.340564 bw: 25.1734 +[8:3] time: 0.34087 bw: 25.1508 +[8:4] time: 0.340515 bw: 25.177 +[9:0] time: 0.413884 bw: 20.7139 +[9:1] time: 0.340744 bw: 25.1601 +[9:2] time: 0.341396 bw: 25.112 +[9:3] time: 0.340438 bw: 25.1827 +[9:4] time: 0.340947 bw: 25.1451 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.344716 +time0: 0.407632 +bw: 24.9099 +bw0: 21.034 +======================================= + +Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.6412 +[0:0] time: 15.6363 bw: 0.548285 +[0:1] time: 0.623145 bw: 13.7579 +[0:2] time: 0.619911 bw: 13.8297 +[0:3] time: 0.619609 bw: 13.8364 +[0:4] time: 0.617857 bw: 13.8756 +[1:0] time: 15.5781 bw: 0.550333 +[1:1] time: 0.617464 bw: 13.8845 +[1:2] time: 0.621994 bw: 13.7833 +[1:3] time: 0.619007 bw: 13.8499 +[1:4] time: 0.619483 bw: 13.8392 +[2:0] time: 15.5712 bw: 0.550579 +[2:1] time: 0.618117 bw: 13.8698 +[2:2] time: 0.618362 bw: 13.8643 +[2:3] time: 0.618695 bw: 13.8568 +[2:4] time: 0.618004 bw: 13.8723 +[3:0] time: 15.5992 bw: 0.54959 +[3:1] time: 0.617651 bw: 13.8803 +[3:2] time: 0.618077 bw: 13.8707 +[3:3] time: 0.617745 bw: 13.8782 +[3:4] time: 0.618475 bw: 13.8618 +[4:0] time: 15.6196 bw: 0.548873 +[4:1] time: 0.618061 bw: 13.871 +[4:2] time: 0.617581 bw: 13.8818 +[4:3] time: 0.619259 bw: 13.8442 +[4:4] time: 0.61807 bw: 13.8709 +[5:0] time: 15.5974 bw: 0.549653 +[5:1] time: 0.618309 bw: 13.8655 +[5:2] time: 0.61813 bw: 13.8695 +[5:3] time: 0.618115 bw: 13.8699 +[5:4] time: 0.618512 bw: 13.8609 +[6:0] time: 15.5842 bw: 0.55012 +[6:1] time: 0.622479 bw: 13.7726 +[6:2] time: 0.61994 bw: 13.829 +[6:3] time: 0.621177 bw: 13.8015 +[6:4] time: 0.619289 bw: 13.8435 +[7:0] time: 15.6367 bw: 0.548272 +[7:1] time: 0.618717 bw: 13.8564 +[7:2] time: 0.619462 bw: 13.8397 +[7:3] time: 0.619411 bw: 13.8408 +[7:4] time: 0.619985 bw: 13.828 +[8:0] time: 15.6577 bw: 0.547535 +[8:1] time: 0.619204 bw: 13.8454 +[8:2] time: 0.619008 bw: 13.8498 +[8:3] time: 0.619479 bw: 13.8393 +[8:4] time: 0.619244 bw: 13.8446 +[9:0] time: 15.6454 bw: 0.547968 +[9:1] time: 0.61864 bw: 13.8581 +[9:2] time: 0.620228 bw: 13.8226 +[9:3] time: 0.618714 bw: 13.8564 +[9:4] time: 0.619637 bw: 13.8358 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.619106 +time0: 15.6126 +bw: 13.8477 +bw0: 0.549121 +======================================= + +Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 2.21199 +[0:0] time: 2.07827 bw: 4.12515 +[0:1] time: 2.07861 bw: 4.12447 +[0:2] time: 2.07795 bw: 4.12578 +[0:3] time: 2.07792 bw: 4.12583 +[0:4] time: 2.07786 bw: 4.12596 +[1:0] time: 2.08027 bw: 4.12118 +[1:1] time: 2.07816 bw: 4.12537 +[1:2] time: 2.07811 bw: 4.12545 +[1:3] time: 2.07815 bw: 4.12537 +[1:4] time: 2.07829 bw: 4.1251 +[2:0] time: 2.07862 bw: 4.12445 +[2:1] time: 2.07792 bw: 4.12584 +[2:2] time: 2.07783 bw: 4.12601 +[2:3] time: 2.07787 bw: 4.12594 +[2:4] time: 2.07823 bw: 4.12522 +[3:0] time: 2.08091 bw: 4.11991 +[3:1] time: 2.07814 bw: 4.1254 +[3:2] time: 2.07815 bw: 4.12537 +[3:3] time: 2.0781 bw: 4.12548 +[3:4] time: 2.07818 bw: 4.12531 +[4:0] time: 2.07997 bw: 4.12177 +[4:1] time: 2.07789 bw: 4.1259 +[4:2] time: 2.07802 bw: 4.12563 +[4:3] time: 2.07809 bw: 4.12551 +[4:4] time: 2.07782 bw: 4.12604 +[5:0] time: 2.08055 bw: 4.12063 +[5:1] time: 2.0782 bw: 4.12528 +[5:2] time: 2.0783 bw: 4.12508 +[5:3] time: 2.07827 bw: 4.12515 +[5:4] time: 2.07832 bw: 4.12505 +[6:0] time: 2.08008 bw: 4.12155 +[6:1] time: 2.07834 bw: 4.12501 +[6:2] time: 2.07784 bw: 4.126 +[6:3] time: 2.07789 bw: 4.1259 +[6:4] time: 2.07786 bw: 4.12596 +[7:0] time: 2.08229 bw: 4.11718 +[7:1] time: 2.07821 bw: 4.12526 +[7:2] time: 2.0782 bw: 4.12529 +[7:3] time: 2.07814 bw: 4.1254 +[7:4] time: 2.07824 bw: 4.1252 +[8:0] time: 2.08106 bw: 4.11962 +[8:1] time: 2.07785 bw: 4.12597 +[8:2] time: 2.07783 bw: 4.12602 +[8:3] time: 2.07795 bw: 4.12579 +[8:4] time: 2.07789 bw: 4.12589 +[9:0] time: 2.08201 bw: 4.11773 +[9:1] time: 2.07825 bw: 4.12518 +[9:2] time: 2.07817 bw: 4.12535 +[9:3] time: 2.07824 bw: 4.12519 +[9:4] time: 2.07817 bw: 4.12535 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 2.07809 +time0: 2.0804 +bw: 4.12551 +bw0: 4.12092 +======================================= + +Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 2.26663 +[0:0] time: 2.15454 bw: 3.9791 +[0:1] time: 2.15178 bw: 3.98422 +[0:2] time: 2.15178 bw: 3.98421 +[0:3] time: 2.15186 bw: 3.98408 +[0:4] time: 2.15183 bw: 3.98412 +[1:0] time: 2.15656 bw: 3.97538 +[1:1] time: 2.14897 bw: 3.98943 +[1:2] time: 2.14905 bw: 3.98927 +[1:3] time: 2.14897 bw: 3.98942 +[1:4] time: 2.14898 bw: 3.9894 +[2:0] time: 2.15449 bw: 3.97921 +[2:1] time: 2.15189 bw: 3.98401 +[2:2] time: 2.15179 bw: 3.98419 +[2:3] time: 2.1519 bw: 3.98399 +[2:4] time: 2.15173 bw: 3.98431 +[3:0] time: 2.14985 bw: 3.9878 +[3:1] time: 2.14905 bw: 3.98929 +[3:2] time: 2.14906 bw: 3.98925 +[3:3] time: 2.14908 bw: 3.98922 +[3:4] time: 2.14906 bw: 3.98925 +[4:0] time: 2.15416 bw: 3.97981 +[4:1] time: 2.15193 bw: 3.98393 +[4:2] time: 2.15189 bw: 3.98401 +[4:3] time: 2.1518 bw: 3.98418 +[4:4] time: 2.15174 bw: 3.98429 +[5:0] time: 2.15071 bw: 3.98619 +[5:1] time: 2.14929 bw: 3.98882 +[5:2] time: 2.14917 bw: 3.98906 +[5:3] time: 2.14899 bw: 3.98939 +[5:4] time: 2.14905 bw: 3.98928 +[6:0] time: 2.16012 bw: 3.96883 +[6:1] time: 2.15197 bw: 3.98386 +[6:2] time: 2.1519 bw: 3.98399 +[6:3] time: 2.15198 bw: 3.98385 +[6:4] time: 2.15198 bw: 3.98385 +[7:0] time: 2.15243 bw: 3.98302 +[7:1] time: 2.14897 bw: 3.98943 +[7:2] time: 2.14906 bw: 3.98926 +[7:3] time: 2.14895 bw: 3.98947 +[7:4] time: 2.14898 bw: 3.98941 +[8:0] time: 2.15217 bw: 3.9835 +[8:1] time: 2.15188 bw: 3.98402 +[8:2] time: 2.15189 bw: 3.98401 +[8:3] time: 2.15189 bw: 3.98402 +[8:4] time: 2.15189 bw: 3.98401 +[9:0] time: 2.14922 bw: 3.98896 +[9:1] time: 2.14898 bw: 3.9894 +[9:2] time: 2.1492 bw: 3.98899 +[9:3] time: 2.14915 bw: 3.9891 +[9:4] time: 2.14888 bw: 3.9896 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 2.15046 +time0: 2.15343 +bw: 3.98667 +bw0: 3.98118 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out new file mode 100644 index 0000000..6a3aaf4 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out @@ -0,0 +1,4375 @@ +All nodes and tasks are healthy. +All ranks passed the checks. +loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh +Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 0 + +[-1] time: 0.93078 +[0:0] time: 0.573745 bw: 14.9571 +[0:1] time: 0.384237 bw: 22.334 +[0:2] time: 0.382073 bw: 22.4605 +[0:3] time: 0.370926 bw: 23.1355 +[0:4] time: 0.373009 bw: 23.0063 +[1:0] time: 0.417102 bw: 20.5742 +[1:1] time: 0.362265 bw: 23.6886 +[1:2] time: 0.362608 bw: 23.6662 +[1:3] time: 0.363429 bw: 23.6127 +[1:4] time: 0.363704 bw: 23.5948 +[2:0] time: 0.429774 bw: 19.9676 +[2:1] time: 0.362724 bw: 23.6586 +[2:2] time: 0.362866 bw: 23.6494 +[2:3] time: 0.363263 bw: 23.6235 +[2:4] time: 0.362008 bw: 23.7054 +[3:0] time: 0.429733 bw: 19.9695 +[3:1] time: 0.361766 bw: 23.7213 +[3:2] time: 0.362216 bw: 23.6918 +[3:3] time: 0.362651 bw: 23.6634 +[3:4] time: 0.362349 bw: 23.6831 +[4:0] time: 0.434529 bw: 19.7491 +[4:1] time: 0.362643 bw: 23.6639 +[4:2] time: 0.361182 bw: 23.7596 +[4:3] time: 0.363303 bw: 23.6209 +[4:4] time: 0.361469 bw: 23.7408 +[5:0] time: 0.43231 bw: 19.8504 +[5:1] time: 0.362727 bw: 23.6584 +[5:2] time: 0.361562 bw: 23.7347 +[5:3] time: 0.362921 bw: 23.6458 +[5:4] time: 0.36226 bw: 23.6889 +[6:0] time: 0.430324 bw: 19.942 +[6:1] time: 0.364444 bw: 23.547 +[6:2] time: 0.362648 bw: 23.6636 +[6:3] time: 0.363774 bw: 23.5903 +[6:4] time: 0.364418 bw: 23.5486 +[7:0] time: 0.434093 bw: 19.7689 +[7:1] time: 0.366189 bw: 23.4348 +[7:2] time: 0.36353 bw: 23.6062 +[7:3] time: 0.36432 bw: 23.555 +[7:4] time: 0.364396 bw: 23.5501 +[8:0] time: 0.431242 bw: 19.8996 +[8:1] time: 0.364277 bw: 23.5577 +[8:2] time: 0.362703 bw: 23.66 +[8:3] time: 0.363077 bw: 23.6356 +[8:4] time: 0.363684 bw: 23.5962 +[9:0] time: 0.469404 bw: 18.2818 +[9:1] time: 0.372588 bw: 23.0323 +[9:2] time: 0.362764 bw: 23.656 +[9:3] time: 0.367559 bw: 23.3474 +[9:4] time: 0.380095 bw: 22.5774 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 0 +time: 0.365266 +time0: 0.448226 +bw: 23.4991 +bw0: 19.296 +======================================= + +Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 0 + +[-1] time: 15.7304 +[0:0] time: 15.6843 bw: 0.547144 +[0:1] time: 0.636695 bw: 13.4783 +[0:2] time: 0.637389 bw: 13.4636 +[0:3] time: 0.638144 bw: 13.4477 +[0:4] time: 0.639936 bw: 13.41 +[1:0] time: 15.5927 bw: 0.550356 +[1:1] time: 0.636032 bw: 13.4923 +[1:2] time: 0.646497 bw: 13.2739 +[1:3] time: 0.638227 bw: 13.4459 +[1:4] time: 0.636875 bw: 13.4745 +[2:0] time: 15.5966 bw: 0.55022 +[2:1] time: 0.635211 bw: 13.5098 +[2:2] time: 0.636567 bw: 13.481 +[2:3] time: 0.637265 bw: 13.4662 +[2:4] time: 0.636442 bw: 13.4836 +[3:0] time: 15.6246 bw: 0.549234 +[3:1] time: 0.634735 bw: 13.5199 +[3:2] time: 0.635924 bw: 13.4946 +[3:3] time: 0.635061 bw: 13.513 +[3:4] time: 0.635848 bw: 13.4962 +[4:0] time: 15.5765 bw: 0.55093 +[4:1] time: 0.635537 bw: 13.5028 +[4:2] time: 0.635678 bw: 13.4998 +[4:3] time: 0.63573 bw: 13.4987 +[4:4] time: 0.636334 bw: 13.4859 +[5:0] time: 15.6045 bw: 0.549939 +[5:1] time: 0.635054 bw: 13.5131 +[5:2] time: 0.636599 bw: 13.4803 +[5:3] time: 0.636055 bw: 13.4918 +[5:4] time: 0.63568 bw: 13.4998 +[6:0] time: 15.6929 bw: 0.546842 +[6:1] time: 0.636633 bw: 13.4796 +[6:2] time: 0.637347 bw: 13.4645 +[6:3] time: 0.636632 bw: 13.4796 +[6:4] time: 0.637188 bw: 13.4678 +[7:0] time: 15.6332 bw: 0.548931 +[7:1] time: 0.638007 bw: 13.4505 +[7:2] time: 0.638792 bw: 13.434 +[7:3] time: 0.639163 bw: 13.4262 +[7:4] time: 0.639133 bw: 13.4269 +[8:0] time: 15.6072 bw: 0.549845 +[8:1] time: 0.636126 bw: 13.4903 +[8:2] time: 0.636368 bw: 13.4852 +[8:3] time: 0.636972 bw: 13.4724 +[8:4] time: 0.636832 bw: 13.4754 +[9:0] time: 15.6575 bw: 0.548079 +[9:1] time: 0.636896 bw: 13.474 +[9:2] time: 0.636869 bw: 13.4746 +[9:3] time: 0.638079 bw: 13.449 +[9:4] time: 0.636434 bw: 13.4838 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 0 +time: 0.637025 +time0: 15.627 +bw: 13.4714 +bw0: 0.549152 +======================================= + +Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 0 +naive algo: 1 + +[-1] time: 6.78062 +[0:0] time: 7.83042 bw: 1.09592 +[0:1] time: 7.95677 bw: 1.07852 +[0:2] time: 8.0796 bw: 1.06213 +[0:3] time: 8.22408 bw: 1.04347 +[0:4] time: 8.40486 bw: 1.02102 +[1:0] time: 8.36945 bw: 1.02534 +[1:1] time: 8.68631 bw: 0.987939 +[1:2] time: 8.74781 bw: 0.980993 +[1:3] time: 9.07966 bw: 0.94514 +[1:4] time: 8.93379 bw: 0.960571 +[2:0] time: 9.00591 bw: 0.95288 +[2:1] time: 9.02889 bw: 0.950454 +[2:2] time: 9.089 bw: 0.944168 +[2:3] time: 9.19958 bw: 0.932819 +[2:4] time: 9.24249 bw: 0.928489 +[3:0] time: 9.29612 bw: 0.923132 +[3:1] time: 9.38297 bw: 0.914587 +[3:2] time: 9.26814 bw: 0.925919 +[3:3] time: 9.2751 bw: 0.925224 +[3:4] time: 9.29843 bw: 0.922903 +[4:0] time: 8.91455 bw: 0.962645 +[4:1] time: 9.31935 bw: 0.920831 +[4:2] time: 9.35474 bw: 0.917348 +[4:3] time: 9.48555 bw: 0.904697 +[4:4] time: 9.59272 bw: 0.89459 +[5:0] time: 9.44209 bw: 0.908861 +[5:1] time: 9.38511 bw: 0.914379 +[5:2] time: 9.48948 bw: 0.904322 +[5:3] time: 9.27111 bw: 0.925622 +[5:4] time: 9.12718 bw: 0.940219 +[6:0] time: 9.34983 bw: 0.917829 +[6:1] time: 9.32988 bw: 0.919792 +[6:2] time: 9.34334 bw: 0.918466 +[6:3] time: 9.40047 bw: 0.912884 +[6:4] time: 9.51846 bw: 0.901569 +[7:0] time: 9.5379 bw: 0.899731 +[7:1] time: 9.63821 bw: 0.890367 +[7:2] time: 9.41054 bw: 0.911908 +[7:3] time: 9.4825 bw: 0.904988 +[7:4] time: 9.47691 bw: 0.905521 +[8:0] time: 9.41786 bw: 0.911199 +[8:1] time: 9.45006 bw: 0.908095 +[8:2] time: 9.38986 bw: 0.913916 +[8:3] time: 9.38717 bw: 0.914178 +[8:4] time: 9.42353 bw: 0.910651 +[9:0] time: 9.50029 bw: 0.903293 +[9:1] time: 9.54789 bw: 0.89879 +[9:2] time: 9.46385 bw: 0.906771 +[9:3] time: 9.49768 bw: 0.903542 +[9:4] time: 9.53211 bw: 0.900277 + +======================================= +type: device +size: 4294967296 +pad: 0 +naive: 1 +time: 9.20538 +time0: 9.06644 +bw: 0.934202 +bw0: 0.950084 +======================================= + +Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] +Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +mem_type: device +size: 1073741824 +padding: 1 +naive algo: 1 + +[-1] time: 8.20451 +[0:0] time: 9.61907 bw: 0.892139 +[0:1] time: 9.44606 bw: 0.908479 +[0:2] time: 9.4207 bw: 0.910924 +[0:3] time: 9.4264 bw: 0.910374 +[0:4] time: 9.45642 bw: 0.907483 +[1:0] time: 9.61165 bw: 0.892828 +[1:1] time: 9.47711 bw: 0.905502 +[1:2] time: 9.51776 bw: 0.901635 +[1:3] time: 9.54085 bw: 0.899453 +[1:4] time: 9.8328 bw: 0.872747 +[2:0] time: 9.60359 bw: 0.893577 +[2:1] time: 9.49282 bw: 0.904004 +[2:2] time: 9.51094 bw: 0.902282 +[2:3] time: 9.64736 bw: 0.889523 +[2:4] time: 9.73577 bw: 0.881445 +[3:0] time: 9.737 bw: 0.881334 +[3:1] time: 9.9104 bw: 0.865913 +[3:2] time: 10.0438 bw: 0.854412 +[3:3] time: 9.6104 bw: 0.892943 +[3:4] time: 9.73436 bw: 0.881573 +[4:0] time: 9.78161 bw: 0.877314 +[4:1] time: 9.64655 bw: 0.889597 +[4:2] time: 9.68356 bw: 0.886198 +[4:3] time: 9.69721 bw: 0.88495 +[4:4] time: 9.73303 bw: 0.881693 +[5:0] time: 9.65955 bw: 0.8884 +[5:1] time: 9.70502 bw: 0.884238 +[5:2] time: 9.75326 bw: 0.879864 +[5:3] time: 9.33703 bw: 0.919087 +[5:4] time: 9.57752 bw: 0.896009 +[6:0] time: 9.59446 bw: 0.894427 +[6:1] time: 9.48658 bw: 0.904598 +[6:2] time: 9.53572 bw: 0.899937 +[6:3] time: 9.55059 bw: 0.898536 +[6:4] time: 9.54488 bw: 0.899073 +[7:0] time: 9.45357 bw: 0.907757 +[7:1] time: 9.13632 bw: 0.939278 +[7:2] time: 9.61145 bw: 0.892846 +[7:3] time: 9.55951 bw: 0.897697 +[7:4] time: 9.52563 bw: 0.90089 +[8:0] time: 9.20274 bw: 0.932499 +[8:1] time: 9.55038 bw: 0.898556 +[8:2] time: 9.27045 bw: 0.925688 +[8:3] time: 9.39211 bw: 0.913697 +[8:4] time: 9.35636 bw: 0.917188 +[9:0] time: 9.17322 bw: 0.9355 +[9:1] time: 9.18198 bw: 0.934608 +[9:2] time: 9.34243 bw: 0.918556 +[9:3] time: 8.99746 bw: 0.953774 +[9:4] time: 9.24351 bw: 0.928386 + +======================================= +type: device +size: 4294967296 +pad: 4 +naive: 1 +time: 9.53056 +time0: 9.54365 +bw: 0.900841 +bw0: 0.899577 +======================================= + diff --git a/allreduce-perf/mpi-cpp/run/plot.py b/allreduce-perf/mpi-cpp/run/plot.py index 989ceeb..30d31aa 100644 --- a/allreduce-perf/mpi-cpp/run/plot.py +++ b/allreduce-perf/mpi-cpp/run/plot.py @@ -5,27 +5,30 @@ # List of files to process file_paths = [ - 'data/job_n_00004_N_0001_TPN_4.out', - 'data/job_n_00004_N_0004_TPN_1.out', - 'data/job_n_00016_N_0004_TPN_4.out', - 'data/job_n_00016_N_0016_TPN_1.out', - 'data/job_n_00032_N_0008_TPN_4.out', - 'data/job_n_00032_N_0032_TPN_1.out', - 'data/job_n_00064_N_0016_TPN_4.out', - 'data/job_n_00064_N_0064_TPN_1.out', - 'data/job_n_00128_N_0032_TPN_4.out', - 'data/job_n_00128_N_0128_TPN_1.out', - 'data/job_n_00256_N_0064_TPN_4.out', - 'data/job_n_00256_N_0256_TPN_1.out', + "data/job_n_00004_N_0001_TPN_4.out", + "data/job_n_00004_N_0004_TPN_1.out", + "data/job_n_00016_N_0004_TPN_4.out", + "data/job_n_00016_N_0016_TPN_1.out", + "data/job_n_00032_N_0008_TPN_4.out", + "data/job_n_00032_N_0032_TPN_1.out", + "data/job_n_00064_N_0016_TPN_4.out", + "data/job_n_00064_N_0064_TPN_1.out", + "data/job_n_00128_N_0032_TPN_4.out", + "data/job_n_00128_N_0128_TPN_1.out", + "data/job_n_00256_N_0064_TPN_4.out", + "data/job_n_00256_N_0256_TPN_1.out", + "data/job_n_00512_N_0128_TPN_4.out", + "data/job_n_01024_N_0256_TPN_4.out", ] # Output directory for the plots -output_dir = 'plots' +output_dir = "plots" + # Function to parse integers from the filename def parse_filename(file_path): filename = os.path.basename(file_path) - pattern = re.compile(r'job_n_(\d+)_N_(\d+)_TPN_(\d+)\.out') + pattern = re.compile(r"job_n_(\d+)_N_(\d+)_TPN_(\d+)\.out") match = pattern.match(filename) if match: n = int(match.group(1)) @@ -35,168 +38,299 @@ def parse_filename(file_path): else: raise ValueError(f"Filename {filename} does not match the expected pattern") + # Function to parse a single file and extract the blocks def parse_file(file_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: content = file.read() - + # Regex pattern to match the blocks - pattern = re.compile(r'={39}\ntype:\s*(\w+)\nsize:\s*([0-9.]+)\npad:\s*([0-9.]+)\nnaive:\s*([0-9.]+)\ntime:\s*([0-9.]+)\ntime0:\s*([0-9.]+)\nbw:\s*([0-9.]+)\nbw0:\s*([0-9.]+)\n={39}') + pattern = re.compile( + r"={39}\ntype:\s*(\w+)\nsize:\s*([0-9.]+)\npad:\s*([0-9.]+)\nnaive:\s*([0-9.]+)\ntime:\s*([0-9.]+)\ntime0:\s*([0-9.]+)\nbw:\s*([0-9.]+)\nbw0:\s*([0-9.]+)\n={39}" + ) blocks = pattern.findall(content) - + # Extracting data into a list of dictionaries data = [] for block in blocks: - data.append({ - 'type': block[0], - 'size': float(block[1]), - 'pad': int(block[2]), - 'naive': int(block[3]), - 'time': float(block[4]), - 'time0': float(block[5]), - 'bw': float(block[6]), - 'bw0': float(block[7]) - }) - + data.append( + { + "type": block[0], + "size": float(block[1]), + "pad": int(block[2]), + "naive": int(block[3]), + "time": float(block[4]), + "time0": float(block[5]), + "bw": float(block[6]), + "bw0": float(block[7]), + } + ) return data + # Function to generate block names based on type, naive, and pad def create_names(block): base_name = f"{block['type']}_memory" block_name = base_name - if block['naive'] == 1: + if block["naive"] == 1: block_name += "_naive" small_block_name = "naive" else: block_name += "_mpi" small_block_name = "mpi" - if block['pad'] != 0: + if block["pad"] != 0: block_name += "_pad" small_block_name += "_pad" return base_name, block_name, small_block_name + +def plot_data_2(grouped_lines, base_name, ns, tpn, pad, bw0, combined=False): + colors = sns.color_palette("husl", len(grouped_lines) + 4) + markers = ["o", "s", "D", "^", "v", "p", "*", "X"] + markers = [ + "o", + "p", + "s", + "D", + "^", + "v", + ">", + "<", + "d", + "P", + "h", + "x", + "|", + "H", + "+", + "_", + ] + line_styles = ["-", "--", "-.", ":", "-", "--", "-.", ":"] + alpha = 0.7 + markersize = 10 + + if combined: + title = f"{base_name}: allreduce bus-bandwidth (4 GiB fp32)" + filename = f"{base_name}" + else: + title = f"{base_name}: allreduce bus-bandwidth (4 GiB fp32, {tpn} ranks/node)" + filename = f"{base_name}_{tpn}" + if pad: + title += ": padded MPI alloc" + filename += "_pad" + if bw0: + title += ", with startup iteration" + filename += "_startup" + + print(f" plotting {title} and saving in {filename}") + + plt.figure(figsize=(10, 6)) + plt.title(title) + plt.ylabel("bandwidth [GB/s]") + plt.xlabel("number of ranks") + plt.xticks(sorted(set(ns))) + plt.grid(True) + plt.yscale("log") + plt.xscale("log") + + idx = 0 + l = 0 + for small_block_name, data in grouped_lines.items(): + if combined or ( + data["tpn"] == tpn + and ( + (pad and ("pad" in small_block_name)) + or (not pad and (not "pad" in small_block_name)) + ) + ): + color = colors[idx % len(colors)] + marker = markers[idx % len(markers)] + line_style = line_styles[idx % len(line_styles)] + plt.plot( + data["n"], + data["bw"], + color=color, + marker=marker, + linestyle=line_style, + label=f"{small_block_name}", + alpha=alpha, + markersize=markersize, + ) + l = l + 1 + if bw0: + plt.scatter( + data["n"], + data["bw0"], + facecolors="none", + edgecolors=color, + marker=marker, + label=f"{small_block_name}_0", + alpha=alpha, + s=markersize * 10, + ) + l = l + 1 + idx = idx + 1 + + if base_name == "device_memory": + plt.ylim([0.4, 500]) + plt.xlim([3, 1500]) + + if combined or tpn == 4: + n_nccl = [4, 16, 32, 64, 128, 256, 512, 1024] + bw_nccl = [344.23, 91.29, 91.22, 91.36, 91.39, 91.43, 91.42, 91.08] + plt.plot( + n_nccl, + bw_nccl, + color=colors[idx % len(colors)], + marker=markers[idx % len(markers)], + linestyle=line_styles[idx % len(line_styles)], + label=f"NCCL-Tests (ranks/node=4)", + alpha=alpha, + markersize=markersize, + ) + idx = idx + 1 + + if combined or tpn == 1: + n_nccl = [4, 16, 32, 64, 128, 256] + bw_nccl = [22.84, 22.87, 22.89, 22.90, 22.90, 22.92] + plt.plot( + n_nccl, + bw_nccl, + color=colors[idx % len(colors)], + marker=markers[idx % len(markers)], + linestyle=line_styles[idx % len(line_styles)], + label=f"NCCL/uenv (ranks/node=1)", + alpha=alpha, + markersize=markersize, + ) + idx = idx + 1 + + if combined or tpn == 4: + n_nccl = [4, 16, 32, 64] + bw_nccl = [343.360, 91.740, 91.673, 91.646] + plt.plot( + n_nccl, + bw_nccl, + color=colors[idx % len(colors)], + marker=markers[idx % len(markers)], + linestyle=line_styles[idx % len(line_styles)], + label=f"NCCL/container (ranks/node=4)", + alpha=alpha, + markersize=markersize, + ) + idx = idx + 1 + + if combined or tpn == 4: + n_nccl = [4, 16, 32, 64, 128, 256] + bw_nccl = [342.326, 91.305, 91.311, 91.331, 91.437, 91.462] + plt.plot( + n_nccl, + bw_nccl, + color=colors[idx % len(colors)], + marker=markers[idx % len(markers)], + linestyle=line_styles[idx % len(line_styles)], + label=f"NCCL/uenv (ranks/node=4)", + alpha=alpha, + markersize=markersize, + ) + idx = idx + 1 + + # Add custom legend entries + handles, labels = plt.gca().get_legend_handles_labels() + unique_labels = list( + dict.fromkeys(labels) + ) # Remove duplicates while preserving order + if bw0: + unique_handles = [ + handles[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >= l + ] # Keep every second handle + unique_labels = [ + unique_labels[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >= l + ] # Keep every second label + plt.legend( + unique_handles, + unique_labels, + fontsize="small", + loc="center left", + bbox_to_anchor=(1, 0.5), + ) + else: + unique_handles = [handles[i] for i in range(len(unique_labels))] + unique_labels = [unique_labels[i] for i in range(len(unique_labels))] + plt.legend( + unique_handles, + unique_labels, + fontsize="small", + loc="center left", + bbox_to_anchor=(1, 0.5), + ) + + # Use tight layout + plt.tight_layout(rect=[0, 0, 1, 1]) + + # Save the plot to a file + plt.savefig( + os.path.join(output_dir, f"{filename}.png"), dpi=300, bbox_inches="tight" + ) + plt.savefig( + os.path.join(output_dir, f"{filename}.svg"), dpi=300, bbox_inches="tight" + ) + plt.close() + + # Function to plot the data blocks and save to files def plot_data(blocks, ns, tpns, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) - + # Create a dictionary to group blocks by their base names grouped_blocks = {} for file_blocks in blocks: for block in file_blocks: base_name, block_name, small_block_name = create_names(block) - print(base_name, block_name, small_block_name) if base_name not in grouped_blocks: grouped_blocks[base_name] = [] - grouped_blocks[base_name].append((small_block_name, block, ns[blocks.index(file_blocks)], tpns[blocks.index(file_blocks)])) + grouped_blocks[base_name].append( + ( + small_block_name, + block, + ns[blocks.index(file_blocks)], + tpns[blocks.index(file_blocks)], + ) + ) for base_name, block_data in grouped_blocks.items(): print(f"plotting {base_name}") - + + # Create a dictionary to group lines by their (short) block name grouped_lines = {} for idx, (small_block_name, block, n, tpn) in enumerate(block_data): - #color = colors[idx % len(colors)] small_block_name = f"{small_block_name} (ranks/node={tpn})" - print(small_block_name) - print("") if small_block_name not in grouped_lines: - grouped_lines[small_block_name] = { 'tpn' : tpn, 'n' : [], 'bw' : [], 'bw0' : [] } - grouped_lines[small_block_name]['n'].append(n) - grouped_lines[small_block_name]['bw'].append(block['bw']) - grouped_lines[small_block_name]['bw0'].append(block['bw0']) - - print(grouped_lines) - - colors = sns.color_palette("husl", len(grouped_lines)+4) - #colors = sns.color_palette("tab10") - markers = ['o', 's', 'D', '^', 'v', 'p', '*', 'X'] - markers = ['o', 'p', 's', 'D', '^', 'v', '>', '<', 'd', 'P', 'h', 'x', '|', 'H', '+', '_'] - line_styles = ['-', '--', '-.', ':', '-', '--', '-.', ':'] - alpha=0.7 - markersize=10 - #line_styles = ['-', '--', '-.', ':', (0, (3, 5, 1, 5)), (0, (1, 10)), (0, (3, 10, 1, 10)), (0, (5, 10)), (0, (3, 1, 1, 1)), (0, (5, 1, 1, 1))] - - plt.figure(figsize=(10, 6)) - plt.title(base_name) - plt.ylabel('bandwidth [GB/s]') - plt.xlabel('number of ranks') - plt.xticks(sorted(set(ns))) - plt.grid(True) # Enable grid lines - plt.yscale('log') - plt.xscale('log') - - idx = 0 - l = 0 - for small_block_name, data in grouped_lines.items(): - #if idx==0 or idx==2: - #if idx==0 or idx==1 or idx==2 or idx==3: - if True: - #if idx==4 or idx==6: - #if idx==4 or idx==5 or idx==6 or idx==7: - - color = colors[idx % len(colors)] - marker = markers[idx % len(markers)] - line_style = line_styles[idx % len(line_styles)] - plt.plot(data['n'], data['bw'], color=color, marker=marker, linestyle=line_style, label=f'{small_block_name}', alpha=alpha, markersize=markersize) - l = l+1 - plt.scatter(data['n'], data['bw0'], facecolors ='none', edgecolors=color, marker=marker, label=f'{small_block_name}_0', alpha=alpha, s=markersize*10) - l = l+1 - idx = idx + 1 - - if base_name == "device_memory": - - plt.ylim([4,5000]) + grouped_lines[small_block_name] = { + "tpn": tpn, + "n": [], + "bw": [], + "bw0": [], + } + grouped_lines[small_block_name]["n"].append(n) + grouped_lines[small_block_name]["bw"].append(block["bw"]) + grouped_lines[small_block_name]["bw0"].append(block["bw0"]) - n_nccl = [4, 16, 32, 64] - bw_nccl = [343.360, 91.740, 91.673, 91.646] - plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f'NCCL/container (ranks/node=4)', alpha=alpha, markersize=markersize) - idx = idx + 1 + # plot different selections - n_nccl = [4, 16, 32, 64, 128, 256] - bw_nccl = [342.326, 91.305, 91.311, 91.331, 91.437, 91.462] - plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f'NCCL/uenv (ranks/node=4)', alpha=alpha, markersize=markersize) - idx = idx + 1 - - n_nccl = [4, 16, 32, 64] - bw_nccl = [22.846, 22.869, 22.889, 22.896] - plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f'NCCL/uenv (ranks/node=1)', alpha=alpha, markersize=markersize) - idx = idx + 1 - - n_nccl = [4, 16, 32, 64, 128, 256] - bw_nccl = [343.613, 91.2704, 91.2404, 91.339, 91.3899, 91.4304] - print(n_nccl) - print(bw_nccl) - plt.plot(n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f'NCCL-Tests (ranks/node=4)', alpha=alpha, markersize=markersize) - idx = idx + 1 - - # - ### Add custom legend entries for TPN - ##handles, labels = plt.gca().get_legend_handles_labels() - ##unique_labels = list(dict.fromkeys(labels)) # Remove duplicates while preserving order - ##plt.legend(handles[:len(unique_labels)], unique_labels, loc='best') - # Add custom legend entries - handles, labels = plt.gca().get_legend_handles_labels() - unique_labels = list(dict.fromkeys(labels)) # Remove duplicates while preserving order - print(unique_labels) - unique_handles = [handles[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >=l] # Keep every second handle - unique_labels = [unique_labels[i] for i in range(len(unique_labels)) if i % 2 == 0 or i >=l] # Keep every second label - #unique_handles = [handles[i] for i in range(len(unique_labels))] # Keep every second handle - #unique_labels = [unique_labels[i] for i in range(len(unique_labels))] # Keep every second label - plt.legend(unique_handles, unique_labels, fontsize='small', - loc='center left', bbox_to_anchor=(1, 0.5)) + plot_data_2(grouped_lines, base_name, ns, 4, False, False) + plot_data_2(grouped_lines, base_name, ns, 4, False, True) + plot_data_2(grouped_lines, base_name, ns, 4, True, False) + plot_data_2(grouped_lines, base_name, ns, 4, True, True) - - # Use tight layout - plt.tight_layout(rect=[0, 0, 1, 1]) + plot_data_2(grouped_lines, base_name, ns, 1, False, False) + plot_data_2(grouped_lines, base_name, ns, 1, False, True) + plot_data_2(grouped_lines, base_name, ns, 1, True, False) + plot_data_2(grouped_lines, base_name, ns, 1, True, True) - # Save the plot to a file - output_file = os.path.join(output_dir, f'{base_name}.png') - plt.savefig(output_file, dpi=300) - plt.close() + plot_data_2(grouped_lines, base_name, ns, 0, True, True, True) + plot_data_2(grouped_lines, base_name, ns, 0, True, False, True) # Main function to process multiple files @@ -204,15 +338,16 @@ def main(file_paths, output_dir): all_blocks = [] ns = [] tpns = [] - + for file_path in file_paths: n, _, tpn = parse_filename(file_path) blocks = parse_file(file_path) ns.append(n) tpns.append(tpn) all_blocks.append(blocks) - + plot_data(all_blocks, ns, tpns, output_dir) + # Run the main function main(file_paths, output_dir) diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory.svg new file mode 100644 index 0000000..b1b0b68 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory.svg @@ -0,0 +1,2199 @@ + + + + + + + + 2024-06-03T10:00:36.159442 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg new file mode 100644 index 0000000..6bb5ea3 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg @@ -0,0 +1,1631 @@ + + + + + + + + 2024-06-03T10:00:32.883918 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg new file mode 100644 index 0000000..af399e4 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg @@ -0,0 +1,1701 @@ + + + + + + + + 2024-06-03T10:00:34.088510 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg new file mode 100644 index 0000000..366f644 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg @@ -0,0 +1,1801 @@ + + + + + + + + 2024-06-03T10:00:34.716763 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg new file mode 100644 index 0000000..c87042b --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg @@ -0,0 +1,1731 @@ + + + + + + + + 2024-06-03T10:00:33.501859 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg new file mode 100644 index 0000000..1b2a043 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg @@ -0,0 +1,1782 @@ + + + + + + + + 2024-06-03T10:00:30.344287 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg new file mode 100644 index 0000000..860d6ec --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg @@ -0,0 +1,1848 @@ + + + + + + + + 2024-06-03T10:00:31.626443 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg new file mode 100644 index 0000000..76cfe22 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg @@ -0,0 +1,1965 @@ + + + + + + + + 2024-06-03T10:00:32.298486 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg new file mode 100644 index 0000000..0fd7f55 --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg @@ -0,0 +1,1903 @@ + + + + + + + + 2024-06-03T10:00:31.007817 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg new file mode 100644 index 0000000..71d9d7b --- /dev/null +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg @@ -0,0 +1,2574 @@ + + + + + + + + 2024-06-03T10:00:35.420229 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 419cfcae11f09398753965c0b410454236cbe040 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:09:01 +0200 Subject: [PATCH 5/8] better run scripts --- allreduce-perf/mpi-cpp/run/run.sh | 69 +++++++++++++++++++---- allreduce-perf/nccl-pytorch/run/run.sh | 70 ++++++++++++++++++++---- allreduce-perf/nccl-tests-cpp/run/run.sh | 66 +++++++++++++++++++--- 3 files changed, 174 insertions(+), 31 deletions(-) diff --git a/allreduce-perf/mpi-cpp/run/run.sh b/allreduce-perf/mpi-cpp/run/run.sh index fdce8b4..2771d7b 100755 --- a/allreduce-perf/mpi-cpp/run/run.sh +++ b/allreduce-perf/mpi-cpp/run/run.sh @@ -23,25 +23,72 @@ sbatch \ #SBATCH --cpus-per-task=${CPUS_PER_TASK} #SBATCH --exclusive -export OMP_NUM_THREADS=${CPUS_PER_TASK} - -check_health() { - local srunlist=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.srunlist +set -e - srun --cpu-bind=verbose,rank_ldom --network=single_node_vni hostname > \$srunlist +export OMP_NUM_THREADS=${CPUS_PER_TASK} - local unique_nodes=\$(sort \$srunlist | uniq | wc -l) - local total_tasks=\$(wc -l < \$srunlist) +check_gpus() { - if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then - echo "One or more nodes or tasks are faulty. Exiting." - exit 1 + local srunlist=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.srunlist + srun -l --cpu-bind=verbose,rank_ldom --network=single_node_vni hostname > \$srunlist + + local mem_state=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.mem + srun -l --cpu-bind=rank_ldom nvidia-smi --query-gpu=index,uuid,memory.free,ecc.errors.uncorrected.volatile.total --format=csv,nounits,noheader > \${mem_state} + + local min_free_mem=80000 + + # Read the node information into an associative array + declare -A node_map + while IFS=: read -r rank node; do + node_map[\$rank]=\$node + done < "\$srunlist" + + # Count unique values in node_map + local unique_values=() + for value in "\${node_map[@]}"; do + if [[ ! " \${unique_values[@]} " =~ " \$value " ]]; then + unique_values+=("\$value") + fi + done + + local num_entries=\${#node_map[@]} + local num_unique=\${#unique_values[@]} + if [ "\$num_unique" -ne "$NODES" ] || [ "\$num_entries" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty." + return 1 else echo "All nodes and tasks are healthy." fi + + # Process the GPU info file and check conditions + local failure=false + while IFS=: read -r rank gpu_info; do + # Split the GPU info into fields + IFS=',' read -r gpu_id gpu_name free_mem error_indicator <<< "\$gpu_info" + + # Trim leading and trailing whitespace from fields + gpu_id=\$(echo "\$gpu_id" | xargs) + gpu_name=\$(echo "\$gpu_name" | xargs) + free_mem=\$(echo "\$free_mem" | xargs) + error_indicator=\$(echo "\$error_indicator" | xargs) + + # Check conditions + if (( free_mem <= min_free_mem || error_indicator != 0 )); then + echo "Failing rank: \$rank, Node name: \${node_map[\$rank]}, GPU ID: \$gpu_id, GPU name: \$gpu_name, Free memory: \$free_mem, Error indicator: \$error_indicator" + failure=true + fi + done < "\$mem_state" + + if \$failure; then + echo "One or more gpus are faulty." + return 1 + else + echo "All ranks passed the checks." + return 0 + fi } -check_health +check_gpus uenv view default export LD_LIBRARY_PATH=/user-environment/env/default/lib64:\$LD_LIBRARY_PATH diff --git a/allreduce-perf/nccl-pytorch/run/run.sh b/allreduce-perf/nccl-pytorch/run/run.sh index 19c93a7..d1b7ef7 100755 --- a/allreduce-perf/nccl-pytorch/run/run.sh +++ b/allreduce-perf/nccl-pytorch/run/run.sh @@ -22,26 +22,72 @@ sbatch < \$srunlist +check_gpus() { - local unique_nodes=\$(sort \$srunlist | uniq | wc -l) - local total_tasks=\$(wc -l < \$srunlist) - - if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then - echo "One or more nodes or tasks are faulty. Exiting." - exit 1 + local srunlist=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.srunlist + srun -l --cpu-bind=verbose,rank_ldom --network=single_node_vni hostname > \$srunlist + + local mem_state=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.mem + srun -l --cpu-bind=rank_ldom nvidia-smi --query-gpu=index,uuid,memory.free,ecc.errors.uncorrected.volatile.total --format=csv,nounits,noheader > \${mem_state} + + local min_free_mem=80000 + + # Read the node information into an associative array + declare -A node_map + while IFS=: read -r rank node; do + node_map[\$rank]=\$node + done < "\$srunlist" + + # Count unique values in node_map + local unique_values=() + for value in "\${node_map[@]}"; do + if [[ ! " \${unique_values[@]} " =~ " \$value " ]]; then + unique_values+=("\$value") + fi + done + + local num_entries=\${#node_map[@]} + local num_unique=\${#unique_values[@]} + if [ "\$num_unique" -ne "$NODES" ] || [ "\$num_entries" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty." + return 1 else echo "All nodes and tasks are healthy." fi + + # Process the GPU info file and check conditions + local failure=false + while IFS=: read -r rank gpu_info; do + # Split the GPU info into fields + IFS=',' read -r gpu_id gpu_name free_mem error_indicator <<< "\$gpu_info" + + # Trim leading and trailing whitespace from fields + gpu_id=\$(echo "\$gpu_id" | xargs) + gpu_name=\$(echo "\$gpu_name" | xargs) + free_mem=\$(echo "\$free_mem" | xargs) + error_indicator=\$(echo "\$error_indicator" | xargs) + + # Check conditions + if (( free_mem <= min_free_mem || error_indicator != 0 )); then + echo "Failing rank: \$rank, Node name: \${node_map[\$rank]}, GPU ID: \$gpu_id, GPU name: \$gpu_name, Free memory: \$free_mem, Error indicator: \$error_indicator" + failure=true + fi + done < "\$mem_state" + + if \$failure; then + echo "One or more gpus are faulty." + return 1 + else + echo "All ranks passed the checks." + return 0 + fi } -check_health +check_gpus # Set torch.distributed variables export MASTER_ADDR=\$(scontrol show hostname \$SLURM_NODELIST | head -n1) @@ -52,7 +98,7 @@ export MAX_JOBS=64 export NCCL_DEBUG=INFO -run_cmd="CUDA_VISIBLE_DEVICES=0,1,2,3 python -u -m torch.distributed.run --nproc_per_node=$TASKS_PER_NODE --nnodes $NODES --rdzv_endpoint \$(scontrol show hostnames \$SLURM_JOB_NODELIST | head -n 1):6000 --rdzv_backend c10d ../src/all_reduce.py" +run_cmd="CUDA_VISIBLE_DEVICES=0,1,2,3 python -u -m torch.distributed.run --nproc_per_node=$NTASKS_PER_NODE --nnodes $NODES --rdzv_endpoint \$(scontrol show hostnames \$SLURM_JOB_NODELIST | head -n 1):6000 --rdzv_backend c10d ../src/all_reduce.py" srun -ul \ --cpu-bind=verbose,rank_ldom \ diff --git a/allreduce-perf/nccl-tests-cpp/run/run.sh b/allreduce-perf/nccl-tests-cpp/run/run.sh index 7957839..5590f9e 100755 --- a/allreduce-perf/nccl-tests-cpp/run/run.sh +++ b/allreduce-perf/nccl-tests-cpp/run/run.sh @@ -22,26 +22,76 @@ sbatch < \$srunlist + + local mem_state=job_n_${NTASKS_STR}_N_${NODES_STR}_TPN_${NTASKS_PER_NODE}.mem + srun -l --cpu-bind=rank_ldom nvidia-smi --query-gpu=index,uuid,memory.free,ecc.errors.uncorrected.volatile.total --format=csv,nounits,noheader > \${mem_state} - srun --cpu-bind=verbose,rank_ldom --network=single_node_vni hostname > \$srunlist + local min_free_mem=80000 - local unique_nodes=\$(sort \$srunlist | uniq | wc -l) - local total_tasks=\$(wc -l < \$srunlist) + # Read the node information into an associative array + declare -A node_map + while IFS=: read -r rank node; do + node_map[\$rank]=\$node + done < "\$srunlist" - if [ "\$unique_nodes" -ne "$NODES" ] || [ "\$total_tasks" -ne "$NTASKS" ]; then - echo "One or more nodes or tasks are faulty. Exiting." - exit 1 + # Count unique values in node_map + local unique_values=() + for value in "\${node_map[@]}"; do + if [[ ! " \${unique_values[@]} " =~ " \$value " ]]; then + unique_values+=("\$value") + fi + done + + local num_entries=\${#node_map[@]} + local num_unique=\${#unique_values[@]} + if [ "\$num_unique" -ne "$NODES" ] || [ "\$num_entries" -ne "$NTASKS" ]; then + echo "One or more nodes or tasks are faulty." + return 1 else echo "All nodes and tasks are healthy." fi + + # Process the GPU info file and check conditions + local failure=false + while IFS=: read -r rank gpu_info; do + # Split the GPU info into fields + IFS=',' read -r gpu_id gpu_name free_mem error_indicator <<< "\$gpu_info" + + # Trim leading and trailing whitespace from fields + gpu_id=\$(echo "\$gpu_id" | xargs) + gpu_name=\$(echo "\$gpu_name" | xargs) + free_mem=\$(echo "\$free_mem" | xargs) + error_indicator=\$(echo "\$error_indicator" | xargs) + + # Check conditions + if (( free_mem <= min_free_mem || error_indicator != 0 )); then + echo "Failing rank: \$rank, Node name: \${node_map[\$rank]}, GPU ID: \$gpu_id, GPU name: \$gpu_name, Free memory: \$free_mem, Error indicator: \$error_indicator" + failure=true + fi + done < "\$mem_state" + + if \$failure; then + echo "One or more gpus are faulty." + return 1 + else + echo "All ranks passed the checks." + return 0 + fi } -check_health +check_gpus + +unset UENV_MOUNT_LIST srun \ --cpu-bind=verbose,rank_ldom \ From f403f9c1c67f527f5811b15dd7a420860cd00a6c Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:11:23 +0200 Subject: [PATCH 6/8] README --- allreduce-perf/nccl-tests-cpp/README | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 allreduce-perf/nccl-tests-cpp/README diff --git a/allreduce-perf/nccl-tests-cpp/README b/allreduce-perf/nccl-tests-cpp/README new file mode 100644 index 0000000..4f2af2b --- /dev/null +++ b/allreduce-perf/nccl-tests-cpp/README @@ -0,0 +1,3 @@ +These tests are taken from https://github.com/NVIDIA/nccl-tests +and run within a container based on the image from +docker://jfrog.svc.cscs.ch#reframe-oci/nccl-tests:cuda12.3 From 8dc8b5fbe4522001390642f4fa91e9fd93d96979 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:19:26 +0200 Subject: [PATCH 7/8] avoid overflow --- allreduce-perf/mpi-cpp/src/allreduce.hpp | 38 ++++++++++++++++++------ allreduce-perf/mpi-cpp/src/bench.cpp | 38 ++++++++++++++++++++---- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/allreduce-perf/mpi-cpp/src/allreduce.hpp b/allreduce-perf/mpi-cpp/src/allreduce.hpp index 2423460..863e26a 100644 --- a/allreduce-perf/mpi-cpp/src/allreduce.hpp +++ b/allreduce-perf/mpi-cpp/src/allreduce.hpp @@ -29,19 +29,39 @@ void sum(buffer& dest, buffer const & src) { template void copy(buffer& dest, buffer const & src) { - switch (dest.type()) { - case mem_type::device: + if (dest.type() == mem_type::device) { + if (src.type() == mem_type::device) { CHECK_CUDA(cudaMemcpy(dest.data(), src.data(), dest.size()*sizeof(T), cudaMemcpyDeviceToDevice)); - CHECK_CUDA(cudaDeviceSynchronize()); - break; + } + else { + CHECK_CUDA(cudaMemcpy(dest.data(), src.data(), dest.size()*sizeof(T), cudaMemcpyHostToDevice)); + } + CHECK_CUDA(cudaDeviceSynchronize()); + } + else if (src.type() == mem_type::device) { + CHECK_CUDA(cudaMemcpy(dest.data(), src.data(), dest.size()*sizeof(T), cudaMemcpyDeviceToHost)); + CHECK_CUDA(cudaDeviceSynchronize()); + } + else { + std::memcpy(dest.data(), src.data(), dest.size()*sizeof(T)); + } +} - default: - std::memcpy(dest.data(), src.data(), dest.size()*sizeof(T)); +template +bool check(const buffer& b, T expected) { + buffer c(mem_type::host, b.size(), 0); + copy(c, b); + for (std::size_t i=0; i -void all_reduce_naive_ring(buffer& b) { +void all_reduce_naive_ring(buffer& b, MPI_Datatype mpi_type) { static thread_local std::unique_ptr> recv_buffer_ptr; if (!recv_buffer_ptr || (recv_buffer_ptr->size() < b.size()) || @@ -76,8 +96,8 @@ void all_reduce_naive_ring(buffer& b) { for (int i=0; idata(), s->size()*sizeof(T), MPI_BYTE, send_rank, 0, - r->data(), s->size()*sizeof(T), MPI_BYTE, recv_rank, 0, + s->data(), s->size(), mpi_type, send_rank, 0, + r->data(), s->size(), mpi_type, recv_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); sum(b, *r); std::swap(s, r); diff --git a/allreduce-perf/mpi-cpp/src/bench.cpp b/allreduce-perf/mpi-cpp/src/bench.cpp index d9feede..3283208 100644 --- a/allreduce-perf/mpi-cpp/src/bench.cpp +++ b/allreduce-perf/mpi-cpp/src/bench.cpp @@ -1,8 +1,9 @@ #include "memory.hpp" #include "allreduce.hpp" -#include #include +#include +#include #include #include #include @@ -49,7 +50,29 @@ int main(int argc, char** argv) { std::size_t const padding = std::stoul(argv[5]); bool const naive = std::stoul(argv[6]); - CHECK_CUDA(cudaDeviceSynchronize()); + if (size > (std::size_t)std::numeric_limits::max()) { + std::cout << "buffer is too large" << std::endl; + exit(1); + } + + if (t == test::mem_type::device) { + int dev_count, local_rank, local_size; + CHECK_CUDA(cudaGetDeviceCount(&dev_count)) + + MPI_Comm local_comm; + CHECK_MPI(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, comm_rank, MPI_INFO_NULL, &local_comm)); + CHECK_MPI(MPI_Comm_rank(local_comm, &local_rank)); + CHECK_MPI(MPI_Comm_size(local_comm, &local_size)); + auto did = local_rank % dev_count; + std::cout << "Found " << dev_count << " devices for local rank " << local_rank << " (" << local_size << ") -> setting device = " << did << std::endl; + + CHECK_CUDA(cudaSetDevice(did)); + CHECK_CUDA(cudaDeviceSynchronize()); + + CHECK_MPI(MPI_Comm_free(&local_comm)); + CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); + } + if (comm_rank == 0) { std::cout << "mem_type: " << argv[3] << '\n'; std::cout << "size: " << size << '\n'; @@ -71,6 +94,7 @@ int main(int argc, char** argv) { for (auto& v : times) v.resize(num_sub_iterations-1); for (auto& v : bws) v.resize(num_sub_iterations-1); + CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); // Warmup { const std::chrono::time_point start = @@ -82,7 +106,11 @@ int main(int argc, char** argv) { if (!naive) test::all_reduce(buffer, mpi_type); else - test::all_reduce_naive_ring(buffer); + test::all_reduce_naive_ring(buffer, mpi_type); + + //if (!check(buffer, (type)comm_size)) { + // std::exit(1); + //} } @@ -98,16 +126,16 @@ int main(int argc, char** argv) { for (std::size_t i = 0; i < num_iterations; ++i) { auto buffer = test::buffer(t, size, padding); - test::fill(buffer, (type)1); for (std::size_t j = 0; j < num_sub_iterations; ++j) { + test::fill(buffer, (type)1); const std::chrono::time_point start = std::chrono::steady_clock::now(); if (!naive) test::all_reduce(buffer, mpi_type); else - test::all_reduce_naive_ring(buffer); + test::all_reduce_naive_ring(buffer, mpi_type); CHECK_MPI(MPI_Barrier(MPI_COMM_WORLD)); From 3e01cad76d98fa54f5cecd50acdb4b084da43b79 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:36:31 +0200 Subject: [PATCH 8/8] new data --- .../run/data/job_n_00004_N_0001_TPN_4.out | 394 +- .../run/data/job_n_00004_N_0004_TPN_1.out | 394 +- .../run/data/job_n_00016_N_0004_TPN_4.out | 490 +- .../run/data/job_n_00016_N_0016_TPN_1.out | 490 +- .../run/data/job_n_00032_N_0008_TPN_4.out | 618 +- .../run/data/job_n_00032_N_0032_TPN_1.out | 618 +- .../run/data/job_n_00064_N_0016_TPN_4.out | 874 +- .../run/data/job_n_00064_N_0064_TPN_1.out | 874 +- .../run/data/job_n_00128_N_0032_TPN_4.out | 1386 ++- .../run/data/job_n_00128_N_0128_TPN_1.out | 1366 ++- .../run/data/job_n_00256_N_0064_TPN_4.out | 2410 +++-- .../run/data/job_n_00256_N_0256_TPN_1.out | 2410 +++-- .../run/data/job_n_00512_N_0128_TPN_4.out | 4384 ++++----- .../run/data/job_n_01024_N_0256_TPN_4.out | 8554 ++++++++--------- allreduce-perf/mpi-cpp/run/plot.py | 27 +- .../mpi-cpp/run/plots/device_memory.svg | 843 +- .../mpi-cpp/run/plots/device_memory_1.svg | 306 +- .../mpi-cpp/run/plots/device_memory_1_pad.svg | 314 +- .../run/plots/device_memory_1_pad_startup.svg | 430 +- .../run/plots/device_memory_1_startup.svg | 424 +- .../mpi-cpp/run/plots/device_memory_4.svg | 401 +- .../mpi-cpp/run/plots/device_memory_4_pad.svg | 411 +- .../run/plots/device_memory_4_pad_startup.svg | 503 +- .../run/plots/device_memory_4_startup.svg | 495 +- .../run/plots/device_memory_startup.svg | 1259 +-- 25 files changed, 13681 insertions(+), 16994 deletions(-) diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out index 3631ea0..1edd6b4 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0001_TPN_4.out @@ -1,295 +1,157 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid005332 -Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005066 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid005066 +Hostname: nid005066 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.348798 -[0:0] time: 0.103512 bw: 62.2384 -[0:1] time: 0.0975796 bw: 66.0225 -[0:2] time: 0.0976147 bw: 65.9988 -[0:3] time: 0.0974713 bw: 66.0959 -[0:4] time: 0.097489 bw: 66.0839 -[1:0] time: 0.103568 bw: 62.2051 -[1:1] time: 0.0976669 bw: 65.9635 -[1:2] time: 0.0975045 bw: 66.0734 -[1:3] time: 0.0975854 bw: 66.0186 -[1:4] time: 0.0976583 bw: 65.9693 -[2:0] time: 0.103796 bw: 62.0685 -[2:1] time: 0.0977194 bw: 65.928 -[2:2] time: 0.0977674 bw: 65.8957 -[2:3] time: 0.0973601 bw: 66.1714 -[2:4] time: 0.097463 bw: 66.1015 -[3:0] time: 0.105657 bw: 60.9749 -[3:1] time: 0.0974052 bw: 66.1407 -[3:2] time: 0.0975839 bw: 66.0196 -[3:3] time: 0.0975276 bw: 66.0577 -[3:4] time: 0.0976227 bw: 65.9934 -[4:0] time: 0.10359 bw: 62.1919 -[4:1] time: 0.0975199 bw: 66.0629 -[4:2] time: 0.0975719 bw: 66.0277 -[4:3] time: 0.0975666 bw: 66.0313 -[4:4] time: 0.0976924 bw: 65.9463 -[5:0] time: 0.107909 bw: 59.7028 -[5:1] time: 0.0975342 bw: 66.0533 -[5:2] time: 0.0976476 bw: 65.9766 -[5:3] time: 0.0975732 bw: 66.0269 -[5:4] time: 0.0974735 bw: 66.0944 -[6:0] time: 0.103833 bw: 62.0464 -[6:1] time: 0.0978823 bw: 65.8183 -[6:2] time: 0.0979781 bw: 65.754 -[6:3] time: 0.097787 bw: 65.8825 -[6:4] time: 0.0977631 bw: 65.8986 -[7:0] time: 0.10384 bw: 62.0418 -[7:1] time: 0.0977532 bw: 65.9053 -[7:2] time: 0.0977929 bw: 65.8785 -[7:3] time: 0.0979134 bw: 65.7975 -[7:4] time: 0.0978088 bw: 65.8678 -[8:0] time: 0.108783 bw: 59.2231 -[8:1] time: 0.097989 bw: 65.7466 -[8:2] time: 0.097793 bw: 65.8785 -[8:3] time: 0.0979342 bw: 65.7834 -[8:4] time: 0.0977989 bw: 65.8744 -[9:0] time: 0.1039 bw: 62.0065 -[9:1] time: 0.0979841 bw: 65.75 -[9:2] time: 0.0979718 bw: 65.7582 -[9:3] time: 0.0981458 bw: 65.6417 -[9:4] time: 0.0978096 bw: 65.8672 +[-1] time: 0.349024 +[0:0] time: 0.104134 bw: 61.8672 +[0:1] time: 0.0984388 bw: 65.4462 +[0:2] time: 0.0983045 bw: 65.5357 +[0:3] time: 0.0981081 bw: 65.6669 +[0:4] time: 0.0982657 bw: 65.5616 +[1:0] time: 0.104553 bw: 61.6189 +[1:1] time: 0.0985256 bw: 65.3886 +[1:2] time: 0.0986397 bw: 65.313 +[1:3] time: 0.0987565 bw: 65.2357 +[1:4] time: 0.098472 bw: 65.4242 +[2:0] time: 0.106711 bw: 60.3728 +[2:1] time: 0.0985837 bw: 65.3501 +[2:2] time: 0.0984828 bw: 65.417 +[2:3] time: 0.0985135 bw: 65.3966 +[2:4] time: 0.0985541 bw: 65.3697 +[3:0] time: 0.104247 bw: 61.8001 +[3:1] time: 0.0986414 bw: 65.3118 +[3:2] time: 0.0987943 bw: 65.2108 +[3:3] time: 0.0985138 bw: 65.3964 +[3:4] time: 0.0983782 bw: 65.4866 +[4:0] time: 0.104339 bw: 61.7454 +[4:1] time: 0.0984836 bw: 65.4165 +[4:2] time: 0.0982274 bw: 65.5871 +[4:3] time: 0.0984484 bw: 65.4399 +[4:4] time: 0.0982334 bw: 65.5831 +[5:0] time: 0.104839 bw: 61.4511 +[5:1] time: 0.0989071 bw: 65.1364 +[5:2] time: 0.0991509 bw: 64.9762 +[5:3] time: 0.0988247 bw: 65.1907 +[5:4] time: 0.0984352 bw: 65.4487 +[6:0] time: 0.10503 bw: 61.3394 +[6:1] time: 0.0986091 bw: 65.3332 +[6:2] time: 0.098447 bw: 65.4408 +[6:3] time: 0.0987215 bw: 65.2588 +[6:4] time: 0.09866 bw: 65.2995 +[7:0] time: 0.104397 bw: 61.7113 +[7:1] time: 0.0985353 bw: 65.3821 +[7:2] time: 0.0983509 bw: 65.5048 +[7:3] time: 0.0984036 bw: 65.4697 +[7:4] time: 0.0985731 bw: 65.3571 +[8:0] time: 0.104087 bw: 61.8951 +[8:1] time: 0.0982648 bw: 65.5621 +[8:2] time: 0.0981557 bw: 65.635 +[8:3] time: 0.0981288 bw: 65.653 +[8:4] time: 0.0985666 bw: 65.3614 +[9:0] time: 0.104625 bw: 61.5768 +[9:1] time: 0.0988162 bw: 65.1963 +[9:2] time: 0.098727 bw: 65.2552 +[9:3] time: 0.0985673 bw: 65.3609 +[9:4] time: 0.0987365 bw: 65.2489 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.0976926 -time0: 0.104839 -bw: 65.9464 -bw0: 61.4699 +time: 0.0985237 +time0: 0.104696 +bw: 65.3902 +bw0: 61.5378 ======================================= -Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid005332 -Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005066 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid005066 +Hostname: nid005066 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 14.952 -[0:0] time: 14.9227 bw: 0.431722 -[0:1] time: 0.0976961 bw: 65.9438 -[0:2] time: 0.0975225 bw: 66.0612 -[0:3] time: 0.0977217 bw: 65.9265 -[0:4] time: 0.0976766 bw: 65.9569 -[1:0] time: 14.9305 bw: 0.431495 -[1:1] time: 0.0975571 bw: 66.0377 -[1:2] time: 0.0976218 bw: 65.994 -[1:3] time: 0.0977372 bw: 65.9161 -[1:4] time: 0.0975366 bw: 66.0517 -[2:0] time: 14.9384 bw: 0.431268 -[2:1] time: 0.0972738 bw: 66.23 -[2:2] time: 0.097445 bw: 66.1137 -[2:3] time: 0.0974536 bw: 66.1079 -[2:4] time: 0.0974845 bw: 66.0869 -[3:0] time: 14.9404 bw: 0.431211 -[3:1] time: 0.0973405 bw: 66.1847 -[3:2] time: 0.0974357 bw: 66.1201 -[3:3] time: 0.0974774 bw: 66.0917 -[3:4] time: 0.0974584 bw: 66.1046 -[4:0] time: 14.968 bw: 0.430415 -[4:1] time: 0.0975701 bw: 66.0289 -[4:2] time: 0.0974869 bw: 66.0853 -[4:3] time: 0.0976788 bw: 65.9555 -[4:4] time: 0.0976532 bw: 65.9728 -[5:0] time: 14.9486 bw: 0.430973 -[5:1] time: 0.0972953 bw: 66.2154 -[5:2] time: 0.0973075 bw: 66.2071 -[5:3] time: 0.0971813 bw: 66.2931 -[5:4] time: 0.0972392 bw: 66.2537 -[6:0] time: 14.9752 bw: 0.430208 -[6:1] time: 0.0976714 bw: 65.9605 -[6:2] time: 0.0976596 bw: 65.9684 -[6:3] time: 0.0976185 bw: 65.9962 -[6:4] time: 0.0977148 bw: 65.9312 -[7:0] time: 14.9511 bw: 0.430901 -[7:1] time: 0.0975293 bw: 66.0566 -[7:2] time: 0.0976393 bw: 65.9822 -[7:3] time: 0.0975174 bw: 66.0646 -[7:4] time: 0.0977363 bw: 65.9167 -[8:0] time: 14.9418 bw: 0.431169 -[8:1] time: 0.0974611 bw: 66.1028 -[8:2] time: 0.0976367 bw: 65.9839 -[8:3] time: 0.0974381 bw: 66.1184 -[8:4] time: 0.0975926 bw: 66.0137 -[9:0] time: 14.9554 bw: 0.430777 -[9:1] time: 0.0976213 bw: 65.9943 -[9:2] time: 0.0976296 bw: 65.9887 -[9:3] time: 0.0975525 bw: 66.0409 -[9:4] time: 0.0975892 bw: 66.016 +[-1] time: 14.8462 +[0:0] time: 14.8938 bw: 0.43256 +[0:1] time: 0.0980822 bw: 65.6842 +[0:2] time: 0.0982305 bw: 65.5851 +[0:3] time: 0.0982166 bw: 65.5943 +[0:4] time: 0.0980554 bw: 65.7022 +[1:0] time: 14.8596 bw: 0.433556 +[1:1] time: 0.0981696 bw: 65.6257 +[1:2] time: 0.0982016 bw: 65.6043 +[1:3] time: 0.0982873 bw: 65.5471 +[1:4] time: 0.0981629 bw: 65.6302 +[2:0] time: 14.8851 bw: 0.432811 +[2:1] time: 0.0981743 bw: 65.6226 +[2:2] time: 0.0982659 bw: 65.5614 +[2:3] time: 0.0981308 bw: 65.6516 +[2:4] time: 0.0982912 bw: 65.5445 +[3:0] time: 14.8734 bw: 0.433154 +[3:1] time: 0.0982415 bw: 65.5777 +[3:2] time: 0.0983114 bw: 65.5311 +[3:3] time: 0.0983579 bw: 65.5001 +[3:4] time: 0.0984724 bw: 65.4239 +[4:0] time: 14.8678 bw: 0.433314 +[4:1] time: 0.0982261 bw: 65.588 +[4:2] time: 0.0981781 bw: 65.62 +[4:3] time: 0.0981592 bw: 65.6327 +[4:4] time: 0.0981481 bw: 65.6401 +[5:0] time: 14.8587 bw: 0.433582 +[5:1] time: 0.0982778 bw: 65.5535 +[5:2] time: 0.0982901 bw: 65.5453 +[5:3] time: 0.0982992 bw: 65.5392 +[5:4] time: 0.0983163 bw: 65.5278 +[6:0] time: 14.8971 bw: 0.432465 +[6:1] time: 0.0984986 bw: 65.4065 +[6:2] time: 0.0984132 bw: 65.4633 +[6:3] time: 0.0984688 bw: 65.4263 +[6:4] time: 0.0982974 bw: 65.5404 +[7:0] time: 14.8828 bw: 0.432879 +[7:1] time: 0.0983158 bw: 65.5282 +[7:2] time: 0.0984325 bw: 65.4505 +[7:3] time: 0.0986286 bw: 65.3203 +[7:4] time: 0.0985999 bw: 65.3394 +[8:0] time: 14.8631 bw: 0.433452 +[8:1] time: 0.098213 bw: 65.5968 +[8:2] time: 0.0982337 bw: 65.5829 +[8:3] time: 0.0983646 bw: 65.4956 +[8:4] time: 0.0982486 bw: 65.573 +[9:0] time: 14.9171 bw: 0.431884 +[9:1] time: 0.0984047 bw: 65.4689 +[9:2] time: 0.0981122 bw: 65.6641 +[9:3] time: 0.0980827 bw: 65.6839 +[9:4] time: 0.0983252 bw: 65.5219 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.0975365 -time0: 14.9472 -bw: 66.0519 -bw0: 0.431014 -======================================= - -Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid005332 -Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.0550143 -[0:0] time: 0.015113 bw: 426.287 -[0:1] time: 0.0150717 bw: 427.454 -[0:2] time: 0.0150779 bw: 427.276 -[0:3] time: 0.0150775 bw: 427.29 -[0:4] time: 0.0150858 bw: 427.055 -[1:0] time: 0.0152073 bw: 423.641 -[1:1] time: 0.0150675 bw: 427.573 -[1:2] time: 0.0150675 bw: 427.572 -[1:3] time: 0.0150641 bw: 427.669 -[1:4] time: 0.0150662 bw: 427.61 -[2:0] time: 0.0152628 bw: 422.102 -[2:1] time: 0.0150829 bw: 427.137 -[2:2] time: 0.0150693 bw: 427.522 -[2:3] time: 0.0150759 bw: 427.334 -[2:4] time: 0.0150794 bw: 427.235 -[3:0] time: 0.0152469 bw: 422.543 -[3:1] time: 0.0150704 bw: 427.491 -[3:2] time: 0.0150789 bw: 427.25 -[3:3] time: 0.0150714 bw: 427.461 -[3:4] time: 0.0150602 bw: 427.78 -[4:0] time: 0.0152731 bw: 421.817 -[4:1] time: 0.0150689 bw: 427.533 -[4:2] time: 0.0150675 bw: 427.572 -[4:3] time: 0.0150681 bw: 427.555 -[4:4] time: 0.0150679 bw: 427.561 -[5:0] time: 0.0152159 bw: 423.402 -[5:1] time: 0.0150591 bw: 427.81 -[5:2] time: 0.0150599 bw: 427.788 -[5:3] time: 0.0150616 bw: 427.74 -[5:4] time: 0.0150604 bw: 427.773 -[6:0] time: 0.0151478 bw: 425.305 -[6:1] time: 0.0150695 bw: 427.517 -[6:2] time: 0.015073 bw: 427.418 -[6:3] time: 0.015065 bw: 427.645 -[6:4] time: 0.0150675 bw: 427.571 -[7:0] time: 0.0152497 bw: 422.465 -[7:1] time: 0.0150627 bw: 427.709 -[7:2] time: 0.0150674 bw: 427.577 -[7:3] time: 0.0150634 bw: 427.689 -[7:4] time: 0.0150681 bw: 427.555 -[8:0] time: 0.0237532 bw: 271.225 -[8:1] time: 0.0150711 bw: 427.471 -[8:2] time: 0.0150714 bw: 427.462 -[8:3] time: 0.0150699 bw: 427.503 -[8:4] time: 0.0150675 bw: 427.573 -[9:0] time: 0.015248 bw: 422.512 -[9:1] time: 0.0150674 bw: 427.575 -[9:2] time: 0.0150603 bw: 427.777 -[9:3] time: 0.0150606 bw: 427.769 -[9:4] time: 0.0150619 bw: 427.732 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.0150687 -time0: 0.0160718 -bw: 427.54 -bw0: 408.13 -======================================= - -Hostname: nid005332 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005332 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005332 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid005332 -Hostname: nid005332 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.119388 -[0:0] time: 0.0155304 bw: 414.828 -[0:1] time: 0.0154832 bw: 416.093 -[0:2] time: 0.015483 bw: 416.098 -[0:3] time: 0.015481 bw: 416.152 -[0:4] time: 0.0154971 bw: 415.72 -[1:0] time: 0.0156326 bw: 412.116 -[1:1] time: 0.0154866 bw: 416.001 -[1:2] time: 0.0154855 bw: 416.032 -[1:3] time: 0.0154784 bw: 416.221 -[1:4] time: 0.0154824 bw: 416.115 -[2:0] time: 0.0156866 bw: 410.697 -[2:1] time: 0.0154877 bw: 415.972 -[2:2] time: 0.0154809 bw: 416.156 -[2:3] time: 0.0154839 bw: 416.073 -[2:4] time: 0.0154835 bw: 416.085 -[3:0] time: 0.0155252 bw: 414.968 -[3:1] time: 0.0154827 bw: 416.106 -[3:2] time: 0.0155011 bw: 415.612 -[3:3] time: 0.0154918 bw: 415.862 -[3:4] time: 0.0154768 bw: 416.264 -[4:0] time: 0.0156413 bw: 411.888 -[4:1] time: 0.0154795 bw: 416.192 -[4:2] time: 0.0154801 bw: 416.176 -[4:3] time: 0.0154804 bw: 416.169 -[4:4] time: 0.0154793 bw: 416.198 -[5:0] time: 0.0156836 bw: 410.777 -[5:1] time: 0.0154719 bw: 416.396 -[5:2] time: 0.0154743 bw: 416.334 -[5:3] time: 0.0154743 bw: 416.331 -[5:4] time: 0.015474 bw: 416.341 -[6:0] time: 0.0156298 bw: 412.19 -[6:1] time: 0.0154789 bw: 416.21 -[6:2] time: 0.0154836 bw: 416.082 -[6:3] time: 0.0154804 bw: 416.168 -[6:4] time: 0.0154785 bw: 416.218 -[7:0] time: 0.0157185 bw: 409.863 -[7:1] time: 0.0154786 bw: 416.216 -[7:2] time: 0.015481 bw: 416.152 -[7:3] time: 0.0154747 bw: 416.322 -[7:4] time: 0.0154832 bw: 416.093 -[8:0] time: 0.0155806 bw: 413.492 -[8:1] time: 0.015477 bw: 416.259 -[8:2] time: 0.0154789 bw: 416.208 -[8:3] time: 0.0154796 bw: 416.189 -[8:4] time: 0.0154765 bw: 416.274 -[9:0] time: 0.021907 bw: 294.082 -[9:1] time: 0.0154852 bw: 416.039 -[9:2] time: 0.0154728 bw: 416.372 -[9:3] time: 0.0154778 bw: 416.239 -[9:4] time: 0.0154743 bw: 416.334 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.015481 -time0: 0.0162536 -bw: 416.152 -bw0: 400.49 +time: 0.0982796 +time0: 14.8798 +bw: 65.5524 +bw0: 0.432966 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out index 6150814..7c8ca66 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00004_N_0004_TPN_1.out @@ -1,295 +1,157 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[005352,005356,005360,005362] -Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005312 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[005304,005308,005310,005312] +Hostname: nid005304 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005308 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005310 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.381335 -[0:0] time: 0.344453 bw: 18.7034 -[0:1] time: 0.323566 bw: 19.9108 -[0:2] time: 0.323661 bw: 19.9049 -[0:3] time: 0.32359 bw: 19.9093 -[0:4] time: 0.32369 bw: 19.9031 -[1:0] time: 0.344244 bw: 18.7148 -[1:1] time: 0.324131 bw: 19.8761 -[1:2] time: 0.323547 bw: 19.9119 -[1:3] time: 0.323755 bw: 19.8992 -[1:4] time: 0.323589 bw: 19.9094 -[2:0] time: 0.344449 bw: 18.7036 -[2:1] time: 0.323609 bw: 19.9081 -[2:2] time: 0.323543 bw: 19.9122 -[2:3] time: 0.323941 bw: 19.8877 -[2:4] time: 0.32369 bw: 19.9032 -[3:0] time: 0.344098 bw: 18.7227 -[3:1] time: 0.324104 bw: 19.8777 -[3:2] time: 0.324189 bw: 19.8725 -[3:3] time: 0.324084 bw: 19.8789 -[3:4] time: 0.32463 bw: 19.8455 -[4:0] time: 0.344621 bw: 18.6943 -[4:1] time: 0.324155 bw: 19.8746 -[4:2] time: 0.324157 bw: 19.8745 -[4:3] time: 0.323858 bw: 19.8928 -[4:4] time: 0.323788 bw: 19.8972 -[5:0] time: 0.343632 bw: 18.7481 -[5:1] time: 0.323413 bw: 19.9202 -[5:2] time: 0.323192 bw: 19.9338 -[5:3] time: 0.323099 bw: 19.9396 -[5:4] time: 0.323184 bw: 19.9343 -[6:0] time: 0.344311 bw: 18.7111 -[6:1] time: 0.32341 bw: 19.9204 -[6:2] time: 0.32358 bw: 19.9099 -[6:3] time: 0.323432 bw: 19.919 -[6:4] time: 0.32353 bw: 19.913 -[7:0] time: 0.344217 bw: 18.7163 -[7:1] time: 0.323508 bw: 19.9144 -[7:2] time: 0.32385 bw: 19.8933 -[7:3] time: 0.323944 bw: 19.8876 -[7:4] time: 0.323675 bw: 19.904 -[8:0] time: 0.344494 bw: 18.7012 -[8:1] time: 0.323187 bw: 19.9341 -[8:2] time: 0.323111 bw: 19.9388 -[8:3] time: 0.323283 bw: 19.9282 -[8:4] time: 0.323258 bw: 19.9298 -[9:0] time: 0.343387 bw: 18.7615 -[9:1] time: 0.323274 bw: 19.9288 -[9:2] time: 0.323297 bw: 19.9273 -[9:3] time: 0.323303 bw: 19.927 -[9:4] time: 0.323448 bw: 19.9181 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 0.355146 +[0:0] time: 0.344826 bw: 18.6832 +[0:1] time: 0.323922 bw: 19.8889 +[0:2] time: 0.323691 bw: 19.9031 +[0:3] time: 0.323924 bw: 19.8888 +[0:4] time: 0.323875 bw: 19.8918 +[1:0] time: 0.345623 bw: 18.6401 +[1:1] time: 0.324382 bw: 19.8607 +[1:2] time: 0.323893 bw: 19.8907 +[1:3] time: 0.323828 bw: 19.8947 +[1:4] time: 0.323647 bw: 19.9058 +[2:0] time: 0.344889 bw: 18.6798 +[2:1] time: 0.323606 bw: 19.9083 +[2:2] time: 0.323775 bw: 19.8979 +[2:3] time: 0.32401 bw: 19.8835 +[2:4] time: 0.324216 bw: 19.8709 +[3:0] time: 0.344773 bw: 18.6861 +[3:1] time: 0.324004 bw: 19.8839 +[3:2] time: 0.324088 bw: 19.8787 +[3:3] time: 0.324582 bw: 19.8484 +[3:4] time: 0.32402 bw: 19.8829 +[4:0] time: 0.344893 bw: 18.6795 +[4:1] time: 0.324232 bw: 19.8699 +[4:2] time: 0.323956 bw: 19.8868 +[4:3] time: 0.324 bw: 19.8841 +[4:4] time: 0.323932 bw: 19.8883 +[5:0] time: 0.344994 bw: 18.6741 +[5:1] time: 0.323819 bw: 19.8952 +[5:2] time: 0.323977 bw: 19.8855 +[5:3] time: 0.324002 bw: 19.884 +[5:4] time: 0.323909 bw: 19.8897 +[6:0] time: 0.344524 bw: 18.6996 +[6:1] time: 0.324004 bw: 19.8838 +[6:2] time: 0.323934 bw: 19.8881 +[6:3] time: 0.324028 bw: 19.8824 +[6:4] time: 0.323974 bw: 19.8857 +[7:0] time: 0.345143 bw: 18.666 +[7:1] time: 0.32379 bw: 19.897 +[7:2] time: 0.323829 bw: 19.8946 +[7:3] time: 0.323582 bw: 19.9098 +[7:4] time: 0.323559 bw: 19.9112 +[8:0] time: 0.345131 bw: 18.6667 +[8:1] time: 0.324361 bw: 19.862 +[8:2] time: 0.324354 bw: 19.8624 +[8:3] time: 0.32457 bw: 19.8492 +[8:4] time: 0.324492 bw: 19.8539 +[9:0] time: 0.345663 bw: 18.638 +[9:1] time: 0.32468 bw: 19.8425 +[9:2] time: 0.324636 bw: 19.8452 +[9:3] time: 0.324254 bw: 19.8685 +[9:4] time: 0.323982 bw: 19.8852 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.323631 -time0: 0.344191 -bw: 19.9068 -bw0: 18.7177 +time: 0.324033 +time0: 0.345046 +bw: 19.8821 +bw0: 18.6713 ======================================= -Slurm Job Hostlist: nid[005352,005356,005360,005362] -Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[005304,005308,005310,005312] +Hostname: nid005304 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005310 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005308 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005312 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 1.27841 -[0:0] time: 1.26066 bw: 5.11036 -[0:1] time: 1.25103 bw: 5.14974 -[0:2] time: 1.24951 bw: 5.15599 -[0:3] time: 1.25101 bw: 5.14981 -[0:4] time: 1.25126 bw: 5.14876 -[1:0] time: 1.27675 bw: 5.04596 -[1:1] time: 1.25492 bw: 5.13374 -[1:2] time: 1.25404 bw: 5.13737 -[1:3] time: 1.25667 bw: 5.1266 -[1:4] time: 1.25445 bw: 5.13569 -[2:0] time: 1.27783 bw: 5.04173 -[2:1] time: 1.25564 bw: 5.13079 -[2:2] time: 1.2561 bw: 5.12894 -[2:3] time: 1.25683 bw: 5.12596 -[2:4] time: 1.25325 bw: 5.14061 -[3:0] time: 1.27916 bw: 5.03646 -[3:1] time: 1.25797 bw: 5.12129 -[3:2] time: 1.25733 bw: 5.12391 -[3:3] time: 1.2562 bw: 5.12851 -[3:4] time: 1.25754 bw: 5.12307 -[4:0] time: 1.27943 bw: 5.03542 -[4:1] time: 1.25601 bw: 5.1293 -[4:2] time: 1.25592 bw: 5.12966 -[4:3] time: 1.25932 bw: 5.11583 -[4:4] time: 1.25727 bw: 5.12414 -[5:0] time: 1.28368 bw: 5.01874 -[5:1] time: 1.26102 bw: 5.10893 -[5:2] time: 1.2602 bw: 5.11226 -[5:3] time: 1.26148 bw: 5.10704 -[5:4] time: 1.26043 bw: 5.11132 -[6:0] time: 1.28472 bw: 5.01467 -[6:1] time: 1.26123 bw: 5.10807 -[6:2] time: 1.2631 bw: 5.10049 -[6:3] time: 1.26235 bw: 5.10355 -[6:4] time: 1.26164 bw: 5.10643 -[7:0] time: 1.28281 bw: 5.02215 -[7:1] time: 1.26126 bw: 5.10796 -[7:2] time: 1.26156 bw: 5.10673 -[7:3] time: 1.26152 bw: 5.1069 -[7:4] time: 1.26188 bw: 5.10542 -[8:0] time: 1.28496 bw: 5.01374 -[8:1] time: 1.26111 bw: 5.10854 -[8:2] time: 1.26141 bw: 5.10735 -[8:3] time: 1.26178 bw: 5.10585 -[8:4] time: 1.26287 bw: 5.10143 -[9:0] time: 1.28522 bw: 5.01274 -[9:1] time: 1.26394 bw: 5.09713 -[9:2] time: 1.26422 bw: 5.096 -[9:3] time: 1.26386 bw: 5.09745 -[9:4] time: 1.26429 bw: 5.09572 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.29774 +[0:0] time: 1.26032 bw: 5.11175 +[0:1] time: 1.25232 bw: 5.1444 +[0:2] time: 1.25195 bw: 5.14594 +[0:3] time: 1.25127 bw: 5.14873 +[0:4] time: 1.25269 bw: 5.14289 +[1:0] time: 1.28093 bw: 5.02952 +[1:1] time: 1.25952 bw: 5.11499 +[1:2] time: 1.25902 bw: 5.11704 +[1:3] time: 1.25949 bw: 5.11512 +[1:4] time: 1.26254 bw: 5.10278 +[2:0] time: 1.29536 bw: 4.9735 +[2:1] time: 1.26099 bw: 5.10903 +[2:2] time: 1.25963 bw: 5.11456 +[2:3] time: 1.26217 bw: 5.10427 +[2:4] time: 1.26106 bw: 5.10877 +[3:0] time: 1.27687 bw: 5.04552 +[3:1] time: 1.26219 bw: 5.10418 +[3:2] time: 1.2631 bw: 5.10049 +[3:3] time: 1.26233 bw: 5.10361 +[3:4] time: 1.26315 bw: 5.1003 +[4:0] time: 1.29914 bw: 4.95901 +[4:1] time: 1.26445 bw: 5.09506 +[4:2] time: 1.26325 bw: 5.09992 +[4:3] time: 1.26409 bw: 5.09653 +[4:4] time: 1.26365 bw: 5.09829 +[5:0] time: 1.28475 bw: 5.01456 +[5:1] time: 1.26441 bw: 5.09524 +[5:2] time: 1.26334 bw: 5.09952 +[5:3] time: 1.26317 bw: 5.10024 +[5:4] time: 1.26454 bw: 5.09468 +[6:0] time: 1.3017 bw: 4.94927 +[6:1] time: 1.26341 bw: 5.09924 +[6:2] time: 1.26401 bw: 5.09683 +[6:3] time: 1.26417 bw: 5.09618 +[6:4] time: 1.26428 bw: 5.09574 +[7:0] time: 1.28556 bw: 5.0114 +[7:1] time: 1.26358 bw: 5.09857 +[7:2] time: 1.26563 bw: 5.0903 +[7:3] time: 1.26416 bw: 5.09621 +[7:4] time: 1.26421 bw: 5.09602 +[8:0] time: 1.30499 bw: 4.9368 +[8:1] time: 1.26575 bw: 5.08983 +[8:2] time: 1.26514 bw: 5.0923 +[8:3] time: 1.26489 bw: 5.09327 +[8:4] time: 1.26425 bw: 5.09586 +[9:0] time: 1.28737 bw: 5.00434 +[9:1] time: 1.26508 bw: 5.0925 +[9:2] time: 1.26359 bw: 5.09854 +[9:3] time: 1.26429 bw: 5.0957 +[9:4] time: 1.26334 bw: 5.09954 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 1.25858 -time0: 1.27952 -bw: 5.11886 -bw0: 5.0352 -======================================= - -Slurm Job Hostlist: nid[005352,005356,005360,005362] -Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.0489155 -[0:0] time: 0.0156881 bw: 410.658 -[0:1] time: 0.0150684 bw: 427.548 -[0:2] time: 0.0177612 bw: 362.726 -[0:3] time: 0.0166887 bw: 386.036 -[0:4] time: 0.0644491 bw: 99.9618 -[1:0] time: 0.0157591 bw: 408.807 -[1:1] time: 0.0150642 bw: 427.668 -[1:2] time: 0.015068 bw: 427.559 -[1:3] time: 0.0150626 bw: 427.711 -[1:4] time: 0.0150714 bw: 427.462 -[2:0] time: 0.0159035 bw: 405.098 -[2:1] time: 0.015067 bw: 427.588 -[2:2] time: 0.0150681 bw: 427.557 -[2:3] time: 0.023429 bw: 274.978 -[2:4] time: 0.0150763 bw: 427.324 -[3:0] time: 0.0156805 bw: 410.857 -[3:1] time: 0.0150795 bw: 427.233 -[3:2] time: 0.0150731 bw: 427.414 -[3:3] time: 0.0150706 bw: 427.486 -[3:4] time: 0.01507 bw: 427.501 -[4:0] time: 0.0158737 bw: 405.857 -[4:1] time: 0.0150664 bw: 427.604 -[4:2] time: 0.0150659 bw: 427.617 -[4:3] time: 0.0150626 bw: 427.712 -[4:4] time: 0.0150635 bw: 427.685 -[5:0] time: 0.0158658 bw: 406.058 -[5:1] time: 0.0150823 bw: 427.153 -[5:2] time: 0.0150787 bw: 427.255 -[5:3] time: 0.0150716 bw: 427.457 -[5:4] time: 0.0150721 bw: 427.441 -[6:0] time: 0.0158779 bw: 405.749 -[6:1] time: 0.0150651 bw: 427.639 -[6:2] time: 0.0150648 bw: 427.648 -[6:3] time: 0.0150615 bw: 427.743 -[6:4] time: 0.0150632 bw: 427.695 -[7:0] time: 0.0158243 bw: 407.123 -[7:1] time: 0.0150658 bw: 427.622 -[7:2] time: 0.0150722 bw: 427.441 -[7:3] time: 0.0150726 bw: 427.429 -[7:4] time: 0.015066 bw: 427.616 -[8:0] time: 0.0159407 bw: 404.151 -[8:1] time: 0.0150754 bw: 427.348 -[8:2] time: 0.0150726 bw: 427.428 -[8:3] time: 0.0150671 bw: 427.583 -[8:4] time: 0.0150684 bw: 427.547 -[9:0] time: 0.0161478 bw: 398.968 -[9:1] time: 0.0150664 bw: 427.605 -[9:2] time: 0.0150707 bw: 427.482 -[9:3] time: 0.01507 bw: 427.501 -[9:4] time: 0.0150696 bw: 427.514 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.0166205 -time0: 0.0158562 -bw: 412.863 -bw0: 406.333 -======================================= - -Hostname: nid005362 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[005352,005356,005360,005362] -Hostname: nid005352 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.048395 -[0:0] time: 0.0160341 bw: 401.796 -[0:1] time: 0.0170961 bw: 376.838 -[0:2] time: 0.0160611 bw: 401.123 -[0:3] time: 0.0154836 bw: 416.082 -[0:4] time: 0.0154749 bw: 416.315 -[1:0] time: 0.0162892 bw: 395.504 -[1:1] time: 0.0154839 bw: 416.074 -[1:2] time: 0.0154867 bw: 415.998 -[1:3] time: 0.015489 bw: 415.936 -[1:4] time: 0.0154798 bw: 416.184 -[2:0] time: 0.0161949 bw: 397.807 -[2:1] time: 0.0154822 bw: 416.119 -[2:2] time: 0.0154785 bw: 416.22 -[2:3] time: 0.0154741 bw: 416.339 -[2:4] time: 0.0154698 bw: 416.452 -[3:0] time: 0.0162204 bw: 397.181 -[3:1] time: 0.0154798 bw: 416.186 -[3:2] time: 0.0154841 bw: 416.069 -[3:3] time: 0.015489 bw: 415.938 -[3:4] time: 0.0154909 bw: 415.887 -[4:0] time: 0.0161888 bw: 397.958 -[4:1] time: 0.0154772 bw: 416.255 -[4:2] time: 0.0154811 bw: 416.15 -[4:3] time: 0.0154731 bw: 416.364 -[4:4] time: 0.015482 bw: 416.125 -[5:0] time: 0.0162015 bw: 397.645 -[5:1] time: 0.0154887 bw: 415.946 -[5:2] time: 0.0154923 bw: 415.849 -[5:3] time: 0.0154873 bw: 415.983 -[5:4] time: 0.0154927 bw: 415.837 -[6:0] time: 0.0161586 bw: 398.702 -[6:1] time: 0.0154876 bw: 415.976 -[6:2] time: 0.0154866 bw: 416.001 -[6:3] time: 0.0154885 bw: 415.952 -[6:4] time: 0.0154811 bw: 416.149 -[7:0] time: 0.0161795 bw: 398.187 -[7:1] time: 0.0154785 bw: 416.219 -[7:2] time: 0.0154822 bw: 416.12 -[7:3] time: 0.0154799 bw: 416.181 -[7:4] time: 0.01548 bw: 416.178 -[8:0] time: 0.0161812 bw: 398.145 -[8:1] time: 0.0154757 bw: 416.295 -[8:2] time: 0.0154813 bw: 416.143 -[8:3] time: 0.0154779 bw: 416.235 -[8:4] time: 0.0154783 bw: 416.226 -[9:0] time: 0.0161852 bw: 398.045 -[9:1] time: 0.0154781 bw: 416.23 -[9:2] time: 0.015479 bw: 416.205 -[9:3] time: 0.015484 bw: 416.07 -[9:4] time: 0.0154837 bw: 416.081 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.015537 -time0: 0.0161833 -bw: 414.763 -bw0: 398.097 +time: 1.2621 +time0: 1.2877 +bw: 5.10458 +bw0: 5.00357 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out index e748bd8..7cbbc8e 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0004_TPN_4.out @@ -1,343 +1,205 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004328,004330,004350,004352] -Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005112 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005110 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005112 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005116 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005112 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005112 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005110 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005110 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005110 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005116 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005116 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005116 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005104 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[005104,005110,005112,005116] +Hostname: nid005104 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005104 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005104 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.504726 -[0:0] time: 0.241972 bw: 33.2809 -[0:1] time: 0.208666 bw: 38.593 -[0:2] time: 0.209733 bw: 38.3968 -[0:3] time: 0.208332 bw: 38.6549 -[0:4] time: 0.208196 bw: 38.6803 -[1:0] time: 0.241162 bw: 33.3928 -[1:1] time: 0.208529 bw: 38.6185 -[1:2] time: 0.208481 bw: 38.6274 -[1:3] time: 0.208321 bw: 38.6571 -[1:4] time: 0.208221 bw: 38.6756 -[2:0] time: 0.260553 bw: 30.9075 -[2:1] time: 0.208279 bw: 38.6648 -[2:2] time: 0.208373 bw: 38.6474 -[2:3] time: 0.208177 bw: 38.6838 -[2:4] time: 0.208313 bw: 38.6586 -[3:0] time: 0.258582 bw: 31.1431 -[3:1] time: 0.209427 bw: 38.4529 -[3:2] time: 0.208713 bw: 38.5844 -[3:3] time: 0.208182 bw: 38.6828 -[3:4] time: 0.20828 bw: 38.6646 -[4:0] time: 0.258232 bw: 31.1853 -[4:1] time: 0.209434 bw: 38.4516 -[4:2] time: 0.208277 bw: 38.6652 -[4:3] time: 0.210214 bw: 38.3089 -[4:4] time: 0.210498 bw: 38.2573 -[5:0] time: 0.258393 bw: 31.166 -[5:1] time: 0.20952 bw: 38.4358 -[5:2] time: 0.208331 bw: 38.6551 -[5:3] time: 0.208451 bw: 38.633 -[5:4] time: 0.208236 bw: 38.6727 -[6:0] time: 0.258274 bw: 31.1803 -[6:1] time: 0.209245 bw: 38.4864 -[6:2] time: 0.209108 bw: 38.5115 -[6:3] time: 0.208826 bw: 38.5635 -[6:4] time: 0.208938 bw: 38.5428 -[7:0] time: 0.25848 bw: 31.1555 -[7:1] time: 0.209007 bw: 38.5301 -[7:2] time: 0.210524 bw: 38.2525 -[7:3] time: 0.208789 bw: 38.5704 -[7:4] time: 0.208905 bw: 38.549 -[8:0] time: 0.260194 bw: 30.9502 -[8:1] time: 0.208724 bw: 38.5823 -[8:2] time: 0.208796 bw: 38.569 -[8:3] time: 0.208742 bw: 38.5789 -[8:4] time: 0.209112 bw: 38.5108 -[9:0] time: 0.261271 bw: 30.8226 -[9:1] time: 0.208765 bw: 38.5748 -[9:2] time: 0.208802 bw: 38.5679 -[9:3] time: 0.209494 bw: 38.4406 -[9:4] time: 0.209783 bw: 38.3877 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 0.500642 +[0:0] time: 0.239206 bw: 33.6658 +[0:1] time: 0.206948 bw: 38.9135 +[0:2] time: 0.206808 bw: 38.9398 +[0:3] time: 0.207116 bw: 38.8819 +[0:4] time: 0.20689 bw: 38.9244 +[1:0] time: 0.246294 bw: 32.697 +[1:1] time: 0.207924 bw: 38.7307 +[1:2] time: 0.208374 bw: 38.6472 +[1:3] time: 0.208828 bw: 38.5632 +[1:4] time: 0.20834 bw: 38.6534 +[2:0] time: 0.264207 bw: 30.4801 +[2:1] time: 0.207706 bw: 38.7715 +[2:2] time: 0.207717 bw: 38.7693 +[2:3] time: 0.207862 bw: 38.7423 +[2:4] time: 0.207736 bw: 38.7658 +[3:0] time: 0.260499 bw: 30.914 +[3:1] time: 0.208504 bw: 38.623 +[3:2] time: 0.20738 bw: 38.8323 +[3:3] time: 0.207533 bw: 38.8038 +[3:4] time: 0.208014 bw: 38.714 +[4:0] time: 0.25715 bw: 31.3166 +[4:1] time: 0.207243 bw: 38.858 +[4:2] time: 0.207417 bw: 38.8255 +[4:3] time: 0.207591 bw: 38.793 +[4:4] time: 0.207515 bw: 38.8071 +[5:0] time: 0.258211 bw: 31.1879 +[5:1] time: 0.207608 bw: 38.7897 +[5:2] time: 0.207838 bw: 38.7468 +[5:3] time: 0.207755 bw: 38.7624 +[5:4] time: 0.208053 bw: 38.7069 +[6:0] time: 0.258301 bw: 31.1771 +[6:1] time: 0.20744 bw: 38.8212 +[6:2] time: 0.207473 bw: 38.815 +[6:3] time: 0.207895 bw: 38.7361 +[6:4] time: 0.207609 bw: 38.7896 +[7:0] time: 0.25964 bw: 31.0163 +[7:1] time: 0.20721 bw: 38.8643 +[7:2] time: 0.207549 bw: 38.8009 +[7:3] time: 0.207338 bw: 38.8403 +[7:4] time: 0.207422 bw: 38.8246 +[8:0] time: 0.259748 bw: 31.0034 +[8:1] time: 0.207515 bw: 38.8071 +[8:2] time: 0.207697 bw: 38.7732 +[8:3] time: 0.207507 bw: 38.8087 +[8:4] time: 0.20742 bw: 38.825 +[9:0] time: 0.279802 bw: 28.7813 +[9:1] time: 0.207691 bw: 38.7742 +[9:2] time: 0.207532 bw: 38.804 +[9:3] time: 0.207587 bw: 38.7936 +[9:4] time: 0.20757 bw: 38.7969 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.208868 -time0: 0.255711 -bw: 38.556 -bw0: 31.5184 +time: 0.207629 +time0: 0.258306 +bw: 38.786 +bw0: 31.2239 ======================================= -Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004328,004330,004350,004352] -Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005112 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005112 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005112 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005112 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005116 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005110 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005104 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005104 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005110 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005104 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005104,005110,005112,005116] +Hostname: nid005104 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005110 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005110 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005116 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005116 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005116 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.3198 -[0:0] time: 15.2332 bw: 0.528653 -[0:1] time: 0.410208 bw: 19.6317 -[0:2] time: 0.409929 bw: 19.645 -[0:3] time: 0.409678 bw: 19.657 -[0:4] time: 0.409146 bw: 19.6826 -[1:0] time: 15.2707 bw: 0.527355 -[1:1] time: 0.409097 bw: 19.685 -[1:2] time: 0.412953 bw: 19.5011 -[1:3] time: 0.413536 bw: 19.4737 -[1:4] time: 0.410126 bw: 19.6356 -[2:0] time: 15.2757 bw: 0.527182 -[2:1] time: 0.409817 bw: 19.6504 -[2:2] time: 0.40926 bw: 19.6771 -[2:3] time: 0.408736 bw: 19.7023 -[2:4] time: 0.409518 bw: 19.6647 -[3:0] time: 15.2834 bw: 0.526916 -[3:1] time: 0.409126 bw: 19.6836 -[3:2] time: 0.409713 bw: 19.6554 -[3:3] time: 0.410242 bw: 19.63 -[3:4] time: 0.409805 bw: 19.6509 -[4:0] time: 15.2516 bw: 0.528013 -[4:1] time: 0.409713 bw: 19.6554 -[4:2] time: 0.409568 bw: 19.6623 -[4:3] time: 0.409725 bw: 19.6548 -[4:4] time: 0.410053 bw: 19.6391 -[5:0] time: 15.3014 bw: 0.526295 -[5:1] time: 0.409516 bw: 19.6649 -[5:2] time: 0.409567 bw: 19.6624 -[5:3] time: 0.409492 bw: 19.666 -[5:4] time: 0.409715 bw: 19.6553 -[6:0] time: 15.2714 bw: 0.52733 -[6:1] time: 0.41017 bw: 19.6335 -[6:2] time: 0.410395 bw: 19.6227 -[6:3] time: 0.410084 bw: 19.6376 -[6:4] time: 0.409973 bw: 19.6429 -[7:0] time: 15.344 bw: 0.524834 -[7:1] time: 0.410583 bw: 19.6137 -[7:2] time: 0.410555 bw: 19.6151 -[7:3] time: 0.410706 bw: 19.6079 -[7:4] time: 0.410923 bw: 19.5975 -[8:0] time: 15.2429 bw: 0.528317 -[8:1] time: 0.410859 bw: 19.6005 -[8:2] time: 0.410402 bw: 19.6224 -[8:3] time: 0.41054 bw: 19.6158 -[8:4] time: 0.410281 bw: 19.6282 -[9:0] time: 15.2713 bw: 0.527332 -[9:1] time: 0.409985 bw: 19.6423 -[9:2] time: 0.410118 bw: 19.636 -[9:3] time: 0.410345 bw: 19.6251 -[9:4] time: 0.409929 bw: 19.645 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 469.624 +[0:0] time: 15.1772 bw: 0.530601 +[0:1] time: 0.428426 bw: 18.7969 +[0:2] time: 0.428545 bw: 18.7917 +[0:3] time: 0.428402 bw: 18.7979 +[0:4] time: 0.428949 bw: 18.774 +[1:0] time: 15.211 bw: 0.529425 +[1:1] time: 0.429589 bw: 18.746 +[1:2] time: 0.429979 bw: 18.729 +[1:3] time: 0.429942 bw: 18.7306 +[1:4] time: 0.429654 bw: 18.7431 +[2:0] time: 15.1881 bw: 0.530221 +[2:1] time: 0.432375 bw: 18.6252 +[2:2] time: 0.429936 bw: 18.7308 +[2:3] time: 0.428647 bw: 18.7872 +[2:4] time: 0.428821 bw: 18.7796 +[3:0] time: 15.2023 bw: 0.529726 +[3:1] time: 0.42874 bw: 18.7831 +[3:2] time: 0.429002 bw: 18.7716 +[3:3] time: 0.429179 bw: 18.7639 +[3:4] time: 0.429417 bw: 18.7535 +[4:0] time: 15.1943 bw: 0.530007 +[4:1] time: 0.4291 bw: 18.7673 +[4:2] time: 0.42906 bw: 18.7691 +[4:3] time: 0.428887 bw: 18.7767 +[4:4] time: 0.4289 bw: 18.7761 +[5:0] time: 15.222 bw: 0.529041 +[5:1] time: 0.429323 bw: 18.7576 +[5:2] time: 0.429235 bw: 18.7615 +[5:3] time: 0.429779 bw: 18.7377 +[5:4] time: 0.429748 bw: 18.739 +[6:0] time: 15.2172 bw: 0.529208 +[6:1] time: 0.429641 bw: 18.7437 +[6:2] time: 0.429919 bw: 18.7316 +[6:3] time: 0.429647 bw: 18.7434 +[6:4] time: 0.429552 bw: 18.7476 +[7:0] time: 15.2562 bw: 0.527854 +[7:1] time: 0.429627 bw: 18.7443 +[7:2] time: 0.429232 bw: 18.7616 +[7:3] time: 0.42879 bw: 18.7809 +[7:4] time: 0.428927 bw: 18.7749 +[8:0] time: 15.2048 bw: 0.529638 +[8:1] time: 0.429346 bw: 18.7566 +[8:2] time: 0.428796 bw: 18.7806 +[8:3] time: 0.428835 bw: 18.7789 +[8:4] time: 0.4293 bw: 18.7586 +[9:0] time: 15.2254 bw: 0.528924 +[9:1] time: 0.429348 bw: 18.7565 +[9:2] time: 0.428742 bw: 18.783 +[9:3] time: 0.429006 bw: 18.7714 +[9:4] time: 0.428878 bw: 18.777 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.410102 -time0: 15.2746 -bw: 19.6368 -bw0: 0.527223 -======================================= - -Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004328,004330,004350,004352] -Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.169212 -[0:0] time: 0.0638838 bw: 126.058 -[0:1] time: 0.0638435 bw: 126.138 -[0:2] time: 0.0638601 bw: 126.105 -[0:3] time: 0.0638798 bw: 126.066 -[0:4] time: 0.0638359 bw: 126.153 -[1:0] time: 0.0639934 bw: 125.842 -[1:1] time: 0.0638425 bw: 126.14 -[1:2] time: 0.0638436 bw: 126.137 -[1:3] time: 0.0638445 bw: 126.136 -[1:4] time: 0.0638368 bw: 126.151 -[2:0] time: 0.0639946 bw: 125.84 -[2:1] time: 0.0638369 bw: 126.151 -[2:2] time: 0.0638402 bw: 126.144 -[2:3] time: 0.0638374 bw: 126.15 -[2:4] time: 0.0638435 bw: 126.138 -[3:0] time: 0.0639794 bw: 125.87 -[3:1] time: 0.0638491 bw: 126.126 -[3:2] time: 0.0638452 bw: 126.134 -[3:3] time: 0.0638336 bw: 126.157 -[3:4] time: 0.0638349 bw: 126.155 -[4:0] time: 0.0640293 bw: 125.772 -[4:1] time: 0.0638316 bw: 126.161 -[4:2] time: 0.0638163 bw: 126.191 -[4:3] time: 0.0638206 bw: 126.183 -[4:4] time: 0.0638374 bw: 126.15 -[5:0] time: 0.0639385 bw: 125.95 -[5:1] time: 0.0638342 bw: 126.156 -[5:2] time: 0.0638322 bw: 126.16 -[5:3] time: 0.0638383 bw: 126.148 -[5:4] time: 0.0638489 bw: 126.127 -[6:0] time: 0.0639692 bw: 125.89 -[6:1] time: 0.0638361 bw: 126.152 -[6:2] time: 0.0638218 bw: 126.18 -[6:3] time: 0.0638154 bw: 126.193 -[6:4] time: 0.0638323 bw: 126.16 -[7:0] time: 0.0711399 bw: 113.2 -[7:1] time: 0.0638318 bw: 126.161 -[7:2] time: 0.0638399 bw: 126.145 -[7:3] time: 0.0638385 bw: 126.147 -[7:4] time: 0.0638398 bw: 126.145 -[8:0] time: 0.0640239 bw: 125.782 -[8:1] time: 0.0638172 bw: 126.189 -[8:2] time: 0.0638161 bw: 126.192 -[8:3] time: 0.0638202 bw: 126.184 -[8:4] time: 0.0638288 bw: 126.167 -[9:0] time: 0.0639901 bw: 125.849 -[9:1] time: 0.0638294 bw: 126.165 -[9:2] time: 0.0638373 bw: 126.15 -[9:3] time: 0.0638898 bw: 126.046 -[9:4] time: 0.0667015 bw: 120.733 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.0639091 -time0: 0.0646942 -bw: 126.014 -bw0: 124.605 -======================================= - -Hostname: nid004328 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004328,004330,004350,004352] -Hostname: nid004328 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004350 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004350 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004352 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004330 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.172451 -[0:0] time: 0.0659321 bw: 122.142 -[0:1] time: 0.0658436 bw: 122.306 -[0:2] time: 0.0658629 bw: 122.27 -[0:3] time: 0.0658461 bw: 122.301 -[0:4] time: 0.0658377 bw: 122.317 -[1:0] time: 0.066035 bw: 121.951 -[1:1] time: 0.0658166 bw: 122.356 -[1:2] time: 0.0658196 bw: 122.351 -[1:3] time: 0.0658155 bw: 122.358 -[1:4] time: 0.065819 bw: 122.352 -[2:0] time: 0.0660486 bw: 121.926 -[2:1] time: 0.0658185 bw: 122.353 -[2:2] time: 0.0658158 bw: 122.358 -[2:3] time: 0.0658287 bw: 122.334 -[2:4] time: 0.0658313 bw: 122.329 -[3:0] time: 0.0660506 bw: 121.923 -[3:1] time: 0.065818 bw: 122.354 -[3:2] time: 0.0658214 bw: 122.347 -[3:3] time: 0.0658232 bw: 122.344 -[3:4] time: 0.0658267 bw: 122.337 -[4:0] time: 0.0660649 bw: 121.896 -[4:1] time: 0.0658102 bw: 122.368 -[4:2] time: 0.0658166 bw: 122.356 -[4:3] time: 0.0658205 bw: 122.349 -[4:4] time: 0.0658206 bw: 122.349 -[5:0] time: 0.066022 bw: 121.975 -[5:1] time: 0.0658203 bw: 122.349 -[5:2] time: 0.0658119 bw: 122.365 -[5:3] time: 0.0658274 bw: 122.336 -[5:4] time: 0.0658182 bw: 122.353 -[6:0] time: 0.0666281 bw: 120.866 -[6:1] time: 0.0658188 bw: 122.352 -[6:2] time: 0.0658227 bw: 122.345 -[6:3] time: 0.0658179 bw: 122.354 -[6:4] time: 0.065816 bw: 122.357 -[7:0] time: 0.0660048 bw: 122.007 -[7:1] time: 0.0658132 bw: 122.363 -[7:2] time: 0.0658203 bw: 122.349 -[7:3] time: 0.0658167 bw: 122.356 -[7:4] time: 0.065806 bw: 122.376 -[8:0] time: 0.066024 bw: 121.972 -[8:1] time: 0.065812 bw: 122.365 -[8:2] time: 0.0658108 bw: 122.367 -[8:3] time: 0.0658188 bw: 122.352 -[8:4] time: 0.0658144 bw: 122.36 -[9:0] time: 0.0660467 bw: 121.93 -[9:1] time: 0.0658218 bw: 122.346 -[9:2] time: 0.0658211 bw: 122.348 -[9:3] time: 0.065812 bw: 122.365 -[9:4] time: 0.0658237 bw: 122.343 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.0658214 -time0: 0.0660857 -bw: 122.347 -bw0: 121.859 +time: 0.42928 +time0: 15.2099 +bw: 18.7595 +bw0: 0.529464 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out index 4e10319..8283d26 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00016_N_0016_TPN_1.out @@ -1,343 +1,205 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] -Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001474 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001452 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001436,001440,001448,001450,001452,001454,001456,001458,001460,001462,001468,001470,001472,001474,001480,001482] +Hostname: nid001436 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001450 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001454 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001462 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001448 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001482 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001472 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001458 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001468 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001440 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001480 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001456 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001460 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001470 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.524098 -[0:0] time: 0.45217 bw: 17.8098 -[0:1] time: 0.431128 bw: 18.6791 -[0:2] time: 0.431328 bw: 18.6704 -[0:3] time: 0.431319 bw: 18.6708 -[0:4] time: 0.432948 bw: 18.6005 -[1:0] time: 0.452207 bw: 17.8084 -[1:1] time: 0.431044 bw: 18.6827 -[1:2] time: 0.431515 bw: 18.6623 -[1:3] time: 0.430952 bw: 18.6867 -[1:4] time: 0.432727 bw: 18.61 -[2:0] time: 0.45263 bw: 17.7917 -[2:1] time: 0.431193 bw: 18.6762 -[2:2] time: 0.431659 bw: 18.6561 -[2:3] time: 0.432598 bw: 18.6156 -[2:4] time: 0.431165 bw: 18.6775 -[3:0] time: 0.452535 bw: 17.7954 -[3:1] time: 0.430977 bw: 18.6856 -[3:2] time: 0.43096 bw: 18.6863 -[3:3] time: 0.43086 bw: 18.6907 -[3:4] time: 0.430864 bw: 18.6905 -[4:0] time: 0.45257 bw: 17.7941 -[4:1] time: 0.430953 bw: 18.6867 -[4:2] time: 0.430958 bw: 18.6864 -[4:3] time: 0.43102 bw: 18.6837 -[4:4] time: 0.430785 bw: 18.6939 -[5:0] time: 0.452741 bw: 17.7873 -[5:1] time: 0.431577 bw: 18.6596 -[5:2] time: 0.431073 bw: 18.6814 -[5:3] time: 0.430894 bw: 18.6892 -[5:4] time: 0.430903 bw: 18.6888 -[6:0] time: 0.452656 bw: 17.7907 -[6:1] time: 0.431624 bw: 18.6576 -[6:2] time: 0.431565 bw: 18.6602 -[6:3] time: 0.432193 bw: 18.633 -[6:4] time: 0.431477 bw: 18.6639 -[7:0] time: 0.452481 bw: 17.7976 -[7:1] time: 0.431624 bw: 18.6576 -[7:2] time: 0.431581 bw: 18.6594 -[7:3] time: 0.431617 bw: 18.6579 -[7:4] time: 0.431547 bw: 18.6609 -[8:0] time: 0.453368 bw: 17.7628 -[8:1] time: 0.431407 bw: 18.667 -[8:2] time: 0.431571 bw: 18.6599 -[8:3] time: 0.431661 bw: 18.656 -[8:4] time: 0.431576 bw: 18.6597 -[9:0] time: 0.453671 bw: 17.7509 -[9:1] time: 0.43157 bw: 18.6599 -[9:2] time: 0.431371 bw: 18.6685 -[9:3] time: 0.431705 bw: 18.6541 -[9:4] time: 0.431542 bw: 18.6611 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 0.544043 +[0:0] time: 0.460401 bw: 17.4914 +[0:1] time: 0.430451 bw: 18.7084 +[0:2] time: 0.431837 bw: 18.6484 +[0:3] time: 0.43051 bw: 18.7059 +[0:4] time: 0.43055 bw: 18.7042 +[1:0] time: 0.460077 bw: 17.5037 +[1:1] time: 0.431383 bw: 18.668 +[1:2] time: 0.43346 bw: 18.5786 +[1:3] time: 0.432142 bw: 18.6352 +[1:4] time: 0.432288 bw: 18.6289 +[2:0] time: 0.491603 bw: 16.3812 +[2:1] time: 0.431448 bw: 18.6652 +[2:2] time: 0.431298 bw: 18.6717 +[2:3] time: 0.43129 bw: 18.672 +[2:4] time: 0.431293 bw: 18.6719 +[3:0] time: 0.487787 bw: 16.5094 +[3:1] time: 0.431321 bw: 18.6707 +[3:2] time: 0.431103 bw: 18.6802 +[3:3] time: 0.431256 bw: 18.6735 +[3:4] time: 0.431109 bw: 18.6799 +[4:0] time: 0.487525 bw: 16.5183 +[4:1] time: 0.431085 bw: 18.6809 +[4:2] time: 0.431122 bw: 18.6793 +[4:3] time: 0.430902 bw: 18.6889 +[4:4] time: 0.430995 bw: 18.6848 +[5:0] time: 0.487787 bw: 16.5094 +[5:1] time: 0.430869 bw: 18.6903 +[5:2] time: 0.430848 bw: 18.6912 +[5:3] time: 0.431181 bw: 18.6767 +[5:4] time: 0.430973 bw: 18.6858 +[6:0] time: 0.486025 bw: 16.5692 +[6:1] time: 0.431088 bw: 18.6808 +[6:2] time: 0.430854 bw: 18.6909 +[6:3] time: 0.430995 bw: 18.6848 +[6:4] time: 0.430922 bw: 18.688 +[7:0] time: 0.486638 bw: 16.5484 +[7:1] time: 0.43104 bw: 18.6829 +[7:2] time: 0.43081 bw: 18.6928 +[7:3] time: 0.430832 bw: 18.6919 +[7:4] time: 0.430941 bw: 18.6871 +[8:0] time: 0.486513 bw: 16.5526 +[8:1] time: 0.430779 bw: 18.6942 +[8:2] time: 0.430803 bw: 18.6931 +[8:3] time: 0.43084 bw: 18.6915 +[8:4] time: 0.431107 bw: 18.6799 +[9:0] time: 0.486143 bw: 16.5652 +[9:1] time: 0.430775 bw: 18.6944 +[9:2] time: 0.431109 bw: 18.6799 +[9:3] time: 0.432094 bw: 18.6373 +[9:4] time: 0.430957 bw: 18.6865 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.431426 -time0: 0.452703 -bw: 18.6662 -bw0: 17.7889 +time: 0.431167 +time0: 0.48205 +bw: 18.6774 +bw0: 16.7149 ======================================= -Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] -Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001440 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001436,001440,001448,001450,001452,001454,001456,001458,001460,001462,001468,001470,001472,001474,001480,001482] +Hostname: nid001436 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001482 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001448 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001474 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001458 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001472 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001454 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001452 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001468 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001462 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001460 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001470 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001456 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001480 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001450 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 1.53478 -[0:0] time: 1.51548 bw: 5.31388 -[0:1] time: 1.49732 bw: 5.37831 -[0:2] time: 1.50292 bw: 5.35828 -[0:3] time: 1.49913 bw: 5.37183 -[0:4] time: 1.49535 bw: 5.38541 -[1:0] time: 1.5503 bw: 5.19453 -[1:1] time: 1.4977 bw: 5.37694 -[1:2] time: 1.49808 bw: 5.37558 -[1:3] time: 1.49939 bw: 5.37089 -[1:4] time: 1.49805 bw: 5.37568 -[2:0] time: 1.556 bw: 5.17551 -[2:1] time: 1.5002 bw: 5.36798 -[2:2] time: 1.49921 bw: 5.37152 -[2:3] time: 1.4992 bw: 5.37158 -[2:4] time: 1.49764 bw: 5.37715 -[3:0] time: 1.55047 bw: 5.19396 -[3:1] time: 1.49872 bw: 5.3733 -[3:2] time: 1.49969 bw: 5.36981 -[3:3] time: 1.50234 bw: 5.36033 -[3:4] time: 1.50162 bw: 5.36293 -[4:0] time: 1.56192 bw: 5.15589 -[4:1] time: 1.50549 bw: 5.34915 -[4:2] time: 1.50628 bw: 5.34634 -[4:3] time: 1.50297 bw: 5.35811 -[4:4] time: 1.50391 bw: 5.35475 -[5:0] time: 1.55689 bw: 5.17255 -[5:1] time: 1.50599 bw: 5.34737 -[5:2] time: 1.50475 bw: 5.35176 -[5:3] time: 1.50298 bw: 5.35808 -[5:4] time: 1.50195 bw: 5.36173 -[6:0] time: 1.5636 bw: 5.15035 -[6:1] time: 1.50561 bw: 5.3487 -[6:2] time: 1.50612 bw: 5.34689 -[6:3] time: 1.50444 bw: 5.35286 -[6:4] time: 1.50356 bw: 5.356 -[7:0] time: 1.55807 bw: 5.16862 -[7:1] time: 1.50621 bw: 5.34657 -[7:2] time: 1.51077 bw: 5.33043 -[7:3] time: 1.51147 bw: 5.32798 -[7:4] time: 1.50719 bw: 5.34312 -[8:0] time: 1.56196 bw: 5.15575 -[8:1] time: 1.50816 bw: 5.33967 -[8:2] time: 1.50822 bw: 5.33944 -[8:3] time: 1.50661 bw: 5.34515 -[8:4] time: 1.50987 bw: 5.33361 -[9:0] time: 1.56375 bw: 5.14983 -[9:1] time: 1.51397 bw: 5.31917 -[9:2] time: 1.51274 bw: 5.32349 -[9:3] time: 1.5127 bw: 5.32364 -[9:4] time: 1.50884 bw: 5.33725 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.53486 +[0:0] time: 1.51222 bw: 5.32533 +[0:1] time: 1.50479 bw: 5.3516 +[0:2] time: 1.50534 bw: 5.34965 +[0:3] time: 1.5043 bw: 5.35337 +[0:4] time: 1.50498 bw: 5.35093 +[1:0] time: 1.56888 bw: 5.13299 +[1:1] time: 1.51047 bw: 5.3315 +[1:2] time: 1.50726 bw: 5.34285 +[1:3] time: 1.50623 bw: 5.34651 +[1:4] time: 1.50611 bw: 5.34693 +[2:0] time: 1.56579 bw: 5.14313 +[2:1] time: 1.51202 bw: 5.32604 +[2:2] time: 1.5127 bw: 5.32364 +[2:3] time: 1.51126 bw: 5.32872 +[2:4] time: 1.51101 bw: 5.32958 +[3:0] time: 1.57254 bw: 5.12107 +[3:1] time: 1.51594 bw: 5.31224 +[3:2] time: 1.51335 bw: 5.32136 +[3:3] time: 1.51449 bw: 5.31733 +[3:4] time: 1.5098 bw: 5.33385 +[4:0] time: 1.57037 bw: 5.12813 +[4:1] time: 1.51406 bw: 5.31887 +[4:2] time: 1.5146 bw: 5.31694 +[4:3] time: 1.51361 bw: 5.32042 +[4:4] time: 1.51301 bw: 5.32255 +[5:0] time: 1.57284 bw: 5.12007 +[5:1] time: 1.51537 bw: 5.31425 +[5:2] time: 1.51815 bw: 5.30452 +[5:3] time: 1.51731 bw: 5.30745 +[5:4] time: 1.51633 bw: 5.31091 +[6:0] time: 1.57378 bw: 5.11702 +[6:1] time: 1.51558 bw: 5.31353 +[6:2] time: 1.51414 bw: 5.31858 +[6:3] time: 1.51711 bw: 5.30815 +[6:4] time: 1.51633 bw: 5.31088 +[7:0] time: 1.57867 bw: 5.10118 +[7:1] time: 1.51743 bw: 5.30703 +[7:2] time: 1.51507 bw: 5.3153 +[7:3] time: 1.51834 bw: 5.30386 +[7:4] time: 1.51899 bw: 5.30161 +[8:0] time: 1.57902 bw: 5.10004 +[8:1] time: 1.51826 bw: 5.30413 +[8:2] time: 1.51821 bw: 5.30432 +[8:3] time: 1.51586 bw: 5.31255 +[8:4] time: 1.51729 bw: 5.30752 +[9:0] time: 1.57964 bw: 5.09803 +[9:1] time: 1.51781 bw: 5.30571 +[9:2] time: 1.51528 bw: 5.31459 +[9:3] time: 1.51644 bw: 5.31051 +[9:4] time: 1.51849 bw: 5.30335 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 1.50393 -time0: 1.55384 -bw: 5.35472 -bw0: 5.18309 -======================================= - -Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] -Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.114812 -[0:0] time: 0.0814363 bw: 98.8879 -[0:1] time: 0.0637716 bw: 126.28 -[0:2] time: 0.0637575 bw: 126.308 -[0:3] time: 0.0637743 bw: 126.274 -[0:4] time: 0.063765 bw: 126.293 -[1:0] time: 0.0638736 bw: 126.078 -[1:1] time: 0.0637744 bw: 126.274 -[1:2] time: 0.0637744 bw: 126.274 -[1:3] time: 0.0637857 bw: 126.252 -[1:4] time: 0.0637804 bw: 126.262 -[2:0] time: 0.0639457 bw: 125.936 -[2:1] time: 0.0637546 bw: 126.314 -[2:2] time: 0.0637755 bw: 126.272 -[2:3] time: 0.063776 bw: 126.271 -[2:4] time: 0.0637746 bw: 126.274 -[3:0] time: 0.0640146 bw: 125.8 -[3:1] time: 0.0637866 bw: 126.25 -[3:2] time: 0.0637704 bw: 126.282 -[3:3] time: 0.0637741 bw: 126.275 -[3:4] time: 0.0637741 bw: 126.275 -[4:0] time: 0.0639243 bw: 125.978 -[4:1] time: 0.0637621 bw: 126.299 -[4:2] time: 0.0639204 bw: 125.986 -[4:3] time: 0.0637927 bw: 126.238 -[4:4] time: 0.0637901 bw: 126.243 -[5:0] time: 0.0638428 bw: 126.139 -[5:1] time: 0.0637724 bw: 126.278 -[5:2] time: 0.0637812 bw: 126.261 -[5:3] time: 0.0637823 bw: 126.258 -[5:4] time: 0.0637861 bw: 126.251 -[6:0] time: 0.0638597 bw: 126.106 -[6:1] time: 0.0637637 bw: 126.295 -[6:2] time: 0.063777 bw: 126.269 -[6:3] time: 0.0639618 bw: 125.904 -[6:4] time: 0.063799 bw: 126.226 -[7:0] time: 0.0638307 bw: 126.163 -[7:1] time: 0.0637793 bw: 126.264 -[7:2] time: 0.0637712 bw: 126.28 -[7:3] time: 0.0637853 bw: 126.253 -[7:4] time: 0.0637796 bw: 126.264 -[8:0] time: 0.0638549 bw: 126.115 -[8:1] time: 0.0637794 bw: 126.264 -[8:2] time: 0.0637726 bw: 126.278 -[8:3] time: 0.0637647 bw: 126.293 -[8:4] time: 0.0637743 bw: 126.274 -[9:0] time: 0.0638479 bw: 126.129 -[9:1] time: 0.0637783 bw: 126.266 -[9:2] time: 0.0637822 bw: 126.259 -[9:3] time: 0.0637631 bw: 126.297 -[9:4] time: 0.0637697 bw: 126.284 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.0637839 -time0: 0.065643 -bw: 126.255 -bw0: 123.333 -======================================= - -Slurm Job Hostlist: nid[004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752] -Hostname: nid004702 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.107586 -[0:0] time: 0.0729256 bw: 110.429 -[0:1] time: 0.0706225 bw: 114.03 -[0:2] time: 0.0658227 bw: 122.345 -[0:3] time: 0.0657996 bw: 122.388 -[0:4] time: 0.0657915 bw: 122.403 -[1:0] time: 0.0659209 bw: 122.162 -[1:1] time: 0.065774 bw: 122.435 -[1:2] time: 0.0657776 bw: 122.429 -[1:3] time: 0.0657843 bw: 122.416 -[1:4] time: 0.065796 bw: 122.394 -[2:0] time: 0.0660119 bw: 121.994 -[2:1] time: 0.0657994 bw: 122.388 -[2:2] time: 0.0658116 bw: 122.365 -[2:3] time: 0.0658104 bw: 122.368 -[2:4] time: 0.0657992 bw: 122.389 -[3:0] time: 0.0659366 bw: 122.133 -[3:1] time: 0.0657833 bw: 122.418 -[3:2] time: 0.0657837 bw: 122.417 -[3:3] time: 0.0657796 bw: 122.425 -[3:4] time: 0.0657807 bw: 122.423 -[4:0] time: 0.0659165 bw: 122.171 -[4:1] time: 0.0658036 bw: 122.38 -[4:2] time: 0.0658015 bw: 122.384 -[4:3] time: 0.0658102 bw: 122.368 -[4:4] time: 0.0658293 bw: 122.332 -[5:0] time: 0.0660176 bw: 121.984 -[5:1] time: 0.0657672 bw: 122.448 -[5:2] time: 0.0657704 bw: 122.442 -[5:3] time: 0.0657876 bw: 122.41 -[5:4] time: 0.0657723 bw: 122.438 -[6:0] time: 0.065951 bw: 122.107 -[6:1] time: 0.0658052 bw: 122.377 -[6:2] time: 0.0658152 bw: 122.359 -[6:3] time: 0.0658016 bw: 122.384 -[6:4] time: 0.0658126 bw: 122.364 -[7:0] time: 0.0659276 bw: 122.15 -[7:1] time: 0.0657712 bw: 122.441 -[7:2] time: 0.065771 bw: 122.441 -[7:3] time: 0.0657692 bw: 122.444 -[7:4] time: 0.0657852 bw: 122.415 -[8:0] time: 0.0659151 bw: 122.173 -[8:1] time: 0.065794 bw: 122.398 -[8:2] time: 0.0658168 bw: 122.356 -[8:3] time: 0.0658174 bw: 122.355 -[8:4] time: 0.0658179 bw: 122.354 -[9:0] time: 0.0658831 bw: 122.233 -[9:1] time: 0.0657896 bw: 122.406 -[9:2] time: 0.0657703 bw: 122.442 -[9:3] time: 0.0657817 bw: 122.421 -[9:4] time: 0.0657768 bw: 122.43 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.0659138 -time0: 0.0666406 -bw: 122.191 -bw0: 120.954 +time: 1.51358 +time0: 1.56738 +bw: 5.32059 +bw0: 5.1387 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out index 762d97e..f11289c 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0008_TPN_4.out @@ -1,407 +1,269 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] -Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005138 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005134 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005132 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005138 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005136 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005122 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005138 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005138 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005134 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005148 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005134 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005134 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005132 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005132 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005136 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005136 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005132 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005136 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005148 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005122 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005148 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[005122,005130,005132,005134,005136,005138,005146,005148] +Hostname: nid005122 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005122 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005148 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005130 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005146 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005130 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005146 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005130 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005146 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005130 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005146 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.5358 -[0:0] time: 0.272119 bw: 30.5803 -[0:1] time: 0.237648 bw: 35.0161 -[0:2] time: 0.238383 bw: 34.9082 -[0:3] time: 0.238066 bw: 34.9546 -[0:4] time: 0.238775 bw: 34.8508 -[1:0] time: 0.274917 bw: 30.2691 -[1:1] time: 0.240515 bw: 34.5987 -[1:2] time: 0.240849 bw: 34.5506 -[1:3] time: 0.238045 bw: 34.9577 -[1:4] time: 0.239615 bw: 34.7286 -[2:0] time: 0.287758 bw: 28.9184 -[2:1] time: 0.239333 bw: 34.7696 -[2:2] time: 0.238508 bw: 34.8898 -[2:3] time: 0.237994 bw: 34.9652 -[2:4] time: 0.237478 bw: 35.0411 -[3:0] time: 0.286365 bw: 29.0591 -[3:1] time: 0.23751 bw: 35.0364 -[3:2] time: 0.237439 bw: 35.0469 -[3:3] time: 0.237622 bw: 35.0199 -[3:4] time: 0.237657 bw: 35.0148 -[4:0] time: 0.289036 bw: 28.7906 -[4:1] time: 0.237409 bw: 35.0513 -[4:2] time: 0.237221 bw: 35.0791 -[4:3] time: 0.237719 bw: 35.0056 -[4:4] time: 0.23746 bw: 35.0438 -[5:0] time: 0.288283 bw: 28.8657 -[5:1] time: 0.237349 bw: 35.0602 -[5:2] time: 0.23738 bw: 35.0557 -[5:3] time: 0.237856 bw: 34.9854 -[5:4] time: 0.237546 bw: 35.031 -[6:0] time: 0.289813 bw: 28.7133 -[6:1] time: 0.238364 bw: 34.9109 -[6:2] time: 0.238187 bw: 34.9369 -[6:3] time: 0.23819 bw: 34.9363 -[6:4] time: 0.238636 bw: 34.871 -[7:0] time: 0.28947 bw: 28.7474 -[7:1] time: 0.238051 bw: 34.9567 -[7:2] time: 0.238143 bw: 34.9433 -[7:3] time: 0.237987 bw: 34.9662 -[7:4] time: 0.238179 bw: 34.938 -[8:0] time: 0.292047 bw: 28.4937 -[8:1] time: 0.23808 bw: 34.9526 -[8:2] time: 0.237888 bw: 34.9807 -[8:3] time: 0.238048 bw: 34.9573 -[8:4] time: 0.237946 bw: 34.9722 -[9:0] time: 0.294056 bw: 28.299 -[9:1] time: 0.237718 bw: 35.0057 -[9:2] time: 0.2376 bw: 35.0232 -[9:3] time: 0.23795 bw: 34.9716 -[9:4] time: 0.239134 bw: 34.7985 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +[-1] time: 0.538603 +[0:0] time: 0.273301 bw: 30.4481 +[0:1] time: 0.238777 bw: 34.8505 +[0:2] time: 0.239052 bw: 34.8104 +[0:3] time: 0.23917 bw: 34.7932 +[0:4] time: 0.241143 bw: 34.5086 +[1:0] time: 0.27859 bw: 29.87 +[1:1] time: 0.243268 bw: 34.2071 +[1:2] time: 0.243154 bw: 34.2231 +[1:3] time: 0.241263 bw: 34.4913 +[1:4] time: 0.241716 bw: 34.4268 +[2:0] time: 0.293815 bw: 28.3223 +[2:1] time: 0.241516 bw: 34.4552 +[2:2] time: 0.239747 bw: 34.7095 +[2:3] time: 0.239516 bw: 34.7429 +[2:4] time: 0.23974 bw: 34.7105 +[3:0] time: 0.291196 bw: 28.577 +[3:1] time: 0.239702 bw: 34.7161 +[3:2] time: 0.239863 bw: 34.6927 +[3:3] time: 0.239983 bw: 34.6754 +[3:4] time: 0.23984 bw: 34.696 +[4:0] time: 0.291961 bw: 28.5021 +[4:1] time: 0.239877 bw: 34.6907 +[4:2] time: 0.239806 bw: 34.701 +[4:3] time: 0.23961 bw: 34.7293 +[4:4] time: 0.239471 bw: 34.7496 +[5:0] time: 0.290183 bw: 28.6767 +[5:1] time: 0.239997 bw: 34.6734 +[5:2] time: 0.239811 bw: 34.7002 +[5:3] time: 0.239709 bw: 34.7151 +[5:4] time: 0.239766 bw: 34.7068 +[6:0] time: 0.288638 bw: 28.8302 +[6:1] time: 0.239916 bw: 34.685 +[6:2] time: 0.240373 bw: 34.6191 +[6:3] time: 0.239905 bw: 34.6867 +[6:4] time: 0.239941 bw: 34.6814 +[7:0] time: 0.292458 bw: 28.4536 +[7:1] time: 0.239816 bw: 34.6995 +[7:2] time: 0.239856 bw: 34.6938 +[7:3] time: 0.239695 bw: 34.7171 +[7:4] time: 0.240206 bw: 34.6431 +[8:0] time: 0.289533 bw: 28.7411 +[8:1] time: 0.239991 bw: 34.6742 +[8:2] time: 0.239932 bw: 34.6828 +[8:3] time: 0.239797 bw: 34.7022 +[8:4] time: 0.239953 bw: 34.6798 +[9:0] time: 0.289096 bw: 28.7846 +[9:1] time: 0.239663 bw: 34.7216 +[9:2] time: 0.239776 bw: 34.7053 +[9:3] time: 0.239813 bw: 34.7 +[9:4] time: 0.239732 bw: 34.7117 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.238137 -time0: 0.286386 -bw: 34.9446 -bw0: 29.0737 +time: 0.240097 +time0: 0.287877 +bw: 34.6595 +bw0: 28.9206 ======================================= -Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] -Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005130 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005122 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005148 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005136 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005130 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005122 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005122 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005122,005130,005132,005134,005136,005138,005146,005148] +Hostname: nid005122 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005130 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005134 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005130 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005136 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005148 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005148 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005136 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005148 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005136 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005134 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005134 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005134 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005138 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005138 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005138 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005138 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005132 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005132 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005132 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005132 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005146 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005146 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005146 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005146 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.3317 -[0:0] time: 15.3146 bw: 0.543371 -[0:1] time: 0.488911 bw: 17.0205 -[0:2] time: 0.488674 bw: 17.0287 -[0:3] time: 0.495813 bw: 16.7835 -[0:4] time: 0.494635 bw: 16.8235 -[1:0] time: 15.3212 bw: 0.543136 -[1:1] time: 0.491453 bw: 16.9325 -[1:2] time: 0.489134 bw: 17.0127 -[1:3] time: 0.488371 bw: 17.0393 -[1:4] time: 0.489111 bw: 17.0135 -[2:0] time: 15.3396 bw: 0.542484 -[2:1] time: 0.489068 bw: 17.015 -[2:2] time: 0.488835 bw: 17.0231 -[2:3] time: 0.489616 bw: 16.996 -[2:4] time: 0.489063 bw: 17.0152 -[3:0] time: 15.3209 bw: 0.543147 -[3:1] time: 0.487861 bw: 17.0571 -[3:2] time: 0.488828 bw: 17.0234 -[3:3] time: 0.488735 bw: 17.0266 -[3:4] time: 0.489019 bw: 17.0167 -[4:0] time: 15.3538 bw: 0.541982 -[4:1] time: 0.488568 bw: 17.0324 -[4:2] time: 0.48805 bw: 17.0505 -[4:3] time: 0.489084 bw: 17.0145 -[4:4] time: 0.489108 bw: 17.0136 -[5:0] time: 15.3286 bw: 0.542875 -[5:1] time: 0.487746 bw: 17.0611 -[5:2] time: 0.488908 bw: 17.0206 -[5:3] time: 0.488416 bw: 17.0377 -[5:4] time: 0.489023 bw: 17.0166 -[6:0] time: 15.3282 bw: 0.542889 -[6:1] time: 0.489106 bw: 17.0137 -[6:2] time: 0.489054 bw: 17.0155 -[6:3] time: 0.489433 bw: 17.0023 -[6:4] time: 0.489527 bw: 16.9991 -[7:0] time: 15.3116 bw: 0.543478 -[7:1] time: 0.489192 bw: 17.0107 -[7:2] time: 0.48956 bw: 16.9979 -[7:3] time: 0.489072 bw: 17.0149 -[7:4] time: 0.489586 bw: 16.997 -[8:0] time: 15.3319 bw: 0.542759 -[8:1] time: 0.489171 bw: 17.0114 -[8:2] time: 0.488715 bw: 17.0273 -[8:3] time: 0.489309 bw: 17.0066 -[8:4] time: 0.489263 bw: 17.0082 -[9:0] time: 15.333 bw: 0.542719 -[9:1] time: 0.489545 bw: 16.9984 -[9:2] time: 0.489924 bw: 16.9853 -[9:3] time: 0.489367 bw: 17.0046 -[9:4] time: 0.489619 bw: 16.9959 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +[-1] time: 463.391 +[0:0] time: 15.3355 bw: 0.54263 +[0:1] time: 0.481525 bw: 17.2816 +[0:2] time: 0.482142 bw: 17.2594 +[0:3] time: 0.487549 bw: 17.068 +[0:4] time: 0.484869 bw: 17.1624 +[1:0] time: 15.3726 bw: 0.541321 +[1:1] time: 0.482899 bw: 17.2324 +[1:2] time: 0.484597 bw: 17.172 +[1:3] time: 0.481815 bw: 17.2712 +[1:4] time: 0.48324 bw: 17.2202 +[2:0] time: 15.3403 bw: 0.542461 +[2:1] time: 0.482558 bw: 17.2445 +[2:2] time: 0.481915 bw: 17.2676 +[2:3] time: 0.483278 bw: 17.2189 +[2:4] time: 0.483435 bw: 17.2133 +[3:0] time: 15.4012 bw: 0.540316 +[3:1] time: 0.481833 bw: 17.2705 +[3:2] time: 0.483248 bw: 17.2199 +[3:3] time: 0.48099 bw: 17.3008 +[3:4] time: 0.483012 bw: 17.2284 +[4:0] time: 15.3372 bw: 0.542568 +[4:1] time: 0.482503 bw: 17.2465 +[4:2] time: 0.482058 bw: 17.2624 +[4:3] time: 0.48381 bw: 17.1999 +[4:4] time: 0.48303 bw: 17.2277 +[5:0] time: 15.3311 bw: 0.542785 +[5:1] time: 0.482504 bw: 17.2465 +[5:2] time: 0.483031 bw: 17.2277 +[5:3] time: 0.481571 bw: 17.2799 +[5:4] time: 0.483638 bw: 17.206 +[6:0] time: 15.3709 bw: 0.541379 +[6:1] time: 0.482876 bw: 17.2332 +[6:2] time: 0.482019 bw: 17.2638 +[6:3] time: 0.483264 bw: 17.2194 +[6:4] time: 0.483438 bw: 17.2132 +[7:0] time: 15.3342 bw: 0.542676 +[7:1] time: 0.481534 bw: 17.2812 +[7:2] time: 0.48258 bw: 17.2438 +[7:3] time: 0.481212 bw: 17.2928 +[7:4] time: 0.48304 bw: 17.2274 +[8:0] time: 15.358 bw: 0.541836 +[8:1] time: 0.482466 bw: 17.2479 +[8:2] time: 0.48233 bw: 17.2527 +[8:3] time: 0.482876 bw: 17.2332 +[8:4] time: 0.482636 bw: 17.2418 +[9:0] time: 15.3777 bw: 0.541141 +[9:1] time: 0.481157 bw: 17.2948 +[9:2] time: 0.482218 bw: 17.2567 +[9:3] time: 0.481511 bw: 17.2821 +[9:4] time: 0.482903 bw: 17.2322 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.489387 -time0: 15.3283 -bw: 17.0041 -bw0: 0.542884 -======================================= - -Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] -Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.23378 -[0:0] time: 0.129277 bw: 64.3697 -[0:1] time: 0.128748 bw: 64.6338 -[0:2] time: 0.129379 bw: 64.3187 -[0:3] time: 0.130838 bw: 63.6015 -[0:4] time: 0.133852 bw: 62.1694 -[1:0] time: 0.138144 bw: 60.2379 -[1:1] time: 0.135822 bw: 61.2676 -[1:2] time: 0.140804 bw: 59.0999 -[1:3] time: 0.159225 bw: 52.2625 -[1:4] time: 0.218732 bw: 38.0443 -[2:0] time: 0.219306 bw: 37.9447 -[2:1] time: 0.232804 bw: 35.7447 -[2:2] time: 0.235519 bw: 35.3326 -[2:3] time: 0.237181 bw: 35.085 -[2:4] time: 0.238294 bw: 34.9212 -[3:0] time: 0.199769 bw: 41.6556 -[3:1] time: 0.230894 bw: 36.0403 -[3:2] time: 0.242589 bw: 34.3029 -[3:3] time: 0.241189 bw: 34.502 -[3:4] time: 0.243914 bw: 34.1165 -[4:0] time: 0.224221 bw: 37.1129 -[4:1] time: 0.240922 bw: 34.5403 -[4:2] time: 0.243774 bw: 34.1361 -[4:3] time: 0.244955 bw: 33.9715 -[4:4] time: 0.244927 bw: 33.9754 -[5:0] time: 0.2026 bw: 41.0735 -[5:1] time: 0.219568 bw: 37.8994 -[5:2] time: 0.246739 bw: 33.7259 -[5:3] time: 0.247412 bw: 33.6341 -[5:4] time: 0.249365 bw: 33.3708 -[6:0] time: 0.226711 bw: 36.7053 -[6:1] time: 0.243836 bw: 34.1275 -[6:2] time: 0.244916 bw: 33.9769 -[6:3] time: 0.244891 bw: 33.9805 -[6:4] time: 0.245022 bw: 33.9622 -[7:0] time: 0.208145 bw: 39.9793 -[7:1] time: 0.218671 bw: 38.0549 -[7:2] time: 0.247365 bw: 33.6406 -[7:3] time: 0.245194 bw: 33.9384 -[7:4] time: 0.246479 bw: 33.7615 -[8:0] time: 0.211283 bw: 39.3856 -[8:1] time: 0.227064 bw: 36.6483 -[8:2] time: 0.262478 bw: 31.7036 -[8:3] time: 0.261062 bw: 31.8756 -[8:4] time: 0.261084 bw: 31.8729 -[9:0] time: 0.21635 bw: 38.4632 -[9:1] time: 0.230681 bw: 36.0736 -[9:2] time: 0.232946 bw: 35.7229 -[9:3] time: 0.245782 bw: 33.8573 -[9:4] time: 0.262406 bw: 31.7123 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.223433 -time0: 0.197581 -bw: 39.1401 -bw0: 43.6928 -======================================= - -Hostname: nid004374 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004360 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004360,004374,004380,004382,004384,004392,004394,004402] -Hostname: nid004360 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004374 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004392 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004374 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004382 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004394 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.241767 -[0:0] time: 0.136765 bw: 60.8453 -[0:1] time: 0.133156 bw: 62.4946 -[0:2] time: 0.138198 bw: 60.2142 -[0:3] time: 0.135602 bw: 61.3672 -[0:4] time: 0.139155 bw: 59.8001 -[1:0] time: 0.139563 bw: 59.6254 -[1:1] time: 0.147225 bw: 56.5223 -[1:2] time: 0.162121 bw: 51.329 -[1:3] time: 0.203298 bw: 40.9325 -[1:4] time: 0.206365 bw: 40.3242 -[2:0] time: 0.191855 bw: 43.3739 -[2:1] time: 0.20539 bw: 40.5156 -[2:2] time: 0.221643 bw: 37.5445 -[2:3] time: 0.229658 bw: 36.2342 -[2:4] time: 0.229823 bw: 36.2084 -[3:0] time: 0.214055 bw: 38.8755 -[3:1] time: 0.241419 bw: 34.4691 -[3:2] time: 0.246477 bw: 33.7617 -[3:3] time: 0.249312 bw: 33.3778 -[3:4] time: 0.249366 bw: 33.3706 -[4:0] time: 0.232967 bw: 35.7196 -[4:1] time: 0.247684 bw: 33.5973 -[4:2] time: 0.246973 bw: 33.694 -[4:3] time: 0.248154 bw: 33.5336 -[4:4] time: 0.250923 bw: 33.1635 -[5:0] time: 0.235223 bw: 35.3771 -[5:1] time: 0.246523 bw: 33.7555 -[5:2] time: 0.245224 bw: 33.9343 -[5:3] time: 0.251414 bw: 33.0988 -[5:4] time: 0.27345 bw: 30.4315 -[6:0] time: 0.256066 bw: 32.4975 -[6:1] time: 0.245473 bw: 33.8998 -[6:2] time: 0.248569 bw: 33.4776 -[6:3] time: 0.247796 bw: 33.582 -[6:4] time: 0.248193 bw: 33.5284 -[7:0] time: 0.212248 bw: 39.2065 -[7:1] time: 0.248331 bw: 33.5097 -[7:2] time: 0.257127 bw: 32.3634 -[7:3] time: 0.258463 bw: 32.1961 -[7:4] time: 0.260985 bw: 31.8849 -[8:0] time: 0.244194 bw: 34.0774 -[8:1] time: 0.254596 bw: 32.6851 -[8:2] time: 0.257055 bw: 32.3724 -[8:3] time: 0.291329 bw: 28.5639 -[8:4] time: 0.294063 bw: 28.2983 -[9:0] time: 0.250627 bw: 33.2027 -[9:1] time: 0.259761 bw: 32.0353 -[9:2] time: 0.262148 bw: 31.7435 -[9:3] time: 0.264955 bw: 31.4072 -[9:4] time: 0.263812 bw: 31.5433 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.23278 -time0: 0.211356 -bw: 37.4191 -bw0: 41.2801 +time: 0.482728 +time0: 15.3559 +bw: 17.2386 +bw0: 0.541911 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out index 759750f..83d29ee 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00032_N_0032_TPN_1.out @@ -1,407 +1,269 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001168 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001212 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001204 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001150 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001180 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001202 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001200 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001226 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001224 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001208 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001162 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001210 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001230 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001178 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001236 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001184 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001160 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001198 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001170 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001194 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001242 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001164 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001148,001150,001152,001160,001162,001164,001168,001170,001178,001180,001184,001186,001194,001198,001200,001202,001204,001208,001210,001212,001216,001224,001226,001228,001230,001234,001236,001238,001240,001242,001244,001246] +Hostname: nid001148 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001152 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001244 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001186 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001234 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001238 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001246 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001240 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001228 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001216 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.633378 -[0:0] time: 0.494195 bw: 16.8385 -[0:1] time: 0.460126 bw: 18.0853 -[0:2] time: 0.461276 bw: 18.0402 -[0:3] time: 0.460579 bw: 18.0675 -[0:4] time: 0.46246 bw: 17.994 -[1:0] time: 0.496649 bw: 16.7553 -[1:1] time: 0.461198 bw: 18.0432 -[1:2] time: 0.461104 bw: 18.0469 -[1:3] time: 0.462001 bw: 18.0119 -[1:4] time: 0.46027 bw: 18.0796 -[2:0] time: 0.498402 bw: 16.6964 -[2:1] time: 0.460073 bw: 18.0874 -[2:2] time: 0.460819 bw: 18.0581 -[2:3] time: 0.460536 bw: 18.0692 -[2:4] time: 0.460496 bw: 18.0707 -[3:0] time: 0.496646 bw: 16.7554 -[3:1] time: 0.460274 bw: 18.0794 -[3:2] time: 0.46012 bw: 18.0855 -[3:3] time: 0.45978 bw: 18.0989 -[3:4] time: 0.459854 bw: 18.0959 -[4:0] time: 0.495814 bw: 16.7835 -[4:1] time: 0.460021 bw: 18.0894 -[4:2] time: 0.459773 bw: 18.0992 -[4:3] time: 0.459722 bw: 18.1011 -[4:4] time: 0.459713 bw: 18.1015 -[5:0] time: 0.496767 bw: 16.7513 -[5:1] time: 0.459741 bw: 18.1004 -[5:2] time: 0.459751 bw: 18.1 -[5:3] time: 0.459853 bw: 18.096 -[5:4] time: 0.459769 bw: 18.0993 -[6:0] time: 0.50076 bw: 16.6177 -[6:1] time: 0.46046 bw: 18.0722 -[6:2] time: 0.460443 bw: 18.0728 -[6:3] time: 0.460526 bw: 18.0696 -[6:4] time: 0.460566 bw: 18.068 -[7:0] time: 0.500478 bw: 16.6271 -[7:1] time: 0.460434 bw: 18.0732 -[7:2] time: 0.460552 bw: 18.0685 -[7:3] time: 0.460665 bw: 18.0641 -[7:4] time: 0.460517 bw: 18.0699 -[8:0] time: 0.500088 bw: 16.6401 -[8:1] time: 0.460416 bw: 18.0739 -[8:2] time: 0.460721 bw: 18.0619 -[8:3] time: 0.460516 bw: 18.07 -[8:4] time: 0.460538 bw: 18.0691 -[9:0] time: 0.500873 bw: 16.614 -[9:1] time: 0.460452 bw: 18.0725 -[9:2] time: 0.46073 bw: 18.0616 -[9:3] time: 0.460643 bw: 18.065 -[9:4] time: 0.460624 bw: 18.0657 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 0.58153 +[0:0] time: 0.499963 bw: 16.6442 +[0:1] time: 0.462618 bw: 17.9878 +[0:2] time: 0.462398 bw: 17.9964 +[0:3] time: 0.463443 bw: 17.9558 +[0:4] time: 0.464634 bw: 17.9098 +[1:0] time: 0.503132 bw: 16.5394 +[1:1] time: 0.463948 bw: 17.9363 +[1:2] time: 0.46478 bw: 17.9042 +[1:3] time: 0.46367 bw: 17.947 +[1:4] time: 0.46429 bw: 17.9231 +[2:0] time: 0.501071 bw: 16.6074 +[2:1] time: 0.463976 bw: 17.9352 +[2:2] time: 0.463095 bw: 17.9693 +[2:3] time: 0.462912 bw: 17.9764 +[2:4] time: 0.463648 bw: 17.9479 +[3:0] time: 0.502144 bw: 16.5719 +[3:1] time: 0.462679 bw: 17.9855 +[3:2] time: 0.462757 bw: 17.9825 +[3:3] time: 0.462681 bw: 17.9854 +[3:4] time: 0.462708 bw: 17.9844 +[4:0] time: 0.500845 bw: 16.6149 +[4:1] time: 0.462624 bw: 17.9876 +[4:2] time: 0.462749 bw: 17.9828 +[4:3] time: 0.462563 bw: 17.99 +[4:4] time: 0.462508 bw: 17.9921 +[5:0] time: 0.502213 bw: 16.5697 +[5:1] time: 0.462459 bw: 17.994 +[5:2] time: 0.462676 bw: 17.9856 +[5:3] time: 0.462494 bw: 17.9927 +[5:4] time: 0.462506 bw: 17.9922 +[6:0] time: 0.502187 bw: 16.5705 +[6:1] time: 0.462509 bw: 17.9921 +[6:2] time: 0.462469 bw: 17.9937 +[6:3] time: 0.46267 bw: 17.9858 +[6:4] time: 0.462674 bw: 17.9857 +[7:0] time: 0.502464 bw: 16.5614 +[7:1] time: 0.462558 bw: 17.9902 +[7:2] time: 0.462794 bw: 17.981 +[7:3] time: 0.462225 bw: 18.0031 +[7:4] time: 0.462655 bw: 17.9864 +[8:0] time: 0.502819 bw: 16.5497 +[8:1] time: 0.462839 bw: 17.9793 +[8:2] time: 0.462386 bw: 17.9969 +[8:3] time: 0.463002 bw: 17.9729 +[8:4] time: 0.46307 bw: 17.9703 +[9:0] time: 0.502523 bw: 16.5594 +[9:1] time: 0.462744 bw: 17.983 +[9:2] time: 0.462458 bw: 17.9941 +[9:3] time: 0.462465 bw: 17.9938 +[9:4] time: 0.462754 bw: 17.9826 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.460453 -time0: 0.498067 -bw: 18.0725 -bw0: 16.7079 +time: 0.462927 +time0: 0.501936 +bw: 17.9759 +bw0: 16.5789 ======================================= -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001230 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001180 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001194 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001234 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001150 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001160 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001202 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001168 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001186 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001208 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001236 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001216 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001210 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001162 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001246 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001204 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001242 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001164 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001226 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001178 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001228 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001184 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001240 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001170 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001198 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001152 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001148,001150,001152,001160,001162,001164,001168,001170,001178,001180,001184,001186,001194,001198,001200,001202,001204,001208,001210,001212,001216,001224,001226,001228,001230,001234,001236,001238,001240,001242,001244,001246] +Hostname: nid001148 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001244 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001212 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001200 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001224 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001238 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 1.61589 -[0:0] time: 1.59151 bw: 5.22868 -[0:1] time: 1.5849 bw: 5.25048 -[0:2] time: 1.58426 bw: 5.25262 -[0:3] time: 1.58139 bw: 5.26215 -[0:4] time: 1.57734 bw: 5.27564 -[1:0] time: 1.62241 bw: 5.12911 -[1:1] time: 1.57625 bw: 5.2793 -[1:2] time: 1.57924 bw: 5.26932 -[1:3] time: 1.5798 bw: 5.26744 -[1:4] time: 1.58211 bw: 5.25974 -[2:0] time: 1.61847 bw: 5.1416 -[2:1] time: 1.58337 bw: 5.25557 -[2:2] time: 1.57872 bw: 5.27104 -[2:3] time: 1.57994 bw: 5.26698 -[2:4] time: 1.57869 bw: 5.27115 -[3:0] time: 1.62336 bw: 5.12611 -[3:1] time: 1.57834 bw: 5.27231 -[3:2] time: 1.58255 bw: 5.2583 -[3:3] time: 1.57513 bw: 5.28305 -[3:4] time: 1.58089 bw: 5.2638 -[4:0] time: 1.62349 bw: 5.12568 -[4:1] time: 1.57971 bw: 5.26775 -[4:2] time: 1.57898 bw: 5.27016 -[4:3] time: 1.58272 bw: 5.25774 -[4:4] time: 1.5777 bw: 5.27444 -[5:0] time: 1.63172 bw: 5.09982 -[5:1] time: 1.57912 bw: 5.26969 -[5:2] time: 1.58129 bw: 5.26247 -[5:3] time: 1.58291 bw: 5.2571 -[5:4] time: 1.58223 bw: 5.25935 -[6:0] time: 1.62536 bw: 5.1198 -[6:1] time: 1.58651 bw: 5.24515 -[6:2] time: 1.57969 bw: 5.2678 -[6:3] time: 1.5824 bw: 5.25879 -[6:4] time: 1.58236 bw: 5.25892 -[7:0] time: 1.63464 bw: 5.09071 -[7:1] time: 1.58165 bw: 5.26129 -[7:2] time: 1.58741 bw: 5.24219 -[7:3] time: 1.57853 bw: 5.27167 -[7:4] time: 1.58302 bw: 5.25671 -[8:0] time: 1.63303 bw: 5.09575 -[8:1] time: 1.58233 bw: 5.25901 -[8:2] time: 1.58062 bw: 5.26471 -[8:3] time: 1.58599 bw: 5.24687 -[8:4] time: 1.57939 bw: 5.26881 -[9:0] time: 1.63824 bw: 5.07954 -[9:1] time: 1.58286 bw: 5.25727 -[9:2] time: 1.58338 bw: 5.25553 -[9:3] time: 1.58319 bw: 5.25615 -[9:4] time: 1.58604 bw: 5.24672 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.62887 +[0:0] time: 1.60726 bw: 5.17745 +[0:1] time: 1.60874 bw: 5.17269 +[0:2] time: 1.60214 bw: 5.19399 +[0:3] time: 1.60215 bw: 5.19396 +[0:4] time: 1.5995 bw: 5.20256 +[1:0] time: 1.64382 bw: 5.0623 +[1:1] time: 1.60188 bw: 5.19484 +[1:2] time: 1.60165 bw: 5.19556 +[1:3] time: 1.60158 bw: 5.19582 +[1:4] time: 1.60398 bw: 5.18804 +[2:0] time: 1.63957 bw: 5.0754 +[2:1] time: 1.60385 bw: 5.18845 +[2:2] time: 1.60312 bw: 5.19082 +[2:3] time: 1.60399 bw: 5.18801 +[2:4] time: 1.60241 bw: 5.19312 +[3:0] time: 1.65108 bw: 5.04002 +[3:1] time: 1.60418 bw: 5.18739 +[3:2] time: 1.60668 bw: 5.17931 +[3:3] time: 1.60335 bw: 5.19009 +[3:4] time: 1.60634 bw: 5.18042 +[4:0] time: 1.65149 bw: 5.03879 +[4:1] time: 1.60741 bw: 5.17697 +[4:2] time: 1.60559 bw: 5.18282 +[4:3] time: 1.60755 bw: 5.17652 +[4:4] time: 1.60587 bw: 5.18194 +[5:0] time: 1.66278 bw: 5.00456 +[5:1] time: 1.60675 bw: 5.17909 +[5:2] time: 1.60822 bw: 5.17435 +[5:3] time: 1.60761 bw: 5.17633 +[5:4] time: 1.60962 bw: 5.16986 +[6:0] time: 1.65353 bw: 5.03258 +[6:1] time: 1.60977 bw: 5.16938 +[6:2] time: 1.60914 bw: 5.1714 +[6:3] time: 1.60987 bw: 5.16905 +[6:4] time: 1.60534 bw: 5.18363 +[7:0] time: 1.66331 bw: 5.00296 +[7:1] time: 1.60931 bw: 5.17086 +[7:2] time: 1.61072 bw: 5.16631 +[7:3] time: 1.60663 bw: 5.17948 +[7:4] time: 1.6088 bw: 5.17248 +[8:0] time: 1.65757 bw: 5.02029 +[8:1] time: 1.6092 bw: 5.1712 +[8:2] time: 1.60459 bw: 5.18606 +[8:3] time: 1.60667 bw: 5.17934 +[8:4] time: 1.60501 bw: 5.18472 +[9:0] time: 1.66225 bw: 5.00617 +[9:1] time: 1.60517 bw: 5.18417 +[9:2] time: 1.60722 bw: 5.17756 +[9:3] time: 1.60508 bw: 5.18447 +[9:4] time: 1.6064 bw: 5.1802 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 1.58132 -time0: 1.62422 -bw: 5.26238 -bw0: 5.12368 -======================================= - -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.225496 -[0:0] time: 0.136698 bw: 60.8751 -[0:1] time: 0.134958 bw: 61.6599 -[0:2] time: 0.128691 bw: 64.6626 -[0:3] time: 0.128683 bw: 64.6667 -[0:4] time: 0.128685 bw: 64.6655 -[1:0] time: 0.129058 bw: 64.4789 -[1:1] time: 0.128637 bw: 64.69 -[1:2] time: 0.128651 bw: 64.6828 -[1:3] time: 0.128656 bw: 64.6803 -[1:4] time: 0.128647 bw: 64.6847 -[2:0] time: 0.12905 bw: 64.4827 -[2:1] time: 0.128671 bw: 64.6726 -[2:2] time: 0.128657 bw: 64.68 -[2:3] time: 0.128668 bw: 64.6741 -[2:4] time: 0.128664 bw: 64.6761 -[3:0] time: 0.129074 bw: 64.4707 -[3:1] time: 0.128654 bw: 64.681 -[3:2] time: 0.128672 bw: 64.6723 -[3:3] time: 0.128647 bw: 64.6846 -[3:4] time: 0.128664 bw: 64.6761 -[4:0] time: 0.129034 bw: 64.4908 -[4:1] time: 0.128678 bw: 64.6691 -[4:2] time: 0.128691 bw: 64.6626 -[4:3] time: 0.128672 bw: 64.6724 -[4:4] time: 0.128663 bw: 64.6766 -[5:0] time: 0.129052 bw: 64.482 -[5:1] time: 0.128657 bw: 64.6797 -[5:2] time: 0.128769 bw: 64.6234 -[5:3] time: 0.128678 bw: 64.6694 -[5:4] time: 0.128679 bw: 64.6689 -[6:0] time: 0.128968 bw: 64.5238 -[6:1] time: 0.128831 bw: 64.5924 -[6:2] time: 0.128688 bw: 64.664 -[6:3] time: 0.128681 bw: 64.6677 -[6:4] time: 0.128666 bw: 64.6752 -[7:0] time: 0.129101 bw: 64.4575 -[7:1] time: 0.128662 bw: 64.6772 -[7:2] time: 0.128643 bw: 64.6868 -[7:3] time: 0.128637 bw: 64.69 -[7:4] time: 0.128644 bw: 64.686 -[8:0] time: 0.129218 bw: 64.3989 -[8:1] time: 0.128695 bw: 64.6608 -[8:2] time: 0.128658 bw: 64.6791 -[8:3] time: 0.128656 bw: 64.68 -[8:4] time: 0.128653 bw: 64.6815 -[9:0] time: 0.129183 bw: 64.4164 -[9:1] time: 0.12868 bw: 64.668 -[9:2] time: 0.128645 bw: 64.6857 -[9:3] time: 0.128871 bw: 64.5721 -[9:4] time: 0.128674 bw: 64.6714 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.128834 -time0: 0.129843 -bw: 64.5942 -bw0: 64.1077 -======================================= - -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.203091 -[0:0] time: 0.138902 bw: 59.9091 -[0:1] time: 0.138449 bw: 60.1052 -[0:2] time: 0.136795 bw: 60.832 -[0:3] time: 0.135388 bw: 61.464 -[0:4] time: 0.135144 bw: 61.5749 -[1:0] time: 0.137294 bw: 60.6107 -[1:1] time: 0.135363 bw: 61.4753 -[1:2] time: 0.135182 bw: 61.5578 -[1:3] time: 0.135137 bw: 61.5784 -[1:4] time: 0.135177 bw: 61.5599 -[2:0] time: 0.135791 bw: 61.2815 -[2:1] time: 0.132879 bw: 62.6244 -[2:2] time: 0.132895 bw: 62.6172 -[2:3] time: 0.132892 bw: 62.6186 -[2:4] time: 0.132904 bw: 62.6128 -[3:0] time: 0.132944 bw: 62.5942 -[3:1] time: 0.132761 bw: 62.6804 -[3:2] time: 0.132782 bw: 62.6705 -[3:3] time: 0.132793 bw: 62.6651 -[3:4] time: 0.132774 bw: 62.674 -[4:0] time: 0.132916 bw: 62.6073 -[4:1] time: 0.132888 bw: 62.6205 -[4:2] time: 0.132924 bw: 62.6036 -[4:3] time: 0.132898 bw: 62.6155 -[4:4] time: 0.132919 bw: 62.6056 -[5:0] time: 0.13291 bw: 62.6102 -[5:1] time: 0.132767 bw: 62.6775 -[5:2] time: 0.132756 bw: 62.6826 -[5:3] time: 0.132929 bw: 62.6012 -[5:4] time: 0.132815 bw: 62.655 -[6:0] time: 0.133057 bw: 62.541 -[6:1] time: 0.132893 bw: 62.6179 -[6:2] time: 0.132943 bw: 62.5947 -[6:3] time: 0.132916 bw: 62.6073 -[6:4] time: 0.132914 bw: 62.6082 -[7:0] time: 0.133092 bw: 62.5245 -[7:1] time: 0.132775 bw: 62.6737 -[7:2] time: 0.132753 bw: 62.6842 -[7:3] time: 0.132761 bw: 62.6801 -[7:4] time: 0.132743 bw: 62.6886 -[8:0] time: 0.132998 bw: 62.5685 -[8:1] time: 0.132878 bw: 62.6249 -[8:2] time: 0.132914 bw: 62.6083 -[8:3] time: 0.132899 bw: 62.6154 -[8:4] time: 0.132902 bw: 62.6139 -[9:0] time: 0.132978 bw: 62.5778 -[9:1] time: 0.132747 bw: 62.6869 -[9:2] time: 0.13273 bw: 62.6951 -[9:3] time: 0.132761 bw: 62.6803 -[9:4] time: 0.132771 bw: 62.6755 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.133438 -time0: 0.134288 -bw: 62.3682 -bw0: 61.9825 +time: 1.60583 +time0: 1.64927 +bw: 5.18208 +bw0: 5.04605 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out index c2d9994..bf97ad7 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0016_TPN_4.out @@ -1,535 +1,397 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] -Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005152 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005160 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005192 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005160 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005174 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005180 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005192 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005160 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005202 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005160 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005192 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005192 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005180 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005152 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005162 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005152 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005152 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005180 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005154 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005174 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005174 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005154 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005174 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005154 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005202 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005202 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005202 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005162 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005162 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005154 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005162 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005176 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005204 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005168 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005150 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005176 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005176 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005176 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005168 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005168 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005168 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005150 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005150 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Slurm Job Hostlist: nid[005150,005152,005154,005160,005162,005168,005170,005174,005176,005180,005182,005192,005194,005198,005202,005204] +Hostname: nid005150 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005204 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005204 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005204 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005198 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005198 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005194 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005198 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005170 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005198 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005194 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005194 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005182 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005194 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005170 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005170 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005170 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005182 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005182 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005182 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.577848 -[0:0] time: 0.296514 bw: 28.5171 -[0:1] time: 0.260698 bw: 32.435 -[0:2] time: 0.26355 bw: 32.0839 -[0:3] time: 0.263462 bw: 32.0946 -[0:4] time: 0.262884 bw: 32.1652 -[1:0] time: 0.301235 bw: 28.0702 -[1:1] time: 0.265641 bw: 31.8314 -[1:2] time: 0.266208 bw: 31.7636 -[1:3] time: 0.260905 bw: 32.4092 -[1:4] time: 0.264152 bw: 32.0108 -[2:0] time: 0.325059 bw: 26.0128 -[2:1] time: 0.261902 bw: 32.2858 -[2:2] time: 0.263345 bw: 32.1089 -[2:3] time: 0.260814 bw: 32.4204 -[2:4] time: 0.260924 bw: 32.4068 -[3:0] time: 0.320153 bw: 26.4115 -[3:1] time: 0.260319 bw: 32.4821 -[3:2] time: 0.260399 bw: 32.4722 -[3:3] time: 0.26014 bw: 32.5045 -[3:4] time: 0.260093 bw: 32.5104 -[4:0] time: 0.326936 bw: 25.8635 -[4:1] time: 0.260172 bw: 32.5005 -[4:2] time: 0.260607 bw: 32.4462 -[4:3] time: 0.260016 bw: 32.52 -[4:4] time: 0.260194 bw: 32.4977 -[5:0] time: 0.322766 bw: 26.1977 -[5:1] time: 0.26018 bw: 32.4995 -[5:2] time: 0.260792 bw: 32.4233 -[5:3] time: 0.261101 bw: 32.3848 -[5:4] time: 0.261049 bw: 32.3914 -[6:0] time: 0.325079 bw: 26.0112 -[6:1] time: 0.260946 bw: 32.4041 -[6:2] time: 0.260621 bw: 32.4445 -[6:3] time: 0.260808 bw: 32.4212 -[6:4] time: 0.26134 bw: 32.3552 -[7:0] time: 0.326505 bw: 25.8977 -[7:1] time: 0.260917 bw: 32.4077 -[7:2] time: 0.261139 bw: 32.3801 -[7:3] time: 0.261779 bw: 32.301 -[7:4] time: 0.260806 bw: 32.4215 -[8:0] time: 0.323372 bw: 26.1486 -[8:1] time: 0.261384 bw: 32.3497 -[8:2] time: 0.260498 bw: 32.4598 -[8:3] time: 0.260793 bw: 32.4231 -[8:4] time: 0.261004 bw: 32.3969 -[9:0] time: 0.325747 bw: 25.9579 -[9:1] time: 0.261037 bw: 32.3929 -[9:2] time: 0.261697 bw: 32.3112 -[9:3] time: 0.260858 bw: 32.4151 -[9:4] time: 0.260898 bw: 32.4101 +[-1] time: 0.559325 +[0:0] time: 0.295616 bw: 28.6037 +[0:1] time: 0.261176 bw: 32.3756 +[0:2] time: 0.261613 bw: 32.3214 +[0:3] time: 0.264304 bw: 31.9923 +[0:4] time: 0.263692 bw: 32.0666 +[1:0] time: 0.309487 bw: 27.3217 +[1:1] time: 0.269622 bw: 31.3614 +[1:2] time: 0.265702 bw: 31.8241 +[1:3] time: 0.266272 bw: 31.756 +[1:4] time: 0.2641 bw: 32.0171 +[2:0] time: 0.323196 bw: 26.1628 +[2:1] time: 0.264302 bw: 31.9926 +[2:2] time: 0.262747 bw: 32.182 +[2:3] time: 0.263108 bw: 32.1378 +[2:4] time: 0.264344 bw: 31.9876 +[3:0] time: 0.316094 bw: 26.7507 +[3:1] time: 0.262388 bw: 32.226 +[3:2] time: 0.262253 bw: 32.2426 +[3:3] time: 0.262388 bw: 32.226 +[3:4] time: 0.262282 bw: 32.2391 +[4:0] time: 0.313929 bw: 26.9352 +[4:1] time: 0.262568 bw: 32.2039 +[4:2] time: 0.262599 bw: 32.2001 +[4:3] time: 0.262561 bw: 32.2048 +[4:4] time: 0.262513 bw: 32.2106 +[5:0] time: 0.31432 bw: 26.9016 +[5:1] time: 0.262439 bw: 32.2197 +[5:2] time: 0.262375 bw: 32.2276 +[5:3] time: 0.262428 bw: 32.2211 +[5:4] time: 0.262222 bw: 32.2464 +[6:0] time: 0.320892 bw: 26.3507 +[6:1] time: 0.262035 bw: 32.2695 +[6:2] time: 0.26219 bw: 32.2503 +[6:3] time: 0.261967 bw: 32.2777 +[6:4] time: 0.262248 bw: 32.2433 +[7:0] time: 0.319216 bw: 26.489 +[7:1] time: 0.262126 bw: 32.2582 +[7:2] time: 0.26222 bw: 32.2466 +[7:3] time: 0.262444 bw: 32.2191 +[7:4] time: 0.26229 bw: 32.238 +[8:0] time: 0.320473 bw: 26.3851 +[8:1] time: 0.262089 bw: 32.2628 +[8:2] time: 0.262193 bw: 32.25 +[8:3] time: 0.261854 bw: 32.2917 +[8:4] time: 0.262223 bw: 32.2463 +[9:0] time: 0.319192 bw: 26.491 +[9:1] time: 0.262173 bw: 32.2525 +[9:2] time: 0.262172 bw: 32.2525 +[9:3] time: 0.262242 bw: 32.244 +[9:4] time: 0.261879 bw: 32.2886 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.261402 -time0: 0.319337 -bw: 32.3486 -bw0: 26.5088 +time: 0.262859 +time0: 0.315241 +bw: 32.1693 +bw0: 26.8392 ======================================= -Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] -Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005152 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005160 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005152 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005152 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005152 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005162 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005162 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005160 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005162 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005162 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005160 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005160 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005154 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005154 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005154 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005154 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005150 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005150 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[005150,005152,005154,005160,005162,005168,005170,005174,005176,005180,005182,005192,005194,005198,005202,005204] +Hostname: nid005150 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005150 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005180 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005182 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005174 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005182 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005180 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005180 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005182 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005174 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005182 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005174 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005176 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005174 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005168 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005194 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005176 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005176 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005176 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005194 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005168 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005168 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005194 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005194 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005168 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005204 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005170 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005170 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005170 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005170 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005204 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005204 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005204 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005192 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005202 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005202 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005202 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005202 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005192 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005198 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005192 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005192 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005198 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005198 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005198 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.4604 -[0:0] time: 15.3975 bw: 0.549161 -[0:1] time: 0.533471 bw: 15.8504 -[0:2] time: 0.552953 bw: 15.2919 -[0:3] time: 0.544049 bw: 15.5422 -[0:4] time: 0.534672 bw: 15.8148 -[1:0] time: 15.4652 bw: 0.546756 -[1:1] time: 0.532915 bw: 15.8669 -[1:2] time: 0.532406 bw: 15.8821 -[1:3] time: 0.533373 bw: 15.8533 -[1:4] time: 0.533062 bw: 15.8625 -[2:0] time: 15.4293 bw: 0.54803 -[2:1] time: 0.532551 bw: 15.8777 -[2:2] time: 0.533304 bw: 15.8553 -[2:3] time: 0.533145 bw: 15.8601 -[2:4] time: 0.533563 bw: 15.8476 -[3:0] time: 15.4175 bw: 0.548449 -[3:1] time: 0.533184 bw: 15.8589 -[3:2] time: 0.532332 bw: 15.8843 -[3:3] time: 0.533368 bw: 15.8534 -[3:4] time: 0.532978 bw: 15.865 -[4:0] time: 15.397 bw: 0.549179 -[4:1] time: 0.532439 bw: 15.8811 -[4:2] time: 0.532761 bw: 15.8715 -[4:3] time: 0.532601 bw: 15.8763 -[4:4] time: 0.533335 bw: 15.8544 -[5:0] time: 15.4129 bw: 0.548612 -[5:1] time: 0.532273 bw: 15.8861 -[5:2] time: 0.5323 bw: 15.8852 -[5:3] time: 0.533085 bw: 15.8618 -[5:4] time: 0.533529 bw: 15.8487 -[6:0] time: 15.4066 bw: 0.548837 -[6:1] time: 0.533409 bw: 15.8522 -[6:2] time: 0.533869 bw: 15.8386 -[6:3] time: 0.533509 bw: 15.8492 -[6:4] time: 0.533393 bw: 15.8527 -[7:0] time: 15.4442 bw: 0.547502 -[7:1] time: 0.533528 bw: 15.8487 -[7:2] time: 0.532995 bw: 15.8645 -[7:3] time: 0.534086 bw: 15.8321 -[7:4] time: 0.533587 bw: 15.8469 -[8:0] time: 15.4427 bw: 0.547554 -[8:1] time: 0.533102 bw: 15.8613 -[8:2] time: 0.533552 bw: 15.848 -[8:3] time: 0.533553 bw: 15.8479 -[8:4] time: 0.53382 bw: 15.84 -[9:0] time: 15.4224 bw: 0.548275 -[9:1] time: 0.533314 bw: 15.855 -[9:2] time: 0.533487 bw: 15.8499 -[9:3] time: 0.534329 bw: 15.8249 -[9:4] time: 0.533597 bw: 15.8466 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 460.971 +[0:0] time: 15.4962 bw: 0.545664 +[0:1] time: 0.537552 bw: 15.7301 +[0:2] time: 0.549892 bw: 15.3771 +[0:3] time: 0.548492 bw: 15.4163 +[0:4] time: 0.53898 bw: 15.6884 +[1:0] time: 15.4549 bw: 0.547122 +[1:1] time: 0.536587 bw: 15.7583 +[1:2] time: 0.536646 bw: 15.7566 +[1:3] time: 0.53693 bw: 15.7483 +[1:4] time: 0.536442 bw: 15.7626 +[2:0] time: 15.4697 bw: 0.5466 +[2:1] time: 0.535165 bw: 15.8002 +[2:2] time: 0.536489 bw: 15.7612 +[2:3] time: 0.536848 bw: 15.7507 +[2:4] time: 0.537918 bw: 15.7193 +[3:0] time: 15.4484 bw: 0.547352 +[3:1] time: 0.536223 bw: 15.769 +[3:2] time: 0.536082 bw: 15.7732 +[3:3] time: 0.536926 bw: 15.7484 +[3:4] time: 0.536971 bw: 15.7471 +[4:0] time: 15.4972 bw: 0.545629 +[4:1] time: 0.536324 bw: 15.7661 +[4:2] time: 0.535303 bw: 15.7961 +[4:3] time: 0.53577 bw: 15.7824 +[4:4] time: 0.536389 bw: 15.7642 +[5:0] time: 15.5098 bw: 0.545184 +[5:1] time: 0.537027 bw: 15.7454 +[5:2] time: 0.536109 bw: 15.7724 +[5:3] time: 0.53633 bw: 15.7659 +[5:4] time: 0.536349 bw: 15.7653 +[6:0] time: 15.4639 bw: 0.546803 +[6:1] time: 0.536126 bw: 15.7719 +[6:2] time: 0.536396 bw: 15.7639 +[6:3] time: 0.537494 bw: 15.7317 +[6:4] time: 0.537258 bw: 15.7386 +[7:0] time: 15.4413 bw: 0.547603 +[7:1] time: 0.536696 bw: 15.7551 +[7:2] time: 0.535358 bw: 15.7945 +[7:3] time: 0.537195 bw: 15.7405 +[7:4] time: 0.537371 bw: 15.7353 +[8:0] time: 15.4848 bw: 0.546066 +[8:1] time: 0.537091 bw: 15.7436 +[8:2] time: 0.536944 bw: 15.7479 +[8:3] time: 0.535999 bw: 15.7756 +[8:4] time: 0.536042 bw: 15.7744 +[9:0] time: 15.4426 bw: 0.547557 +[9:1] time: 0.537069 bw: 15.7442 +[9:2] time: 0.536844 bw: 15.7508 +[9:3] time: 0.537468 bw: 15.7325 +[9:4] time: 0.536786 bw: 15.7525 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.534019 -time0: 15.4235 -bw: 15.8348 -bw0: 0.548236 -======================================= - -Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] -Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.371044 -[0:0] time: 0.264918 bw: 31.9183 -[0:1] time: 0.258789 bw: 32.6741 -[0:2] time: 0.258964 bw: 32.6521 -[0:3] time: 0.2588 bw: 32.6728 -[0:4] time: 0.258787 bw: 32.6745 -[1:0] time: 0.261152 bw: 32.3786 -[1:1] time: 0.258654 bw: 32.6913 -[1:2] time: 0.258672 bw: 32.689 -[1:3] time: 0.258702 bw: 32.6852 -[1:4] time: 0.258711 bw: 32.684 -[2:0] time: 0.259632 bw: 32.5681 -[2:1] time: 0.25873 bw: 32.6817 -[2:2] time: 0.258757 bw: 32.6782 -[2:3] time: 0.258763 bw: 32.6774 -[2:4] time: 0.258781 bw: 32.6751 -[3:0] time: 0.259519 bw: 32.5822 -[3:1] time: 0.258671 bw: 32.6891 -[3:2] time: 0.258675 bw: 32.6886 -[3:3] time: 0.258644 bw: 32.6925 -[3:4] time: 0.258669 bw: 32.6893 -[4:0] time: 0.260168 bw: 32.501 -[4:1] time: 0.258733 bw: 32.6812 -[4:2] time: 0.258725 bw: 32.6823 -[4:3] time: 0.258742 bw: 32.6801 -[4:4] time: 0.25876 bw: 32.6779 -[5:0] time: 0.259661 bw: 32.5644 -[5:1] time: 0.258665 bw: 32.6898 -[5:2] time: 0.258638 bw: 32.6933 -[5:3] time: 0.260174 bw: 32.5003 -[5:4] time: 0.260529 bw: 32.456 -[6:0] time: 0.259625 bw: 32.569 -[6:1] time: 0.25875 bw: 32.6792 -[6:2] time: 0.258739 bw: 32.6804 -[6:3] time: 0.258753 bw: 32.6788 -[6:4] time: 0.258742 bw: 32.6802 -[7:0] time: 0.25956 bw: 32.5771 -[7:1] time: 0.258662 bw: 32.6902 -[7:2] time: 0.258621 bw: 32.6954 -[7:3] time: 0.258645 bw: 32.6923 -[7:4] time: 0.258651 bw: 32.6917 -[8:0] time: 0.26263 bw: 32.1963 -[8:1] time: 0.258792 bw: 32.6737 -[8:2] time: 0.258731 bw: 32.6814 -[8:3] time: 0.258724 bw: 32.6823 -[8:4] time: 0.258772 bw: 32.6763 -[9:0] time: 0.259719 bw: 32.5572 -[9:1] time: 0.25863 bw: 32.6943 -[9:2] time: 0.258637 bw: 32.6933 -[9:3] time: 0.258607 bw: 32.6971 -[9:4] time: 0.258654 bw: 32.6912 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.258796 -time0: 0.260658 -bw: 32.6733 -bw0: 32.4412 -======================================= - -Hostname: nid004470 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004450 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004438 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470] -Hostname: nid004412 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004460 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004456 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004470 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004464 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004464 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004440 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.404822 -[0:0] time: 0.267391 bw: 31.623 -[0:1] time: 0.267059 bw: 31.6623 -[0:2] time: 0.26707 bw: 31.661 -[0:3] time: 0.267079 bw: 31.66 -[0:4] time: 0.267049 bw: 31.6635 -[1:0] time: 0.267457 bw: 31.6152 -[1:1] time: 0.267035 bw: 31.6652 -[1:2] time: 0.267037 bw: 31.665 -[1:3] time: 0.267048 bw: 31.6637 -[1:4] time: 0.267045 bw: 31.6641 -[2:0] time: 0.267568 bw: 31.6022 -[2:1] time: 0.267068 bw: 31.6613 -[2:2] time: 0.267054 bw: 31.6629 -[2:3] time: 0.267046 bw: 31.6638 -[2:4] time: 0.267073 bw: 31.6607 -[3:0] time: 0.26771 bw: 31.5854 -[3:1] time: 0.267008 bw: 31.6684 -[3:2] time: 0.26706 bw: 31.6622 -[3:3] time: 0.267027 bw: 31.6662 -[3:4] time: 0.267058 bw: 31.6624 -[4:0] time: 0.268078 bw: 31.542 -[4:1] time: 0.267089 bw: 31.6587 -[4:2] time: 0.267068 bw: 31.6613 -[4:3] time: 0.267066 bw: 31.6616 -[4:4] time: 0.267034 bw: 31.6653 -[5:0] time: 0.267991 bw: 31.5523 -[5:1] time: 0.267019 bw: 31.6671 -[5:2] time: 0.267046 bw: 31.6639 -[5:3] time: 0.267005 bw: 31.6687 -[5:4] time: 0.267036 bw: 31.6651 -[6:0] time: 0.268042 bw: 31.5462 -[6:1] time: 0.267038 bw: 31.6648 -[6:2] time: 0.267056 bw: 31.6627 -[6:3] time: 0.267031 bw: 31.6656 -[6:4] time: 0.271913 bw: 31.0971 -[7:0] time: 0.268058 bw: 31.5443 -[7:1] time: 0.267011 bw: 31.6681 -[7:2] time: 0.267023 bw: 31.6666 -[7:3] time: 0.267008 bw: 31.6684 -[7:4] time: 0.267011 bw: 31.6681 -[8:0] time: 0.270079 bw: 31.3084 -[8:1] time: 0.267047 bw: 31.6638 -[8:2] time: 0.267059 bw: 31.6624 -[8:3] time: 0.267095 bw: 31.658 -[8:4] time: 0.267085 bw: 31.6592 -[9:0] time: 0.26802 bw: 31.5488 -[9:1] time: 0.267045 bw: 31.664 -[9:2] time: 0.267024 bw: 31.6666 -[9:3] time: 0.266986 bw: 31.671 -[9:4] time: 0.267004 bw: 31.6689 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.267165 -time0: 0.268039 -bw: 31.65 -bw0: 31.5468 +time: 0.537297 +time0: 15.4709 +bw: 15.7379 +bw0: 0.546558 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out index b20d5b7..9c0d297 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00064_N_0064_TPN_1.out @@ -1,535 +1,397 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] -Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002710 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002680 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002594 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002638 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002626 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002570 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002620 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002632 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002670 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002588 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002642 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002684 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002636 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002740 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002612 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002674 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002664 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002714 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002682 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002552,002554,002570,002588,002590,002592,002594,002600,002602,002606,002608,002610,002612,002614,002620,002622,002624,002626,002632,002634,002636,002638,002640,002642,002644,002646,002652,002654,002656,002658,002664,002666,002668,002670,002674,002676,002678,002680,002682,002684,002686,002688,002690,002696,002698,002700,002702,002704,002706,002708,002710,002712,002714,002716,002718,002720,002722,002728,002730,002732,002734,002736,002738,002740] +Hostname: nid002610 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002552 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002656 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002614 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002554 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002736 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002732 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002708 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002712 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002590 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002686 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002688 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002678 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002698 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002640 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002720 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002734 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002652 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002600 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002668 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002608 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002716 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002706 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002696 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002606 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002722 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002702 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002658 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002728 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002592 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002644 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002730 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002676 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002646 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002602 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002654 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.661234 -[0:0] time: 0.553014 bw: 15.2902 -[0:1] time: 0.512511 bw: 16.4986 -[0:2] time: 0.509409 bw: 16.5991 -[0:3] time: 0.510615 bw: 16.5599 -[0:4] time: 0.510136 bw: 16.5754 -[1:0] time: 0.543105 bw: 15.5692 -[1:1] time: 0.506487 bw: 16.6948 -[1:2] time: 0.505329 bw: 16.7331 -[1:3] time: 0.505525 bw: 16.7266 -[1:4] time: 0.505777 bw: 16.7183 -[2:0] time: 0.542354 bw: 15.5908 -[2:1] time: 0.505128 bw: 16.7397 -[2:2] time: 0.504478 bw: 16.7613 -[2:3] time: 0.504188 bw: 16.771 -[2:4] time: 0.503884 bw: 16.7811 -[3:0] time: 0.541636 bw: 15.6114 -[3:1] time: 0.504279 bw: 16.7679 -[3:2] time: 0.503556 bw: 16.792 -[3:3] time: 0.504248 bw: 16.769 -[3:4] time: 0.5042 bw: 16.7706 -[4:0] time: 0.544191 bw: 15.5381 -[4:1] time: 0.503954 bw: 16.7787 -[4:2] time: 0.503594 bw: 16.7907 -[4:3] time: 0.502879 bw: 16.8146 -[4:4] time: 0.503713 bw: 16.7868 -[5:0] time: 0.543364 bw: 15.5618 -[5:1] time: 0.504404 bw: 16.7638 -[5:2] time: 0.503679 bw: 16.7879 -[5:3] time: 0.504213 bw: 16.7701 -[5:4] time: 0.504002 bw: 16.7771 -[6:0] time: 0.543806 bw: 15.5491 -[6:1] time: 0.50485 bw: 16.749 -[6:2] time: 0.505183 bw: 16.7379 -[6:3] time: 0.504502 bw: 16.7605 -[6:4] time: 0.505087 bw: 16.7411 -[7:0] time: 0.545689 bw: 15.4955 -[7:1] time: 0.504497 bw: 16.7607 -[7:2] time: 0.504201 bw: 16.7705 -[7:3] time: 0.504589 bw: 16.7576 -[7:4] time: 0.508141 bw: 16.6405 -[8:0] time: 0.545215 bw: 15.509 -[8:1] time: 0.50496 bw: 16.7453 -[8:2] time: 0.504247 bw: 16.769 -[8:3] time: 0.504246 bw: 16.769 -[8:4] time: 0.504552 bw: 16.7589 -[9:0] time: 0.545114 bw: 15.5118 -[9:1] time: 0.504756 bw: 16.7521 -[9:2] time: 0.504522 bw: 16.7599 -[9:3] time: 0.505344 bw: 16.7326 -[9:4] time: 0.50697 bw: 16.6789 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 0.774768 +[0:0] time: 0.525096 bw: 16.1032 +[0:1] time: 0.490479 bw: 17.2397 +[0:2] time: 0.492567 bw: 17.1666 +[0:3] time: 0.495231 bw: 17.0743 +[0:4] time: 0.498344 bw: 16.9676 +[1:0] time: 0.533361 bw: 15.8536 +[1:1] time: 0.495781 bw: 17.0553 +[1:2] time: 0.493217 bw: 17.144 +[1:3] time: 0.491962 bw: 17.1878 +[1:4] time: 0.491943 bw: 17.1884 +[2:0] time: 0.527407 bw: 16.0326 +[2:1] time: 0.490666 bw: 17.2331 +[2:2] time: 0.490383 bw: 17.2431 +[2:3] time: 0.490933 bw: 17.2238 +[2:4] time: 0.489394 bw: 17.2779 +[3:0] time: 0.526276 bw: 16.0671 +[3:1] time: 0.487442 bw: 17.3471 +[3:2] time: 0.486489 bw: 17.3811 +[3:3] time: 0.485719 bw: 17.4087 +[3:4] time: 0.486096 bw: 17.3952 +[4:0] time: 0.522478 bw: 16.1839 +[4:1] time: 0.486344 bw: 17.3863 +[4:2] time: 0.485688 bw: 17.4098 +[4:3] time: 0.48574 bw: 17.4079 +[4:4] time: 0.485405 bw: 17.4199 +[5:0] time: 0.523338 bw: 16.1573 +[5:1] time: 0.485293 bw: 17.4239 +[5:2] time: 0.48609 bw: 17.3954 +[5:3] time: 0.485489 bw: 17.4169 +[5:4] time: 0.48628 bw: 17.3886 +[6:0] time: 0.522325 bw: 16.1886 +[6:1] time: 0.486415 bw: 17.3838 +[6:2] time: 0.485534 bw: 17.4153 +[6:3] time: 0.48716 bw: 17.3572 +[6:4] time: 0.48575 bw: 17.4075 +[7:0] time: 0.531676 bw: 15.9039 +[7:1] time: 0.486951 bw: 17.3646 +[7:2] time: 0.487017 bw: 17.3623 +[7:3] time: 0.485719 bw: 17.4087 +[7:4] time: 0.486066 bw: 17.3962 +[8:0] time: 0.521915 bw: 16.2013 +[8:1] time: 0.486439 bw: 17.3829 +[8:2] time: 0.485804 bw: 17.4056 +[8:3] time: 0.485871 bw: 17.4032 +[8:4] time: 0.485799 bw: 17.4058 +[9:0] time: 0.533047 bw: 15.863 +[9:1] time: 0.485828 bw: 17.4047 +[9:2] time: 0.486681 bw: 17.3742 +[9:3] time: 0.485715 bw: 17.4088 +[9:4] time: 0.486352 bw: 17.386 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.505271 -time0: 0.544749 -bw: 16.7353 -bw0: 15.5227 +time: 0.488052 +time0: 0.526692 +bw: 17.3262 +bw0: 16.0555 ======================================= -Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] -Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002710 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002640 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002632 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002644 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002668 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002646 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002636 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002620 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002740 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002714 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002678 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002670 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002680 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002654 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002728 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002658 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002682 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002706 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002606 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002686 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002716 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002702 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002688 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002594 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002674 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002708 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002720 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002602 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002570 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002610 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002554 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002698 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002614 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002590 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002736 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002642 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002652 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002588 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002712 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002684 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002626 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002600 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002732 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002638 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002734 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002730 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002722 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002612 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002656 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002592 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002552,002554,002570,002588,002590,002592,002594,002600,002602,002606,002608,002610,002612,002614,002620,002622,002624,002626,002632,002634,002636,002638,002640,002642,002644,002646,002652,002654,002656,002658,002664,002666,002668,002670,002674,002676,002678,002680,002682,002684,002686,002688,002690,002696,002698,002700,002702,002704,002706,002708,002710,002712,002714,002716,002718,002720,002722,002728,002730,002732,002734,002736,002738,002740] +Hostname: nid002552 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002664 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002608 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002676 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002696 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 1.66756 -[0:0] time: 1.65 bw: 5.12468 -[0:1] time: 1.65037 bw: 5.12352 -[0:2] time: 1.63637 bw: 5.16736 -[0:3] time: 1.63381 bw: 5.17546 -[0:4] time: 1.6333 bw: 5.17707 -[1:0] time: 1.67583 bw: 5.04569 -[1:1] time: 1.63422 bw: 5.17415 -[1:2] time: 1.63415 bw: 5.17439 -[1:3] time: 1.63126 bw: 5.18354 -[1:4] time: 1.63254 bw: 5.17949 -[2:0] time: 1.67461 bw: 5.04937 -[2:1] time: 1.63353 bw: 5.17635 -[2:2] time: 1.63351 bw: 5.1764 -[2:3] time: 1.63255 bw: 5.17945 -[2:4] time: 1.63238 bw: 5.17998 -[3:0] time: 1.67806 bw: 5.03899 -[3:1] time: 1.63336 bw: 5.17687 -[3:2] time: 1.63386 bw: 5.1753 -[3:3] time: 1.63344 bw: 5.17663 -[3:4] time: 1.63266 bw: 5.17909 -[4:0] time: 1.67582 bw: 5.04572 -[4:1] time: 1.63484 bw: 5.17219 -[4:2] time: 1.63459 bw: 5.17299 -[4:3] time: 1.63348 bw: 5.17652 -[4:4] time: 1.63232 bw: 5.18017 -[5:0] time: 1.68209 bw: 5.02692 -[5:1] time: 1.63368 bw: 5.17588 -[5:2] time: 1.63293 bw: 5.17825 -[5:3] time: 1.63366 bw: 5.17593 -[5:4] time: 1.63276 bw: 5.17878 -[6:0] time: 1.67957 bw: 5.03446 -[6:1] time: 1.63476 bw: 5.17245 -[6:2] time: 1.63506 bw: 5.17149 -[6:3] time: 1.63559 bw: 5.16983 -[6:4] time: 1.63509 bw: 5.1714 -[7:0] time: 1.68332 bw: 5.02323 -[7:1] time: 1.63503 bw: 5.1716 -[7:2] time: 1.63607 bw: 5.16831 -[7:3] time: 1.63486 bw: 5.17212 -[7:4] time: 1.63246 bw: 5.17973 -[8:0] time: 1.68721 bw: 5.01165 -[8:1] time: 1.63491 bw: 5.17196 -[8:2] time: 1.63552 bw: 5.17006 -[8:3] time: 1.63514 bw: 5.17124 -[8:4] time: 1.6335 bw: 5.17645 -[9:0] time: 1.68562 bw: 5.01637 -[9:1] time: 1.63633 bw: 5.16748 -[9:2] time: 1.6358 bw: 5.16917 -[9:3] time: 1.63311 bw: 5.17768 -[9:4] time: 1.63399 bw: 5.17488 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.66391 +[0:0] time: 1.64476 bw: 5.14101 +[0:1] time: 1.64686 bw: 5.13446 +[0:2] time: 1.63293 bw: 5.17825 +[0:3] time: 1.63026 bw: 5.18672 +[0:4] time: 1.63269 bw: 5.17902 +[1:0] time: 1.67293 bw: 5.05444 +[1:1] time: 1.63064 bw: 5.18553 +[1:2] time: 1.63157 bw: 5.18256 +[1:3] time: 1.63407 bw: 5.17464 +[1:4] time: 1.63318 bw: 5.17745 +[2:0] time: 1.67139 bw: 5.0591 +[2:1] time: 1.63042 bw: 5.18622 +[2:2] time: 1.63322 bw: 5.17732 +[2:3] time: 1.63371 bw: 5.17578 +[2:4] time: 1.63113 bw: 5.18395 +[3:0] time: 1.67598 bw: 5.04523 +[3:1] time: 1.63183 bw: 5.18172 +[3:2] time: 1.63347 bw: 5.17654 +[3:3] time: 1.63179 bw: 5.18185 +[3:4] time: 1.63168 bw: 5.18222 +[4:0] time: 1.67793 bw: 5.03937 +[4:1] time: 1.63085 bw: 5.18485 +[4:2] time: 1.63223 bw: 5.18046 +[4:3] time: 1.6303 bw: 5.18661 +[4:4] time: 1.63149 bw: 5.18283 +[5:0] time: 1.68256 bw: 5.02551 +[5:1] time: 1.63279 bw: 5.1787 +[5:2] time: 1.63057 bw: 5.18573 +[5:3] time: 1.63221 bw: 5.18053 +[5:4] time: 1.63259 bw: 5.17934 +[6:0] time: 1.68165 bw: 5.02823 +[6:1] time: 1.63297 bw: 5.17812 +[6:2] time: 1.63289 bw: 5.17836 +[6:3] time: 1.63034 bw: 5.18646 +[6:4] time: 1.63235 bw: 5.18009 +[7:0] time: 1.68523 bw: 5.01754 +[7:1] time: 1.63232 bw: 5.1802 +[7:2] time: 1.63211 bw: 5.18084 +[7:3] time: 1.63264 bw: 5.17916 +[7:4] time: 1.63266 bw: 5.17912 +[8:0] time: 1.68576 bw: 5.01597 +[8:1] time: 1.63348 bw: 5.1765 +[8:2] time: 1.63112 bw: 5.184 +[8:3] time: 1.63297 bw: 5.17811 +[8:4] time: 1.63433 bw: 5.1738 +[9:0] time: 1.68523 bw: 5.01754 +[9:1] time: 1.63059 bw: 5.18569 +[9:2] time: 1.63355 bw: 5.17628 +[9:3] time: 1.63293 bw: 5.17823 +[9:4] time: 1.63111 bw: 5.18401 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 1.63442 -time0: 1.67721 -bw: 5.17354 -bw0: 5.04171 -======================================= - -Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] -Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.348451 -[0:0] time: 0.265063 bw: 31.9008 -[0:1] time: 0.259007 bw: 32.6467 -[0:2] time: 0.258838 bw: 32.668 -[0:3] time: 0.258738 bw: 32.6806 -[0:4] time: 0.258746 bw: 32.6796 -[1:0] time: 0.259437 bw: 32.5925 -[1:1] time: 0.258729 bw: 32.6818 -[1:2] time: 0.258714 bw: 32.6836 -[1:3] time: 0.258726 bw: 32.6821 -[1:4] time: 0.258714 bw: 32.6836 -[2:0] time: 0.259239 bw: 32.6174 -[2:1] time: 0.258773 bw: 32.6762 -[2:2] time: 0.258755 bw: 32.6785 -[2:3] time: 0.258763 bw: 32.6774 -[2:4] time: 0.258837 bw: 32.6681 -[3:0] time: 0.259261 bw: 32.6147 -[3:1] time: 0.258712 bw: 32.6838 -[3:2] time: 0.258719 bw: 32.683 -[3:3] time: 0.258746 bw: 32.6796 -[3:4] time: 0.258733 bw: 32.6812 -[4:0] time: 0.259452 bw: 32.5907 -[4:1] time: 0.258776 bw: 32.6759 -[4:2] time: 0.258747 bw: 32.6795 -[4:3] time: 0.258762 bw: 32.6776 -[4:4] time: 0.258885 bw: 32.6621 -[5:0] time: 0.259056 bw: 32.6405 -[5:1] time: 0.258695 bw: 32.6861 -[5:2] time: 0.258728 bw: 32.6819 -[5:3] time: 0.258711 bw: 32.684 -[5:4] time: 0.258708 bw: 32.6844 -[6:0] time: 0.258975 bw: 32.6506 -[6:1] time: 0.258741 bw: 32.6803 -[6:2] time: 0.258764 bw: 32.6773 -[6:3] time: 0.258812 bw: 32.6712 -[6:4] time: 0.258767 bw: 32.677 -[7:0] time: 0.259106 bw: 32.6342 -[7:1] time: 0.258743 bw: 32.6799 -[7:2] time: 0.258765 bw: 32.6773 -[7:3] time: 0.258771 bw: 32.6765 -[7:4] time: 0.258684 bw: 32.6875 -[8:0] time: 0.259206 bw: 32.6217 -[8:1] time: 0.258739 bw: 32.6804 -[8:2] time: 0.258745 bw: 32.6797 -[8:3] time: 0.258746 bw: 32.6796 -[8:4] time: 0.258748 bw: 32.6794 -[9:0] time: 0.259128 bw: 32.6314 -[9:1] time: 0.258723 bw: 32.6825 -[9:2] time: 0.258742 bw: 32.6801 -[9:3] time: 0.258746 bw: 32.6796 -[9:4] time: 0.258759 bw: 32.678 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.258756 -time0: 0.259792 -bw: 32.6783 -bw0: 32.5495 -======================================= - -Slurm Job Hostlist: nid[004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786] -Hostname: nid004168 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.374189 -[0:0] time: 0.302522 bw: 27.9507 -[0:1] time: 0.26733 bw: 31.6302 -[0:2] time: 0.26711 bw: 31.6563 -[0:3] time: 0.267154 bw: 31.6511 -[0:4] time: 0.267168 bw: 31.6494 -[1:0] time: 0.267367 bw: 31.6259 -[1:1] time: 0.266981 bw: 31.6716 -[1:2] time: 0.266991 bw: 31.6704 -[1:3] time: 0.266983 bw: 31.6714 -[1:4] time: 0.266985 bw: 31.6711 -[2:0] time: 0.267516 bw: 31.6083 -[2:1] time: 0.267168 bw: 31.6494 -[2:2] time: 0.267161 bw: 31.6502 -[2:3] time: 0.267157 bw: 31.6508 -[2:4] time: 0.267202 bw: 31.6454 -[3:0] time: 0.267328 bw: 31.6305 -[3:1] time: 0.266962 bw: 31.6738 -[3:2] time: 0.266987 bw: 31.6709 -[3:3] time: 0.266998 bw: 31.6696 -[3:4] time: 0.266974 bw: 31.6725 -[4:0] time: 0.267575 bw: 31.6013 -[4:1] time: 0.267145 bw: 31.6521 -[4:2] time: 0.267163 bw: 31.65 -[4:3] time: 0.267159 bw: 31.6505 -[4:4] time: 0.267189 bw: 31.6469 -[5:0] time: 0.267419 bw: 31.6197 -[5:1] time: 0.267003 bw: 31.669 -[5:2] time: 0.266964 bw: 31.6737 -[5:3] time: 0.266965 bw: 31.6735 -[5:4] time: 0.266995 bw: 31.67 -[6:0] time: 0.267626 bw: 31.5953 -[6:1] time: 0.267199 bw: 31.6458 -[6:2] time: 0.267202 bw: 31.6454 -[6:3] time: 0.267137 bw: 31.6531 -[6:4] time: 0.267165 bw: 31.6498 -[7:0] time: 0.267331 bw: 31.6301 -[7:1] time: 0.266989 bw: 31.6706 -[7:2] time: 0.266989 bw: 31.6707 -[7:3] time: 0.267005 bw: 31.6687 -[7:4] time: 0.267034 bw: 31.6654 -[8:0] time: 0.267876 bw: 31.5658 -[8:1] time: 0.267225 bw: 31.6427 -[8:2] time: 0.267156 bw: 31.6509 -[8:3] time: 0.267182 bw: 31.6478 -[8:4] time: 0.267177 bw: 31.6484 -[9:0] time: 0.267372 bw: 31.6253 -[9:1] time: 0.266982 bw: 31.6715 -[9:2] time: 0.266988 bw: 31.6708 -[9:3] time: 0.267005 bw: 31.6687 -[9:4] time: 0.267011 bw: 31.6681 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.267084 -time0: 0.270993 -bw: 31.6595 -bw0: 31.2453 +time: 1.63252 +time0: 1.67634 +bw: 5.17956 +bw0: 5.04439 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out index 629cdac..62ee613 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0032_TPN_4.out @@ -1,791 +1,653 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] -Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005292 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005210 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005236 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005212 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005234 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005236 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005228 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005292 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005240 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005216 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005292 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005212 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005236 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005292 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005242 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005236 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005274 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005212 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005298 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005258 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005272 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005216 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005240 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005274 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005240 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005264 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005240 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005216 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005274 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005276 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005258 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005232 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005212 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005272 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005228 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005210 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005258 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005274 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005216 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005302 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005228 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005258 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005270 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005228 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005272 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005266 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005272 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005244 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005250 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005244 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005210 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005244 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005242 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005250 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[005210,005212,005216,005218,005228,005230,005232,005234,005236,005238,005240,005242,005244,005246,005248,005250,005258,005264,005266,005268,005270,005272,005274,005276,005280,005288,005290,005292,005294,005296,005298,005302] +Hostname: nid005210 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005250 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005244 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005242 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005234 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005280 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005234 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005242 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005234 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005250 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005230 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005232 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005232 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005232 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005294 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005288 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005276 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005268 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005276 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005266 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005266 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005288 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005276 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005266 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005298 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005298 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005280 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005248 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005298 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005280 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005230 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005230 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005230 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005238 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005280 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005294 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005268 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005288 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005294 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005294 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005288 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005264 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005270 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005238 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005270 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005268 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005264 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005302 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005302 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005238 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005264 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005270 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005268 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005218 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005238 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005302 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005248 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005248 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005218 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005218 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005218 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005246 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005296 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005246 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005296 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005290 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005296 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005246 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005296 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005246 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005290 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005290 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005290 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.591198 -[0:0] time: 0.326292 bw: 26.1202 -[0:1] time: 0.283811 bw: 30.0299 -[0:2] time: 0.287136 bw: 29.6822 -[0:3] time: 0.29624 bw: 28.77 -[0:4] time: 0.300375 bw: 28.374 -[1:0] time: 0.338962 bw: 25.1439 -[1:1] time: 0.293263 bw: 29.0621 -[1:2] time: 0.28744 bw: 29.6508 -[1:3] time: 0.2853 bw: 29.8732 -[1:4] time: 0.28324 bw: 30.0904 -[2:0] time: 0.338344 bw: 25.1898 -[2:1] time: 0.283691 bw: 30.0426 -[2:2] time: 0.283325 bw: 30.0814 -[2:3] time: 0.283101 bw: 30.1052 -[2:4] time: 0.283151 bw: 30.1 -[3:0] time: 0.340839 bw: 25.0054 -[3:1] time: 0.282444 bw: 30.1753 -[3:2] time: 0.282981 bw: 30.118 -[3:3] time: 0.283065 bw: 30.1091 -[3:4] time: 0.282786 bw: 30.1388 -[4:0] time: 0.338792 bw: 25.1565 -[4:1] time: 0.283255 bw: 30.0889 -[4:2] time: 0.282958 bw: 30.1205 -[4:3] time: 0.283124 bw: 30.1028 -[4:4] time: 0.283498 bw: 30.0631 -[5:0] time: 0.339911 bw: 25.0737 -[5:1] time: 0.283083 bw: 30.1072 -[5:2] time: 0.28267 bw: 30.1512 -[5:3] time: 0.28261 bw: 30.1576 -[5:4] time: 0.282612 bw: 30.1574 -[6:0] time: 0.341024 bw: 24.9919 -[6:1] time: 0.283905 bw: 30.0199 -[6:2] time: 0.283659 bw: 30.046 -[6:3] time: 0.283697 bw: 30.042 -[6:4] time: 0.283859 bw: 30.0248 -[7:0] time: 0.343417 bw: 24.8177 -[7:1] time: 0.283913 bw: 30.0192 -[7:2] time: 0.283966 bw: 30.0135 -[7:3] time: 0.284001 bw: 30.0099 -[7:4] time: 0.283763 bw: 30.035 -[8:0] time: 0.341885 bw: 24.929 -[8:1] time: 0.28372 bw: 30.0396 -[8:2] time: 0.283598 bw: 30.0525 -[8:3] time: 0.284167 bw: 29.9924 -[8:4] time: 0.283792 bw: 30.0319 -[9:0] time: 0.345817 bw: 24.6455 -[9:1] time: 0.283857 bw: 30.0251 -[9:2] time: 0.284069 bw: 30.0027 -[9:3] time: 0.293016 bw: 29.0865 -[9:4] time: 0.284317 bw: 29.9765 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 0.584018 +[0:0] time: 0.31969 bw: 26.6597 +[0:1] time: 0.281344 bw: 30.2933 +[0:2] time: 0.291324 bw: 29.2555 +[0:3] time: 0.293715 bw: 29.0173 +[0:4] time: 0.294006 bw: 28.9886 +[1:0] time: 0.33809 bw: 25.2087 +[1:1] time: 0.296366 bw: 28.7578 +[1:2] time: 0.284625 bw: 29.9441 +[1:3] time: 0.283986 bw: 30.0115 +[1:4] time: 0.282463 bw: 30.1732 +[2:0] time: 0.339307 bw: 25.1183 +[2:1] time: 0.282546 bw: 30.1644 +[2:2] time: 0.282709 bw: 30.147 +[2:3] time: 0.283283 bw: 30.0859 +[2:4] time: 0.282949 bw: 30.1214 +[3:0] time: 0.342663 bw: 24.8723 +[3:1] time: 0.28265 bw: 30.1532 +[3:2] time: 0.282558 bw: 30.1631 +[3:3] time: 0.282514 bw: 30.1678 +[3:4] time: 0.282317 bw: 30.1889 +[4:0] time: 0.339059 bw: 25.1367 +[4:1] time: 0.282185 bw: 30.2029 +[4:2] time: 0.282378 bw: 30.1823 +[4:3] time: 0.282463 bw: 30.1733 +[4:4] time: 0.282016 bw: 30.2211 +[5:0] time: 0.335098 bw: 25.4338 +[5:1] time: 0.282488 bw: 30.1706 +[5:2] time: 0.282437 bw: 30.176 +[5:3] time: 0.282175 bw: 30.204 +[5:4] time: 0.282226 bw: 30.1986 +[6:0] time: 0.335316 bw: 25.4173 +[6:1] time: 0.281995 bw: 30.2233 +[6:2] time: 0.282092 bw: 30.2129 +[6:3] time: 0.282782 bw: 30.1392 +[6:4] time: 0.282294 bw: 30.1913 +[7:0] time: 0.336553 bw: 25.3239 +[7:1] time: 0.283184 bw: 30.0964 +[7:2] time: 0.28224 bw: 30.1971 +[7:3] time: 0.28192 bw: 30.2314 +[7:4] time: 0.282505 bw: 30.1688 +[8:0] time: 0.335773 bw: 25.3827 +[8:1] time: 0.281952 bw: 30.228 +[8:2] time: 0.282883 bw: 30.1284 +[8:3] time: 0.282329 bw: 30.1875 +[8:4] time: 0.281957 bw: 30.2274 +[9:0] time: 0.342655 bw: 24.8729 +[9:1] time: 0.282186 bw: 30.2029 +[9:2] time: 0.281682 bw: 30.2569 +[9:3] time: 0.282238 bw: 30.1973 +[9:4] time: 0.282309 bw: 30.1897 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.284911 -time0: 0.339528 -bw: 29.9192 -bw0: 25.1074 +time: 0.283607 +time0: 0.33642 +bw: 30.056 +bw0: 25.3426 ======================================= -Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] -Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005210 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005302 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005212 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005236 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005270 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005264 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005248 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005212 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005210 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005210 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005212 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[005210,005212,005216,005218,005228,005230,005232,005234,005236,005238,005240,005242,005244,005246,005248,005250,005258,005264,005266,005268,005270,005272,005274,005276,005280,005288,005290,005292,005294,005296,005298,005302] +Hostname: nid005236 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005210 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005212 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005236 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005236 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005248 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005228 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005248 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005216 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005270 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005274 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005292 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005290 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005270 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005230 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005216 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005246 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005270 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005234 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005218 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005216 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005272 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005216 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005294 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005302 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005264 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005302 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005264 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005302 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005264 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005234 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005272 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005268 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005272 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005272 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005296 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005258 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005228 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005234 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005230 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005234 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005232 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005230 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005228 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005228 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005230 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005246 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005218 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005218 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005258 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005274 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005294 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005274 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005258 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005274 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005232 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005218 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005294 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005232 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005280 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005296 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005258 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005294 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005246 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005232 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005298 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005296 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005296 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005268 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005246 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005290 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005290 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005266 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005290 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005268 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005292 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005298 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005238 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005292 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005292 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005298 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005268 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005298 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005266 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005266 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005280 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005280 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005266 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005288 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005280 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005288 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005238 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005238 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005288 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005238 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005288 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005244 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005244 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005276 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005244 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005244 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005276 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005242 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005276 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005276 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005242 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005242 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005242 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005250 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005250 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005250 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005250 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005240 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005240 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005240 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005240 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.5125 -[0:0] time: 15.457 bw: 0.551391 -[0:1] time: 0.585132 bw: 14.5656 -[0:2] time: 0.596263 bw: 14.2937 -[0:3] time: 0.570848 bw: 14.9301 -[0:4] time: 0.565644 bw: 15.0675 -[1:0] time: 15.515 bw: 0.549328 -[1:1] time: 0.564512 bw: 15.0977 -[1:2] time: 0.56464 bw: 15.0943 -[1:3] time: 0.564064 bw: 15.1097 -[1:4] time: 0.567994 bw: 15.0051 -[2:0] time: 15.591 bw: 0.546649 -[2:1] time: 0.565183 bw: 15.0798 -[2:2] time: 0.563374 bw: 15.1282 -[2:3] time: 0.564642 bw: 15.0942 -[2:4] time: 0.565448 bw: 15.0727 -[3:0] time: 15.6007 bw: 0.546312 -[3:1] time: 0.564545 bw: 15.0968 -[3:2] time: 0.564509 bw: 15.0978 -[3:3] time: 0.563897 bw: 15.1141 -[3:4] time: 0.565939 bw: 15.0596 -[4:0] time: 15.5648 bw: 0.54757 -[4:1] time: 0.566016 bw: 15.0576 -[4:2] time: 0.563935 bw: 15.1131 -[4:3] time: 0.564616 bw: 15.0949 -[4:4] time: 0.564514 bw: 15.0976 -[5:0] time: 15.5723 bw: 0.547305 -[5:1] time: 0.564027 bw: 15.1107 -[5:2] time: 0.56434 bw: 15.1023 -[5:3] time: 0.563738 bw: 15.1184 -[5:4] time: 0.565801 bw: 15.0633 -[6:0] time: 15.5523 bw: 0.548009 -[6:1] time: 0.566283 bw: 15.0505 -[6:2] time: 0.565632 bw: 15.0678 -[6:3] time: 0.566279 bw: 15.0506 -[6:4] time: 0.565684 bw: 15.0664 -[7:0] time: 15.5316 bw: 0.54874 -[7:1] time: 0.564823 bw: 15.0894 -[7:2] time: 0.565513 bw: 15.071 -[7:3] time: 0.565421 bw: 15.0734 -[7:4] time: 0.567205 bw: 15.026 -[8:0] time: 15.5539 bw: 0.547954 -[8:1] time: 0.566359 bw: 15.0485 -[8:2] time: 0.565391 bw: 15.0742 -[8:3] time: 0.565871 bw: 15.0614 -[8:4] time: 0.564878 bw: 15.0879 -[9:0] time: 15.5067 bw: 0.549624 -[9:1] time: 0.565169 bw: 15.0801 -[9:2] time: 0.566242 bw: 15.0516 -[9:3] time: 0.565887 bw: 15.061 -[9:4] time: 0.566875 bw: 15.0348 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 460.664 +[0:0] time: 15.4316 bw: 0.552296 +[0:1] time: 0.57873 bw: 14.7268 +[0:2] time: 0.59943 bw: 14.2182 +[0:3] time: 0.572114 bw: 14.8971 +[0:4] time: 0.565347 bw: 15.0754 +[1:0] time: 15.4697 bw: 0.550937 +[1:1] time: 0.565954 bw: 15.0592 +[1:2] time: 0.566423 bw: 15.0467 +[1:3] time: 0.56592 bw: 15.0601 +[1:4] time: 0.566302 bw: 15.05 +[2:0] time: 15.4729 bw: 0.550824 +[2:1] time: 0.56607 bw: 15.0561 +[2:2] time: 0.566369 bw: 15.0482 +[2:3] time: 0.566474 bw: 15.0454 +[2:4] time: 0.566066 bw: 15.0562 +[3:0] time: 15.4643 bw: 0.551129 +[3:1] time: 0.564919 bw: 15.0868 +[3:2] time: 0.565384 bw: 15.0744 +[3:3] time: 0.565699 bw: 15.066 +[3:4] time: 0.566744 bw: 15.0382 +[4:0] time: 15.4839 bw: 0.550431 +[4:1] time: 0.565911 bw: 15.0604 +[4:2] time: 0.565439 bw: 15.0729 +[4:3] time: 0.566148 bw: 15.0541 +[4:4] time: 0.566277 bw: 15.0506 +[5:0] time: 15.4764 bw: 0.550697 +[5:1] time: 0.564962 bw: 15.0857 +[5:2] time: 0.565712 bw: 15.0657 +[5:3] time: 0.56618 bw: 15.0532 +[5:4] time: 0.567042 bw: 15.0303 +[6:0] time: 15.4452 bw: 0.55181 +[6:1] time: 0.565698 bw: 15.066 +[6:2] time: 0.565534 bw: 15.0704 +[6:3] time: 0.566873 bw: 15.0348 +[6:4] time: 0.566765 bw: 15.0377 +[7:0] time: 15.4766 bw: 0.55069 +[7:1] time: 0.565436 bw: 15.073 +[7:2] time: 0.566073 bw: 15.0561 +[7:3] time: 0.565812 bw: 15.063 +[7:4] time: 0.566723 bw: 15.0388 +[8:0] time: 15.4991 bw: 0.549893 +[8:1] time: 0.565664 bw: 15.0669 +[8:2] time: 0.565964 bw: 15.059 +[8:3] time: 0.566436 bw: 15.0464 +[8:4] time: 0.565907 bw: 15.0605 +[9:0] time: 15.5029 bw: 0.549758 +[9:1] time: 0.565269 bw: 15.0775 +[9:2] time: 0.566465 bw: 15.0456 +[9:3] time: 0.566155 bw: 15.0539 +[9:4] time: 0.566604 bw: 15.042 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.566678 -time0: 15.5445 -bw: 15.0415 -bw0: 0.548288 -======================================= - -Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] -Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 0.667972 -[0:0] time: 0.5193 bw: 16.4121 -[0:1] time: 0.518693 bw: 16.4313 -[0:2] time: 0.51868 bw: 16.4318 -[0:3] time: 0.518733 bw: 16.4301 -[0:4] time: 0.518752 bw: 16.4295 -[1:0] time: 0.520381 bw: 16.3781 -[1:1] time: 0.518597 bw: 16.4344 -[1:2] time: 0.518634 bw: 16.4332 -[1:3] time: 0.518553 bw: 16.4358 -[1:4] time: 0.518552 bw: 16.4358 -[2:0] time: 0.520946 bw: 16.3603 -[2:1] time: 0.518697 bw: 16.4312 -[2:2] time: 0.518714 bw: 16.4307 -[2:3] time: 0.518654 bw: 16.4326 -[2:4] time: 0.518661 bw: 16.4324 -[3:0] time: 0.519394 bw: 16.4092 -[3:1] time: 0.51852 bw: 16.4368 -[3:2] time: 0.518582 bw: 16.4349 -[3:3] time: 0.518563 bw: 16.4355 -[3:4] time: 0.518599 bw: 16.4343 -[4:0] time: 0.519963 bw: 16.3912 -[4:1] time: 0.518634 bw: 16.4332 -[4:2] time: 0.518633 bw: 16.4333 -[4:3] time: 0.518577 bw: 16.435 -[4:4] time: 0.518585 bw: 16.4348 -[5:0] time: 0.520673 bw: 16.3689 -[5:1] time: 0.518647 bw: 16.4328 -[5:2] time: 0.518862 bw: 16.426 -[5:3] time: 0.518672 bw: 16.432 -[5:4] time: 0.518515 bw: 16.437 -[6:0] time: 0.520594 bw: 16.3713 -[6:1] time: 0.51861 bw: 16.434 -[6:2] time: 0.518593 bw: 16.4345 -[6:3] time: 0.518637 bw: 16.4331 -[6:4] time: 0.518613 bw: 16.4339 -[7:0] time: 0.519122 bw: 16.4178 -[7:1] time: 0.518617 bw: 16.4338 -[7:2] time: 0.518567 bw: 16.4353 -[7:3] time: 0.51856 bw: 16.4356 -[7:4] time: 0.518517 bw: 16.4369 -[8:0] time: 0.521128 bw: 16.3546 -[8:1] time: 0.518648 bw: 16.4328 -[8:2] time: 0.518637 bw: 16.4331 -[8:3] time: 0.518654 bw: 16.4326 -[8:4] time: 0.518699 bw: 16.4312 -[9:0] time: 0.520828 bw: 16.364 -[9:1] time: 0.518558 bw: 16.4356 -[9:2] time: 0.518527 bw: 16.4366 -[9:3] time: 0.518639 bw: 16.4331 -[9:4] time: 0.518567 bw: 16.4353 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 0.518624 -time0: 0.520233 -bw: 16.4335 -bw0: 16.3827 -======================================= - -Hostname: nid005480 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005426 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005480 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005422 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005442 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005458 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005422 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005460 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005488 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005442 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005388 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005420 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005450 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005376 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490] -Hostname: nid005376 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005468 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005428 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005434 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005434 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 0.653662 -[0:0] time: 0.536277 bw: 15.8926 -[0:1] time: 0.535427 bw: 15.9178 -[0:2] time: 0.535349 bw: 15.9201 -[0:3] time: 0.535339 bw: 15.9204 -[0:4] time: 0.535387 bw: 15.919 -[1:0] time: 0.538748 bw: 15.8197 -[1:1] time: 0.536134 bw: 15.8968 -[1:2] time: 0.536122 bw: 15.8972 -[1:3] time: 0.536122 bw: 15.8972 -[1:4] time: 0.536121 bw: 15.8972 -[2:0] time: 0.538528 bw: 15.8262 -[2:1] time: 0.535429 bw: 15.9178 -[2:2] time: 0.535365 bw: 15.9196 -[2:3] time: 0.535403 bw: 15.9185 -[2:4] time: 0.535437 bw: 15.9175 -[3:0] time: 0.537047 bw: 15.8698 -[3:1] time: 0.536155 bw: 15.8962 -[3:2] time: 0.536211 bw: 15.8945 -[3:3] time: 0.536123 bw: 15.8971 -[3:4] time: 0.536134 bw: 15.8968 -[4:0] time: 0.536608 bw: 15.8828 -[4:1] time: 0.535337 bw: 15.9205 -[4:2] time: 0.535375 bw: 15.9194 -[4:3] time: 0.535415 bw: 15.9182 -[4:4] time: 0.535426 bw: 15.9178 -[5:0] time: 0.540036 bw: 15.782 -[5:1] time: 0.536098 bw: 15.8979 -[5:2] time: 0.536208 bw: 15.8946 -[5:3] time: 0.536152 bw: 15.8963 -[5:4] time: 0.536112 bw: 15.8975 -[6:0] time: 0.536353 bw: 15.8903 -[6:1] time: 0.535434 bw: 15.9176 -[6:2] time: 0.53537 bw: 15.9195 -[6:3] time: 0.535415 bw: 15.9182 -[6:4] time: 0.535406 bw: 15.9184 -[7:0] time: 0.536922 bw: 15.8735 -[7:1] time: 0.53615 bw: 15.8964 -[7:2] time: 0.536102 bw: 15.8978 -[7:3] time: 0.536164 bw: 15.8959 -[7:4] time: 0.536171 bw: 15.8957 -[8:0] time: 0.538298 bw: 15.8329 -[8:1] time: 0.535419 bw: 15.9181 -[8:2] time: 0.535359 bw: 15.9198 -[8:3] time: 0.53539 bw: 15.9189 -[8:4] time: 0.535357 bw: 15.9199 -[9:0] time: 0.538671 bw: 15.822 -[9:1] time: 0.536115 bw: 15.8974 -[9:2] time: 0.53615 bw: 15.8964 -[9:3] time: 0.536071 bw: 15.8987 -[9:4] time: 0.536244 bw: 15.8935 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 0.535767 -time0: 0.537749 -bw: 15.9077 -bw0: 15.8492 +time: 0.567325 +time0: 15.4723 +bw: 15.0242 +bw0: 0.550846 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out index 47fe185..b97c501 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00128_N_0128_TPN_1.out @@ -1,791 +1,653 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002882 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002848 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 0 - -[-1] time: 0.716111 -[0:0] time: 0.572177 bw: 14.8954 -[0:1] time: 0.55515 bw: 15.3523 -[0:2] time: 0.546648 bw: 15.5911 -[0:3] time: 0.539229 bw: 15.8056 -[0:4] time: 0.531681 bw: 16.03 -[1:0] time: 0.564211 bw: 15.1057 -[1:1] time: 0.53434 bw: 15.9502 -[1:2] time: 0.531578 bw: 16.0331 -[1:3] time: 0.530334 bw: 16.0707 -[1:4] time: 0.530595 bw: 16.0628 -[2:0] time: 0.562702 bw: 15.1463 -[2:1] time: 0.530771 bw: 16.0574 -[2:2] time: 0.530011 bw: 16.0805 -[2:3] time: 0.530063 bw: 16.0789 -[2:4] time: 0.530208 bw: 16.0745 -[3:0] time: 0.56185 bw: 15.1692 -[3:1] time: 0.529129 bw: 16.1073 -[3:2] time: 0.529142 bw: 16.1069 -[3:3] time: 0.529218 bw: 16.1046 -[3:4] time: 0.528903 bw: 16.1141 -[4:0] time: 0.563964 bw: 15.1124 -[4:1] time: 0.529316 bw: 16.1016 -[4:2] time: 0.529431 bw: 16.0981 -[4:3] time: 0.528864 bw: 16.1153 -[4:4] time: 0.528617 bw: 16.1229 -[5:0] time: 0.563973 bw: 15.1121 -[5:1] time: 0.528717 bw: 16.1198 -[5:2] time: 0.528605 bw: 16.1232 -[5:3] time: 0.528723 bw: 16.1197 -[5:4] time: 0.528914 bw: 16.1138 -[6:0] time: 0.56483 bw: 15.0892 -[6:1] time: 0.530213 bw: 16.0744 -[6:2] time: 0.529582 bw: 16.0935 -[6:3] time: 0.529813 bw: 16.0865 -[6:4] time: 0.529612 bw: 16.0926 -[7:0] time: 0.565838 bw: 15.0623 -[7:1] time: 0.529902 bw: 16.0838 -[7:2] time: 0.531852 bw: 16.0248 -[7:3] time: 0.530277 bw: 16.0724 -[7:4] time: 0.530147 bw: 16.0764 -[8:0] time: 0.566229 bw: 15.0519 -[8:1] time: 0.530155 bw: 16.0761 -[8:2] time: 0.529988 bw: 16.0812 -[8:3] time: 0.529909 bw: 16.0836 -[8:4] time: 0.529843 bw: 16.0856 -[9:0] time: 0.568095 bw: 15.0025 -[9:1] time: 0.530555 bw: 16.064 -[9:2] time: 0.530341 bw: 16.0705 -[9:3] time: 0.530028 bw: 16.08 -[9:4] time: 0.530318 bw: 16.0712 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 0 -time: 0.531268 -time0: 0.565387 -bw: 16.0438 -bw0: 15.0747 -======================================= - -Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002872 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002890 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002800,002802,002806,002808,002812,002824,002828,002834,002836,002838,002842,002844,002848,002850,002856,002860,002862,002870,002872,002874,002876,002878,002882,002888,002890,002892,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004884 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002870 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002856 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004858 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002878 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002800 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002844 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002876 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002806 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002860 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002892 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002842 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002802 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 0 - -[-1] time: 1.71178 -[0:0] time: 1.70396 bw: 5.00177 -[0:1] time: 1.70293 bw: 5.00479 -[0:2] time: 1.6719 bw: 5.0977 -[0:3] time: 1.67351 bw: 5.09279 -[0:4] time: 1.67165 bw: 5.09846 -[1:0] time: 1.72297 bw: 4.94659 -[1:1] time: 1.67043 bw: 5.10218 -[1:2] time: 1.67457 bw: 5.08957 -[1:3] time: 1.67374 bw: 5.0921 -[1:4] time: 1.675 bw: 5.08826 -[2:0] time: 1.71821 bw: 4.96028 -[2:1] time: 1.67485 bw: 5.0887 -[2:2] time: 1.67108 bw: 5.10018 -[2:3] time: 1.6739 bw: 5.0916 -[2:4] time: 1.67254 bw: 5.09573 -[3:0] time: 1.7311 bw: 4.92336 -[3:1] time: 1.67282 bw: 5.0949 -[3:2] time: 1.67472 bw: 5.0891 -[3:3] time: 1.67254 bw: 5.09575 -[3:4] time: 1.67546 bw: 5.08687 -[4:0] time: 1.72117 bw: 4.95177 -[4:1] time: 1.67535 bw: 5.08719 -[4:2] time: 1.67402 bw: 5.09123 -[4:3] time: 1.676 bw: 5.08521 -[4:4] time: 1.67294 bw: 5.09452 -[5:0] time: 1.73311 bw: 4.91766 -[5:1] time: 1.67263 bw: 5.09546 -[5:2] time: 1.67451 bw: 5.08974 -[5:3] time: 1.6728 bw: 5.09496 -[5:4] time: 1.6725 bw: 5.09585 -[6:0] time: 1.72353 bw: 4.94497 -[6:1] time: 1.67328 bw: 5.09349 -[6:2] time: 1.67098 bw: 5.1005 -[6:3] time: 1.67246 bw: 5.09598 -[6:4] time: 1.67077 bw: 5.10113 -[7:0] time: 1.7329 bw: 4.91823 -[7:1] time: 1.67098 bw: 5.10048 -[7:2] time: 1.67313 bw: 5.09393 -[7:3] time: 1.67094 bw: 5.10061 -[7:4] time: 1.67307 bw: 5.09412 -[8:0] time: 1.72269 bw: 4.94739 -[8:1] time: 1.67311 bw: 5.09401 -[8:2] time: 1.67077 bw: 5.10112 -[8:3] time: 1.67337 bw: 5.09321 -[8:4] time: 1.67086 bw: 5.10085 -[9:0] time: 1.73321 bw: 4.91737 -[9:1] time: 1.67033 bw: 5.10249 -[9:2] time: 1.67295 bw: 5.09448 -[9:3] time: 1.67019 bw: 5.10291 -[9:4] time: 1.67279 bw: 5.09497 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 0 -time: 1.67356 -time0: 1.72429 -bw: 5.09268 -bw0: 4.94294 -======================================= - -Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002808 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005048 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002836 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002828 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002834 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002824 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002888 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004956 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004942 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005000 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002874 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 -naive algo: 1 +naive algo: 0 -[-1] time: 0.621853 -[0:0] time: 0.527957 bw: 16.143 -[0:1] time: 0.518818 bw: 16.4274 -[0:2] time: 0.519047 bw: 16.4201 -[0:3] time: 0.51914 bw: 16.4172 -[0:4] time: 0.519105 bw: 16.4183 -[1:0] time: 0.525076 bw: 16.2316 -[1:1] time: 0.518668 bw: 16.4321 -[1:2] time: 0.518664 bw: 16.4323 -[1:3] time: 0.518614 bw: 16.4338 -[1:4] time: 0.518686 bw: 16.4316 -[2:0] time: 0.519386 bw: 16.4094 -[2:1] time: 0.518896 bw: 16.4249 -[2:2] time: 0.519053 bw: 16.42 -[2:3] time: 0.519038 bw: 16.4204 -[2:4] time: 0.51902 bw: 16.421 -[3:0] time: 0.519202 bw: 16.4153 -[3:1] time: 0.518679 bw: 16.4318 -[3:2] time: 0.518736 bw: 16.43 -[3:3] time: 0.51866 bw: 16.4324 -[3:4] time: 0.518643 bw: 16.4329 -[4:0] time: 0.519649 bw: 16.4011 -[4:1] time: 0.519031 bw: 16.4206 -[4:2] time: 0.519 bw: 16.4216 -[4:3] time: 0.519053 bw: 16.42 -[4:4] time: 0.519012 bw: 16.4212 -[5:0] time: 0.518999 bw: 16.4217 -[5:1] time: 0.518663 bw: 16.4323 -[5:2] time: 0.518764 bw: 16.4291 -[5:3] time: 0.518685 bw: 16.4316 -[5:4] time: 0.518749 bw: 16.4296 -[6:0] time: 0.519778 bw: 16.397 -[6:1] time: 0.519083 bw: 16.419 -[6:2] time: 0.518857 bw: 16.4262 -[6:3] time: 0.518874 bw: 16.4256 -[6:4] time: 0.518826 bw: 16.4271 -[7:0] time: 0.519398 bw: 16.4091 -[7:1] time: 0.518748 bw: 16.4296 -[7:2] time: 0.518624 bw: 16.4335 -[7:3] time: 0.518654 bw: 16.4326 -[7:4] time: 0.518643 bw: 16.4329 -[8:0] time: 0.519305 bw: 16.412 -[8:1] time: 0.518851 bw: 16.4263 -[8:2] time: 0.518835 bw: 16.4268 -[8:3] time: 0.518836 bw: 16.4268 -[8:4] time: 0.518805 bw: 16.4278 -[9:0] time: 0.519045 bw: 16.4202 -[9:1] time: 0.518683 bw: 16.4317 -[9:2] time: 0.518859 bw: 16.4261 -[9:3] time: 0.518642 bw: 16.433 -[9:4] time: 0.51867 bw: 16.4321 +[-1] time: 0.761905 +[0:0] time: 0.567007 bw: 15.0313 +[0:1] time: 0.54847 bw: 15.5393 +[0:2] time: 0.545596 bw: 15.6211 +[0:3] time: 0.543512 bw: 15.681 +[0:4] time: 0.538873 bw: 15.816 +[1:0] time: 0.564533 bw: 15.0971 +[1:1] time: 0.53499 bw: 15.9308 +[1:2] time: 0.534119 bw: 15.9568 +[1:3] time: 0.533535 bw: 15.9743 +[1:4] time: 0.532972 bw: 15.9911 +[2:0] time: 0.562748 bw: 15.145 +[2:1] time: 0.532495 bw: 16.0054 +[2:2] time: 0.531467 bw: 16.0364 +[2:3] time: 0.531776 bw: 16.0271 +[2:4] time: 0.532993 bw: 15.9905 +[3:0] time: 0.562935 bw: 15.14 +[3:1] time: 0.531084 bw: 16.048 +[3:2] time: 0.528896 bw: 16.1144 +[3:3] time: 0.528964 bw: 16.1123 +[3:4] time: 0.528978 bw: 16.1119 +[4:0] time: 0.5614 bw: 15.1814 +[4:1] time: 0.529245 bw: 16.1038 +[4:2] time: 0.529067 bw: 16.1091 +[4:3] time: 0.528806 bw: 16.1171 +[4:4] time: 0.528142 bw: 16.1374 +[5:0] time: 0.561554 bw: 15.1772 +[5:1] time: 0.528422 bw: 16.1288 +[5:2] time: 0.528904 bw: 16.1141 +[5:3] time: 0.526753 bw: 16.1799 +[5:4] time: 0.525917 bw: 16.2057 +[6:0] time: 0.558943 bw: 15.2481 +[6:1] time: 0.525985 bw: 16.2036 +[6:2] time: 0.526034 bw: 16.202 +[6:3] time: 0.527223 bw: 16.1655 +[6:4] time: 0.526408 bw: 16.1905 +[7:0] time: 0.559207 bw: 15.2409 +[7:1] time: 0.526616 bw: 16.1841 +[7:2] time: 0.525716 bw: 16.2118 +[7:3] time: 0.52606 bw: 16.2012 +[7:4] time: 0.525834 bw: 16.2082 +[8:0] time: 0.561552 bw: 15.1773 +[8:1] time: 0.526075 bw: 16.2008 +[8:2] time: 0.526347 bw: 16.1924 +[8:3] time: 0.52573 bw: 16.2114 +[8:4] time: 0.526376 bw: 16.1915 +[9:0] time: 0.56173 bw: 15.1725 +[9:1] time: 0.527716 bw: 16.1504 +[9:2] time: 0.526424 bw: 16.19 +[9:3] time: 0.52594 bw: 16.2049 +[9:4] time: 0.526137 bw: 16.1989 ======================================= type: device size: 4294967296 pad: 0 -naive: 1 -time: 0.518823 -time0: 0.520779 -bw: 16.4272 -bw0: 16.366 +naive: 0 +time: 0.530115 +time0: 0.562161 +bw: 16.079 +bw0: 15.1611 ======================================= -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002800,002802,002806,002808,002812,002824,002828,002834,002836,002838,002842,002844,002848,002850,002856,002860,002862,002870,002872,002874,002876,002878,002882,002888,002890,002892,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid002800 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002802 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005080 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004858 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002872 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002876 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002824 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002828 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002806 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002860 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002870 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002882 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002834 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005072 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005068 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002890 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004942 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002878 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004956 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005074 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005076 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005048 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002888 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005056 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005000 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002892 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002836 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002808 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005090 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002874 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002842 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002844 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002856 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002848 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 -naive algo: 1 +naive algo: 0 -[-1] time: 0.630766 -[0:0] time: 0.554362 bw: 15.3741 -[0:1] time: 0.535603 bw: 15.9126 -[0:2] time: 0.535613 bw: 15.9123 -[0:3] time: 0.535599 bw: 15.9127 -[0:4] time: 0.53562 bw: 15.9121 -[1:0] time: 0.535283 bw: 15.9221 -[1:1] time: 0.535252 bw: 15.923 -[1:2] time: 0.535348 bw: 15.9202 -[1:3] time: 0.535254 bw: 15.923 -[1:4] time: 0.535358 bw: 15.9199 -[2:0] time: 0.535795 bw: 15.9069 -[2:1] time: 0.535623 bw: 15.912 -[2:2] time: 0.535624 bw: 15.9119 -[2:3] time: 0.53564 bw: 15.9115 -[2:4] time: 0.535655 bw: 15.911 -[3:0] time: 0.53696 bw: 15.8724 -[3:1] time: 0.535262 bw: 15.9227 -[3:2] time: 0.535325 bw: 15.9209 -[3:3] time: 0.535266 bw: 15.9226 -[3:4] time: 0.535373 bw: 15.9194 -[4:0] time: 0.541276 bw: 15.7458 -[4:1] time: 0.535568 bw: 15.9136 -[4:2] time: 0.535632 bw: 15.9117 -[4:3] time: 0.535559 bw: 15.9139 -[4:4] time: 0.535605 bw: 15.9125 -[5:0] time: 0.544686 bw: 15.6472 -[5:1] time: 0.535263 bw: 15.9227 -[5:2] time: 0.535302 bw: 15.9215 -[5:3] time: 0.535404 bw: 15.9185 -[5:4] time: 0.535273 bw: 15.9224 -[6:0] time: 0.53593 bw: 15.9029 -[6:1] time: 0.535583 bw: 15.9132 -[6:2] time: 0.53559 bw: 15.913 -[6:3] time: 0.535631 bw: 15.9117 -[6:4] time: 0.535598 bw: 15.9127 -[7:0] time: 0.535719 bw: 15.9091 -[7:1] time: 0.535284 bw: 15.9221 -[7:2] time: 0.535338 bw: 15.9205 -[7:3] time: 0.535289 bw: 15.9219 -[7:4] time: 0.535391 bw: 15.9189 -[8:0] time: 0.536033 bw: 15.8998 -[8:1] time: 0.535615 bw: 15.9122 -[8:2] time: 0.535615 bw: 15.9122 -[8:3] time: 0.535629 bw: 15.9118 -[8:4] time: 0.535633 bw: 15.9117 -[9:0] time: 0.535771 bw: 15.9076 -[9:1] time: 0.535333 bw: 15.9206 -[9:2] time: 0.535343 bw: 15.9203 -[9:3] time: 0.535304 bw: 15.9215 -[9:4] time: 0.535462 bw: 15.9168 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.70274 +[0:0] time: 1.69397 bw: 5.03128 +[0:1] time: 1.69135 bw: 5.03907 +[0:2] time: 1.6666 bw: 5.1139 +[0:3] time: 1.66717 bw: 5.11216 +[0:4] time: 1.6667 bw: 5.11358 +[1:0] time: 1.71187 bw: 4.97867 +[1:1] time: 1.66799 bw: 5.10965 +[1:2] time: 1.6664 bw: 5.1145 +[1:3] time: 1.66759 bw: 5.11086 +[1:4] time: 1.66847 bw: 5.10817 +[2:0] time: 1.70707 bw: 4.99265 +[2:1] time: 1.66796 bw: 5.10973 +[2:2] time: 1.66823 bw: 5.10891 +[2:3] time: 1.66824 bw: 5.10887 +[2:4] time: 1.66676 bw: 5.1134 +[3:0] time: 1.71443 bw: 4.97124 +[3:1] time: 1.66911 bw: 5.10621 +[3:2] time: 1.66958 bw: 5.10478 +[3:3] time: 1.66953 bw: 5.10493 +[3:4] time: 1.66943 bw: 5.10523 +[4:0] time: 1.71277 bw: 4.97604 +[4:1] time: 1.66922 bw: 5.10586 +[4:2] time: 1.66982 bw: 5.10404 +[4:3] time: 1.66848 bw: 5.10813 +[4:4] time: 1.66914 bw: 5.1061 +[5:0] time: 1.71583 bw: 4.96717 +[5:1] time: 1.6714 bw: 5.0992 +[5:2] time: 1.67137 bw: 5.09931 +[5:3] time: 1.66778 bw: 5.11027 +[5:4] time: 1.669 bw: 5.10655 +[6:0] time: 1.71532 bw: 4.96866 +[6:1] time: 1.67015 bw: 5.10303 +[6:2] time: 1.6697 bw: 5.10441 +[6:3] time: 1.67154 bw: 5.0988 +[6:4] time: 1.67216 bw: 5.0969 +[7:0] time: 1.7169 bw: 4.96408 +[7:1] time: 1.66791 bw: 5.10988 +[7:2] time: 1.66936 bw: 5.10543 +[7:3] time: 1.67025 bw: 5.10273 +[7:4] time: 1.67084 bw: 5.10093 +[8:0] time: 1.7169 bw: 4.96408 +[8:1] time: 1.66954 bw: 5.1049 +[8:2] time: 1.66954 bw: 5.10491 +[8:3] time: 1.67036 bw: 5.10239 +[8:4] time: 1.66823 bw: 5.10891 +[9:0] time: 1.71979 bw: 4.95572 +[9:1] time: 1.66919 bw: 5.10597 +[9:2] time: 1.67032 bw: 5.10252 +[9:3] time: 1.67081 bw: 5.101 +[9:4] time: 1.67018 bw: 5.10295 ======================================= type: device size: 4294967296 pad: 4 -naive: 1 -time: 0.535466 -time0: 0.539181 -bw: 15.9166 -bw0: 15.8088 +naive: 0 +time: 1.66968 +time0: 1.71248 +bw: 5.10448 +bw0: 4.97696 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out index 31915c9..533b85b 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0064_TPN_4.out @@ -1,1303 +1,1165 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002734 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002640 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002684 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002712 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002656 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002588 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002594 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002654 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002708 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002676 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002550 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002636 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002684 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002666 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002620 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002594 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002592 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002614 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002676 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002684 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002684 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002674 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002720 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002608 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002602 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002656 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002588 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002570 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002610 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002594 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002634 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002654 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002638 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002656 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002642 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002670 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002626 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002696 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002718 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002606 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002588 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002666 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002640 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002594 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002592 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002704 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002636 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002714 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002654 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002640 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002676 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002656 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002734 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002718 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002722 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002718 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002640 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002592 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002588 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002712 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002676 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002654 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002734 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002552 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002712 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002734 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002636 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002592 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002698 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002642 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002666 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002708 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002622 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002710 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002712 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002674 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002642 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002636 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002614 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002614 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002570 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002590 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002614 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002708 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002720 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002620 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002570 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002642 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002674 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002728 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002570 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002552 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002674 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002638 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002720 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002638 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002620 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002620 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002638 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002720 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002670 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002708 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002606 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002664 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002722 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002550 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002686 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002670 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002696 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002610 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002714 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002550 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002602 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002722 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002690 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002606 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002698 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002670 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002710 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002696 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002690 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002610 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002606 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002610 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002550,002552,002554,002570,002588,002590,002592,002594,002600,002602,002606,002608,002610,002612,002614,002620,002622,002624,002626,002632,002634,002636,002638,002640,002642,002644,002646,002652,002654,002656,002658,002664,002666,002668,002670,002674,002676,002678,002680,002682,002684,002686,002688,002690,002696,002698,002700,002702,002704,002706,002708,002710,002712,002714,002716,002718,002720,002722,002728,002730,002732,002734,002736,002738] +Hostname: nid002722 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002550 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002710 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002738 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002704 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002710 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002696 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002600 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002698 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002626 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002552 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002704 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002698 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002602 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002632 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002626 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002678 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002608 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002626 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002624 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002608 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002608 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002590 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002658 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002632 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002714 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002688 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002702 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002714 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002658 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002590 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002686 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002716 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002552 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002602 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002686 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002682 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002622 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002590 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002700 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002686 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002716 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002658 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002658 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002716 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002600 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002682 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002728 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002600 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002682 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002716 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002632 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002680 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002682 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002728 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002600 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002632 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002728 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002680 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002680 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002680 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002688 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002664 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002664 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002688 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002688 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002664 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002678 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002678 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002678 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002736 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002736 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002668 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002736 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002668 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002668 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002736 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002668 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002612 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002644 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002612 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002732 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002706 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002652 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002732 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002646 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002732 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002612 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002732 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002612 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002652 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002706 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002644 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002644 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002706 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002644 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002706 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002652 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002554 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002554 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002652 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002554 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002554 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002646 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002646 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002646 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002730 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002730 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002730 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002730 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.647464 -[0:0] time: 0.372584 bw: 22.965 -[0:1] time: 0.353028 bw: 24.2371 -[0:2] time: 0.349702 bw: 24.4676 -[0:3] time: 0.339558 bw: 25.1986 -[0:4] time: 0.329432 bw: 25.9732 -[1:0] time: 0.380745 bw: 22.4728 -[1:1] time: 0.325236 bw: 26.3083 -[1:2] time: 0.322189 bw: 26.557 -[1:3] time: 0.322025 bw: 26.5705 -[1:4] time: 0.323599 bw: 26.4413 -[2:0] time: 0.380471 bw: 22.4889 -[2:1] time: 0.320246 bw: 26.7182 -[2:2] time: 0.321058 bw: 26.6506 -[2:3] time: 0.320511 bw: 26.6961 -[2:4] time: 0.320212 bw: 26.721 -[3:0] time: 0.385449 bw: 22.1985 -[3:1] time: 0.320728 bw: 26.678 -[3:2] time: 0.320348 bw: 26.7096 -[3:3] time: 0.320683 bw: 26.6817 -[3:4] time: 0.320752 bw: 26.676 -[4:0] time: 0.379422 bw: 22.5511 -[4:1] time: 0.321257 bw: 26.6341 -[4:2] time: 0.320537 bw: 26.6939 -[4:3] time: 0.320007 bw: 26.7381 -[4:4] time: 0.321417 bw: 26.6208 -[5:0] time: 0.390293 bw: 21.9229 -[5:1] time: 0.320165 bw: 26.7249 -[5:2] time: 0.320817 bw: 26.6706 -[5:3] time: 0.320363 bw: 26.7084 -[5:4] time: 0.32027 bw: 26.7161 -[6:0] time: 0.396585 bw: 21.5752 -[6:1] time: 0.321708 bw: 26.5967 -[6:2] time: 0.321434 bw: 26.6194 -[6:3] time: 0.322314 bw: 26.5467 -[6:4] time: 0.320983 bw: 26.6568 -[7:0] time: 0.389458 bw: 21.97 -[7:1] time: 0.320854 bw: 26.6675 -[7:2] time: 0.321167 bw: 26.6415 -[7:3] time: 0.321532 bw: 26.6113 -[7:4] time: 0.320994 bw: 26.6559 -[8:0] time: 0.40034 bw: 21.3728 -[8:1] time: 0.321736 bw: 26.5944 -[8:2] time: 0.321058 bw: 26.6506 -[8:3] time: 0.321299 bw: 26.6305 -[8:4] time: 0.321324 bw: 26.6285 -[9:0] time: 0.3882 bw: 22.0412 -[9:1] time: 0.321203 bw: 26.6385 -[9:2] time: 0.321865 bw: 26.5838 -[9:3] time: 0.321176 bw: 26.6408 -[9:4] time: 0.321038 bw: 26.6522 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 0.629292 +[0:0] time: 0.343068 bw: 24.9407 +[0:1] time: 0.306238 bw: 27.9403 +[0:2] time: 0.330506 bw: 25.8887 +[0:3] time: 0.342091 bw: 25.012 +[0:4] time: 0.347334 bw: 24.6344 +[1:0] time: 0.359635 bw: 23.7918 +[1:1] time: 0.304368 bw: 28.1119 +[1:2] time: 0.303222 bw: 28.2182 +[1:3] time: 0.303302 bw: 28.2107 +[1:4] time: 0.303182 bw: 28.2219 +[2:0] time: 0.362862 bw: 23.5802 +[2:1] time: 0.302763 bw: 28.261 +[2:2] time: 0.303479 bw: 28.1943 +[2:3] time: 0.302769 bw: 28.2604 +[2:4] time: 0.30259 bw: 28.2771 +[3:0] time: 0.363585 bw: 23.5334 +[3:1] time: 0.302377 bw: 28.297 +[3:2] time: 0.302523 bw: 28.2834 +[3:3] time: 0.302911 bw: 28.2472 +[3:4] time: 0.303295 bw: 28.2114 +[4:0] time: 0.363342 bw: 23.5491 +[4:1] time: 0.302425 bw: 28.2926 +[4:2] time: 0.302562 bw: 28.2797 +[4:3] time: 0.30201 bw: 28.3315 +[4:4] time: 0.302262 bw: 28.3078 +[5:0] time: 0.366462 bw: 23.3486 +[5:1] time: 0.302339 bw: 28.3006 +[5:2] time: 0.302363 bw: 28.2983 +[5:3] time: 0.302513 bw: 28.2843 +[5:4] time: 0.301986 bw: 28.3337 +[6:0] time: 0.362855 bw: 23.5807 +[6:1] time: 0.302757 bw: 28.2616 +[6:2] time: 0.307936 bw: 27.7862 +[6:3] time: 0.302593 bw: 28.2768 +[6:4] time: 0.302666 bw: 28.2701 +[7:0] time: 0.367087 bw: 23.3089 +[7:1] time: 0.302402 bw: 28.2947 +[7:2] time: 0.302171 bw: 28.3163 +[7:3] time: 0.301926 bw: 28.3393 +[7:4] time: 0.30204 bw: 28.3286 +[8:0] time: 0.365266 bw: 23.4251 +[8:1] time: 0.301972 bw: 28.335 +[8:2] time: 0.302197 bw: 28.3139 +[8:3] time: 0.302428 bw: 28.2923 +[8:4] time: 0.302369 bw: 28.2978 +[9:0] time: 0.36137 bw: 23.6776 +[9:1] time: 0.302814 bw: 28.2562 +[9:2] time: 0.302426 bw: 28.2925 +[9:3] time: 0.301832 bw: 28.3481 +[9:4] time: 0.302473 bw: 28.2881 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.323396 -time0: 0.386355 -bw: 26.4702 -bw0: 22.1558 +time: 0.30561 +time0: 0.361553 +bw: 28.0249 +bw0: 23.6736 ======================================= -Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002684 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002588 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002656 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002708 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002712 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002550 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002612 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002552 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002654 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002636 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002588 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002676 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002714 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002652 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002588 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002656 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002646 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002684 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002552 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002594 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002636 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002678 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002708 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002588 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002656 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002612 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002684 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002626 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002654 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002656 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002714 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002708 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002684 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002552 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002636 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002676 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002664 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002722 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002670 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002636 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002708 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002552 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002594 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002738 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002612 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002654 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002550 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002634 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002550 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002720 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002714 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002652 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002610 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002730 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[002550,002552,002554,002570,002588,002590,002592,002594,002600,002602,002606,002608,002610,002612,002614,002620,002622,002624,002626,002632,002634,002636,002638,002640,002642,002644,002646,002652,002654,002656,002658,002664,002666,002668,002670,002674,002676,002678,002680,002682,002684,002686,002688,002690,002696,002698,002700,002702,002704,002706,002708,002710,002712,002714,002716,002718,002720,002722,002728,002730,002732,002734,002736,002738] +Hostname: nid002714 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002550 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002674 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002612 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002624 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002654 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002688 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002646 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002594 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002644 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002676 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002646 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002570 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002626 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002686 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002592 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002594 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002646 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002642 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002652 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002676 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002670 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002622 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002730 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002716 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002670 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002670 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002730 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002678 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002652 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002610 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002730 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002710 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002728 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002678 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002610 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002680 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002680 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002626 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002626 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002720 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002732 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002610 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002712 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002678 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002698 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002712 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002624 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002624 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002674 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002720 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002712 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002720 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002680 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002624 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002674 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002620 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002674 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002590 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002680 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002634 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002608 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002702 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002732 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002710 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002732 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002570 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002716 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002738 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002738 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002716 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002716 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002634 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002710 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002644 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002710 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002718 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002642 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002722 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002732 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002688 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002686 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002644 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002722 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002706 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002686 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002688 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002688 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002722 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002642 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002554 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002644 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002690 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002686 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002592 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002736 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002622 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002690 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002658 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002690 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002600 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002554 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002554 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002592 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002698 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002642 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002622 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002664 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002600 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002728 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002570 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002592 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002554 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002570 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002664 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002728 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002728 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002718 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002664 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002736 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002600 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002698 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002696 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002658 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002698 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002704 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002600 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002666 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002702 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002736 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002658 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002736 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002704 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002590 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002658 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002620 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002608 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002590 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002590 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002608 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002608 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002606 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002638 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002620 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002620 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002700 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002682 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002606 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002668 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002638 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002606 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002638 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002606 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002706 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002638 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002682 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002706 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002734 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002696 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002706 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002682 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002696 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002632 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002682 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002668 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002700 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002668 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002696 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002668 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002700 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002700 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002640 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002602 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002632 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002632 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002632 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002614 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002602 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002602 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002602 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002614 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002614 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002614 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002640 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002640 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002640 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002734 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002734 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002734 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.5765 -[0:0] time: 15.5273 bw: 0.551054 -[0:1] time: 0.677222 bw: 12.6345 -[0:2] time: 0.613842 bw: 13.9391 -[0:3] time: 0.596115 bw: 14.3536 -[0:4] time: 0.596758 bw: 14.3381 -[1:0] time: 15.5667 bw: 0.549658 -[1:1] time: 0.594328 bw: 14.3967 -[1:2] time: 0.595266 bw: 14.3741 -[1:3] time: 0.596687 bw: 14.3398 -[1:4] time: 0.59707 bw: 14.3306 -[2:0] time: 15.5455 bw: 0.550409 -[2:1] time: 0.595309 bw: 14.373 -[2:2] time: 0.595669 bw: 14.3643 -[2:3] time: 0.595673 bw: 14.3642 -[2:4] time: 0.594826 bw: 14.3847 -[3:0] time: 15.5411 bw: 0.550565 -[3:1] time: 0.595802 bw: 14.3611 -[3:2] time: 0.596905 bw: 14.3346 -[3:3] time: 0.59564 bw: 14.365 -[3:4] time: 0.595908 bw: 14.3586 -[4:0] time: 15.549 bw: 0.550284 -[4:1] time: 0.596075 bw: 14.3545 -[4:2] time: 0.595597 bw: 14.366 -[4:3] time: 0.595974 bw: 14.357 -[4:4] time: 0.595882 bw: 14.3592 -[5:0] time: 15.4998 bw: 0.552032 -[5:1] time: 0.595161 bw: 14.3766 -[5:2] time: 0.595141 bw: 14.3771 -[5:3] time: 0.595226 bw: 14.375 -[5:4] time: 0.59577 bw: 14.3619 -[6:0] time: 15.6076 bw: 0.548219 -[6:1] time: 0.596822 bw: 14.3366 -[6:2] time: 0.596615 bw: 14.3415 -[6:3] time: 0.597674 bw: 14.3161 -[6:4] time: 0.597416 bw: 14.3223 -[7:0] time: 15.562 bw: 0.549824 -[7:1] time: 0.596654 bw: 14.3406 -[7:2] time: 0.596942 bw: 14.3337 -[7:3] time: 0.597281 bw: 14.3256 -[7:4] time: 0.597715 bw: 14.3152 -[8:0] time: 15.5199 bw: 0.551316 -[8:1] time: 0.596257 bw: 14.3502 -[8:2] time: 0.596307 bw: 14.3489 -[8:3] time: 0.597144 bw: 14.3288 -[8:4] time: 0.597151 bw: 14.3287 -[9:0] time: 15.5165 bw: 0.551436 -[9:1] time: 0.595995 bw: 14.3565 -[9:2] time: 0.596341 bw: 14.3481 -[9:3] time: 0.596625 bw: 14.3413 -[9:4] time: 0.597202 bw: 14.3275 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 15.506 +[0:0] time: 15.4896 bw: 0.552395 +[0:1] time: 0.664419 bw: 12.878 +[0:2] time: 0.617138 bw: 13.8646 +[0:3] time: 0.589032 bw: 14.5262 +[0:4] time: 0.5898 bw: 14.5073 +[1:0] time: 15.513 bw: 0.551563 +[1:1] time: 0.589582 bw: 14.5126 +[1:2] time: 0.590242 bw: 14.4964 +[1:3] time: 0.590348 bw: 14.4938 +[1:4] time: 0.591769 bw: 14.459 +[2:0] time: 15.5677 bw: 0.549626 +[2:1] time: 0.589799 bw: 14.5073 +[2:2] time: 0.58956 bw: 14.5132 +[2:3] time: 0.59054 bw: 14.4891 +[2:4] time: 0.590858 bw: 14.4813 +[3:0] time: 15.4788 bw: 0.552781 +[3:1] time: 0.589724 bw: 14.5091 +[3:2] time: 0.589768 bw: 14.508 +[3:3] time: 0.589949 bw: 14.5036 +[3:4] time: 0.59063 bw: 14.4869 +[4:0] time: 15.5471 bw: 0.550351 +[4:1] time: 0.58923 bw: 14.5213 +[4:2] time: 0.589634 bw: 14.5113 +[4:3] time: 0.59015 bw: 14.4986 +[4:4] time: 0.591675 bw: 14.4613 +[5:0] time: 15.5189 bw: 0.551352 +[5:1] time: 0.588624 bw: 14.5362 +[5:2] time: 0.590101 bw: 14.4999 +[5:3] time: 0.58987 bw: 14.5055 +[5:4] time: 0.590494 bw: 14.4902 +[6:0] time: 15.5008 bw: 0.551995 +[6:1] time: 0.589181 bw: 14.5225 +[6:2] time: 0.589586 bw: 14.5125 +[6:3] time: 0.589916 bw: 14.5044 +[6:4] time: 0.590154 bw: 14.4986 +[7:0] time: 15.4775 bw: 0.552827 +[7:1] time: 0.589147 bw: 14.5233 +[7:2] time: 0.589739 bw: 14.5088 +[7:3] time: 0.590218 bw: 14.497 +[7:4] time: 0.590571 bw: 14.4883 +[8:0] time: 15.4629 bw: 0.553348 +[8:1] time: 0.589731 bw: 14.509 +[8:2] time: 0.589509 bw: 14.5144 +[8:3] time: 0.589444 bw: 14.516 +[8:4] time: 0.589941 bw: 14.5038 +[9:0] time: 15.5477 bw: 0.550332 +[9:1] time: 0.589456 bw: 14.5157 +[9:2] time: 0.59001 bw: 14.5021 +[9:3] time: 0.589575 bw: 14.5128 +[9:4] time: 0.590001 bw: 14.5023 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.5987 -time0: 15.5436 -bw: 14.2975 -bw0: 0.55048 -======================================= - -Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 1.17638 -[0:0] time: 1.03872 bw: 8.23741 -[0:1] time: 1.03853 bw: 8.23894 -[0:2] time: 1.0385 bw: 8.23916 -[0:3] time: 1.03861 bw: 8.23833 -[0:4] time: 1.03853 bw: 8.23893 -[1:0] time: 1.0396 bw: 8.23044 -[1:1] time: 1.03876 bw: 8.2371 -[1:2] time: 1.03882 bw: 8.23666 -[1:3] time: 1.03875 bw: 8.2372 -[1:4] time: 1.03879 bw: 8.23691 -[2:0] time: 1.04504 bw: 8.18763 -[2:1] time: 1.0386 bw: 8.23838 -[2:2] time: 1.03858 bw: 8.23851 -[2:3] time: 1.03857 bw: 8.23864 -[2:4] time: 1.03844 bw: 8.23965 -[3:0] time: 1.04059 bw: 8.2226 -[3:1] time: 1.03876 bw: 8.23713 -[3:2] time: 1.03871 bw: 8.23748 -[3:3] time: 1.03871 bw: 8.23749 -[3:4] time: 1.03872 bw: 8.23745 -[4:0] time: 1.04096 bw: 8.21972 -[4:1] time: 1.03857 bw: 8.23862 -[4:2] time: 1.03851 bw: 8.23913 -[4:3] time: 1.03857 bw: 8.23862 -[4:4] time: 1.03855 bw: 8.23876 -[5:0] time: 1.03913 bw: 8.2342 -[5:1] time: 1.03876 bw: 8.23713 -[5:2] time: 1.03879 bw: 8.2369 -[5:3] time: 1.03878 bw: 8.23697 -[5:4] time: 1.03875 bw: 8.23722 -[6:0] time: 1.04029 bw: 8.225 -[6:1] time: 1.03852 bw: 8.23898 -[6:2] time: 1.03856 bw: 8.23868 -[6:3] time: 1.03856 bw: 8.23872 -[6:4] time: 1.0385 bw: 8.23921 -[7:0] time: 1.03946 bw: 8.23158 -[7:1] time: 1.03874 bw: 8.23726 -[7:2] time: 1.03872 bw: 8.23742 -[7:3] time: 1.03871 bw: 8.23747 -[7:4] time: 1.03871 bw: 8.23751 -[8:0] time: 1.03942 bw: 8.23188 -[8:1] time: 1.03853 bw: 8.2389 -[8:2] time: 1.03871 bw: 8.23754 -[8:3] time: 1.03858 bw: 8.23852 -[8:4] time: 1.03855 bw: 8.23879 -[9:0] time: 1.04101 bw: 8.21927 -[9:1] time: 1.03871 bw: 8.23753 -[9:2] time: 1.03874 bw: 8.23724 -[9:3] time: 1.03877 bw: 8.23701 -[9:4] time: 1.0387 bw: 8.2376 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 1.03865 -time0: 1.04042 -bw: 8.23799 -bw0: 8.22397 -======================================= - -Hostname: nid005506 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid004472 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004480 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005522 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004536 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005520 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004508 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004492 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004478 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004512 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004492 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005506 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005522 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005522 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005558 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004542 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004482 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004534 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004478 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004526 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004520 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004546 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005552 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005590 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005524 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004476 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005584 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005496 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 1.19437 -[0:0] time: 1.07402 bw: 7.96667 -[0:1] time: 1.07295 bw: 7.9746 -[0:2] time: 1.07289 bw: 7.97504 -[0:3] time: 1.07288 bw: 7.97513 -[0:4] time: 1.07288 bw: 7.97513 -[1:0] time: 1.07829 bw: 7.93513 -[1:1] time: 1.07365 bw: 7.96942 -[1:2] time: 1.07365 bw: 7.96943 -[1:3] time: 1.07349 bw: 7.97065 -[1:4] time: 1.07351 bw: 7.97049 -[2:0] time: 1.08201 bw: 7.90789 -[2:1] time: 1.07335 bw: 7.97165 -[2:2] time: 1.0735 bw: 7.97052 -[2:3] time: 1.07357 bw: 7.97 -[2:4] time: 1.07354 bw: 7.97023 -[3:0] time: 1.0841 bw: 7.8926 -[3:1] time: 1.07353 bw: 7.97035 -[3:2] time: 1.0736 bw: 7.9698 -[3:3] time: 1.07356 bw: 7.97008 -[3:4] time: 1.07355 bw: 7.97015 -[4:0] time: 1.07738 bw: 7.9418 -[4:1] time: 1.07298 bw: 7.9744 -[4:2] time: 1.07301 bw: 7.97422 -[4:3] time: 1.0729 bw: 7.97497 -[4:4] time: 1.07297 bw: 7.97452 -[5:0] time: 1.07988 bw: 7.92346 -[5:1] time: 1.07352 bw: 7.97037 -[5:2] time: 1.07358 bw: 7.96992 -[5:3] time: 1.07359 bw: 7.96991 -[5:4] time: 1.07364 bw: 7.9695 -[6:0] time: 1.07924 bw: 7.92814 -[6:1] time: 1.07295 bw: 7.97461 -[6:2] time: 1.07296 bw: 7.97458 -[6:3] time: 1.07295 bw: 7.97466 -[6:4] time: 1.073 bw: 7.97424 -[7:0] time: 1.07403 bw: 7.96665 -[7:1] time: 1.07363 bw: 7.96958 -[7:2] time: 1.07363 bw: 7.96956 -[7:3] time: 1.0737 bw: 7.96909 -[7:4] time: 1.07355 bw: 7.97019 -[8:0] time: 1.07536 bw: 7.95675 -[8:1] time: 1.07301 bw: 7.97421 -[8:2] time: 1.07305 bw: 7.9739 -[8:3] time: 1.073 bw: 7.97426 -[8:4] time: 1.07301 bw: 7.97422 -[9:0] time: 1.074 bw: 7.96687 -[9:1] time: 1.07362 bw: 7.96969 -[9:2] time: 1.07359 bw: 7.96988 -[9:3] time: 1.07352 bw: 7.97043 -[9:4] time: 1.07356 bw: 7.97006 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 1.07333 -time0: 1.07783 -bw: 7.97184 -bw0: 7.9386 +time: 0.592478 +time0: 15.5104 +bw: 14.4473 +bw0: 0.551657 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out index eeee8d8..e653e9f 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00256_N_0256_TPN_1.out @@ -1,1303 +1,1165 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001146 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001026 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001106 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001020 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004576 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001066 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001964 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001130 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001144 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001150 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001788 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004506 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001128 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001048 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001018 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001022 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001084 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001888 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001168 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001082 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001870 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001954 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001960 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001006 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001120 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001076 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001890 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001804 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001878 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001896 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001910 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001112 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001776 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001072 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002018 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001906 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001918 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001148 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001836 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001052 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001778 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001984 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002000 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001852 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002016 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001872 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001770 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001108 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001138 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004562 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001064 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001046 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001110 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001016 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001806 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001996 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002010 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001142 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001940 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001914 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001900 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001884 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004344 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004574 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001908 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001838 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001816 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001950 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001864 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004398 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001968 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001952 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001946 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001934 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001056 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004354 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001102 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001930 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004540 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004414 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001972 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001834 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001164 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001944 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004386 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004372 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001932 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001784 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004342 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001874 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001966 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001818 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001868 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001970 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001912 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004566 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001942 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002014 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004442 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001162 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001004 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001948 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001772 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001790 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004552 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004332 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001922 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004362 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001844 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004400 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001160 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001080 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001820 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001846 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001928 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001794 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004336 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004558 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001974 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004572 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002006 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001876 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001122 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004334 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001850 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001810 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001916 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001186 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004444 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004416 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001886 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001986 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004340 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001152 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004348 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001998 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001822 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004448 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004376 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001980 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004410 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001774 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004346 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001090 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004568 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004406 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001036 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001170 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001098 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001936 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001058 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001118 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001086 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001068 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004570 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004514 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001078 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001180 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004524 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001992 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001812 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004560 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004404 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001800 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001854 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001140 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001116 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004556 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001054 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001088 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001184 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001978 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001920 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002002 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001074 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002008 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004564 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001902 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001982 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001994 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001178 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001856 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004554 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001898 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001976 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001782 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001802 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001136 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001814 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002012 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001792 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001882 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001114 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004368 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004366 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001132 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004408 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001808 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001104 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001002,001004,001006,001016,001018,001020,001022,001026,001036,001046,001048,001052,001054,001056,001058,001064,001066,001068,001072,001074,001076,001078,001080,001082,001084,001086,001088,001090,001098,001102,001104,001106,001108,001110,001112,001114,001116,001118,001120,001122,001128,001130,001132,001136,001138,001140,001142,001144,001146,001148,001150,001152,001160,001162,001164,001168,001170,001178,001180,001184,001186,001770,001772,001774,001776,001778,001782,001784,001788,001790,001792,001794,001800,001802,001804,001806,001808,001810,001812,001814,001816,001818,001820,001822,001834,001836,001838,001844,001846,001850,001852,001854,001856,001864,001868,001870,001872,001874,001876,001878,001882,001884,001886,001888,001890,001896,001898,001900,001902,001906,001908,001910,001912,001914,001916,001918,001920,001922,001928,001930,001932,001934,001936,001938,001940,001942,001944,001946,001948,001950,001952,001954,001960,001964,001966,001968,001970,001972,001974,001976,001978,001980,001982,001984,001986,001992,001994,001996,001998,002000,002002,002006,002008,002010,002012,002014,002016,002018,004328,004332,004334,004336,004340,004342,004344,004346,004348,004350,004352,004354,004360,004362,004366,004368,004372,004374,004376,004380,004382,004384,004386,004392,004394,004398,004400,004402,004404,004406,004408,004410,004412,004414,004416,004424,004426,004438,004440,004442,004444,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004506,004508,004510,004512,004514,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004540,004542,004544,004546,004552,004554,004556,004558,004560,004562,004564,004566,004568,004570,004572,004574,004576] +Hostname: nid001002 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.796547 -[0:0] time: 0.627596 bw: 13.6336 -[0:1] time: 0.613907 bw: 13.9376 -[0:2] time: 0.587998 bw: 14.5517 -[0:3] time: 0.576302 bw: 14.8471 -[0:4] time: 0.568177 bw: 15.0594 -[1:0] time: 0.600988 bw: 14.2372 -[1:1] time: 0.56367 bw: 15.1798 -[1:2] time: 0.562442 bw: 15.2129 -[1:3] time: 0.56351 bw: 15.1841 -[1:4] time: 0.562439 bw: 15.213 -[2:0] time: 0.600875 bw: 14.2399 -[2:1] time: 0.562034 bw: 15.2239 -[2:2] time: 0.561991 bw: 15.2251 -[2:3] time: 0.562089 bw: 15.2225 -[2:4] time: 0.561632 bw: 15.2348 -[3:0] time: 0.60018 bw: 14.2564 -[3:1] time: 0.561366 bw: 15.2421 -[3:2] time: 0.562622 bw: 15.208 -[3:3] time: 0.560694 bw: 15.2603 -[3:4] time: 0.561623 bw: 15.2351 -[4:0] time: 0.600334 bw: 14.2527 -[4:1] time: 0.560786 bw: 15.2578 -[4:2] time: 0.56182 bw: 15.2298 -[4:3] time: 0.561802 bw: 15.2302 -[4:4] time: 0.566383 bw: 15.1071 -[5:0] time: 0.603872 bw: 14.1692 -[5:1] time: 0.565397 bw: 15.1334 -[5:2] time: 0.562761 bw: 15.2043 -[5:3] time: 0.561596 bw: 15.2358 -[5:4] time: 0.56222 bw: 15.2189 -[6:0] time: 0.602409 bw: 14.2036 -[6:1] time: 0.563276 bw: 15.1904 -[6:2] time: 0.562935 bw: 15.1996 -[6:3] time: 0.562979 bw: 15.1984 -[6:4] time: 0.563318 bw: 15.1893 -[7:0] time: 0.600828 bw: 14.241 -[7:1] time: 0.562907 bw: 15.2003 -[7:2] time: 0.561572 bw: 15.2365 -[7:3] time: 0.563703 bw: 15.1789 -[7:4] time: 0.561663 bw: 15.234 -[8:0] time: 0.602416 bw: 14.2035 -[8:1] time: 0.562786 bw: 15.2036 -[8:2] time: 0.561753 bw: 15.2316 -[8:3] time: 0.566742 bw: 15.0975 -[8:4] time: 0.564751 bw: 15.1507 -[9:0] time: 0.605774 bw: 14.1247 -[9:1] time: 0.563437 bw: 15.186 -[9:2] time: 0.562667 bw: 15.2068 -[9:3] time: 0.562367 bw: 15.2149 -[9:4] time: 0.56169 bw: 15.2333 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 0.689599 +[0:0] time: 0.641421 bw: 13.3397 +[0:1] time: 0.614424 bw: 13.9259 +[0:2] time: 0.602054 bw: 14.212 +[0:3] time: 0.57626 bw: 14.8481 +[0:4] time: 0.570183 bw: 15.0064 +[1:0] time: 0.60357 bw: 14.1763 +[1:1] time: 0.565615 bw: 15.1276 +[1:2] time: 0.566076 bw: 15.1153 +[1:3] time: 0.566115 bw: 15.1142 +[1:4] time: 0.566215 bw: 15.1115 +[2:0] time: 0.602624 bw: 14.1985 +[2:1] time: 0.564286 bw: 15.1632 +[2:2] time: 0.564139 bw: 15.1672 +[2:3] time: 0.563686 bw: 15.1793 +[2:4] time: 0.564273 bw: 15.1636 +[3:0] time: 0.604788 bw: 14.1477 +[3:1] time: 0.564362 bw: 15.1612 +[3:2] time: 0.563687 bw: 15.1793 +[3:3] time: 0.56427 bw: 15.1636 +[3:4] time: 0.5668 bw: 15.0959 +[4:0] time: 0.605852 bw: 14.1229 +[4:1] time: 0.565324 bw: 15.1354 +[4:2] time: 0.564147 bw: 15.1669 +[4:3] time: 0.564231 bw: 15.1647 +[4:4] time: 0.563823 bw: 15.1757 +[5:0] time: 0.606908 bw: 14.0983 +[5:1] time: 0.564477 bw: 15.1581 +[5:2] time: 0.564047 bw: 15.1696 +[5:3] time: 0.564132 bw: 15.1673 +[5:4] time: 0.564214 bw: 15.1651 +[6:0] time: 0.617149 bw: 13.8644 +[6:1] time: 0.564021 bw: 15.1703 +[6:2] time: 0.563874 bw: 15.1743 +[6:3] time: 0.564432 bw: 15.1593 +[6:4] time: 0.563467 bw: 15.1852 +[7:0] time: 0.619596 bw: 13.8096 +[7:1] time: 0.563705 bw: 15.1788 +[7:2] time: 0.564454 bw: 15.1587 +[7:3] time: 0.564794 bw: 15.1496 +[7:4] time: 0.563878 bw: 15.1742 +[8:0] time: 0.620716 bw: 13.7847 +[8:1] time: 0.56426 bw: 15.1639 +[8:2] time: 0.564667 bw: 15.153 +[8:3] time: 0.564384 bw: 15.1606 +[8:4] time: 0.564357 bw: 15.1613 +[9:0] time: 0.618582 bw: 13.8323 +[9:1] time: 0.564869 bw: 15.1476 +[9:2] time: 0.564796 bw: 15.1495 +[9:3] time: 0.564147 bw: 15.1669 +[9:4] time: 0.564584 bw: 15.1552 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.565095 -time0: 0.604527 -bw: 15.1452 -bw0: 14.1562 +time: 0.567138 +time0: 0.614121 +bw: 15.0911 +bw0: 13.9374 ======================================= -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004576 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001140 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001020 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001026 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001036 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001128 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004502 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001960 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001054 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001132 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001084 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001800 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001106 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001004 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004504 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001076 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001150 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001110 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001910 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004366 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001940 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001144 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001114 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001052 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004488 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001116 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001872 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001118 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004520 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004544 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001088 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001006 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001058 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001876 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001120 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004478 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001968 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002010 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001090 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001896 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001112 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001066 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004496 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001846 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001102 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001082 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004568 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001104 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001932 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001056 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001136 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004474 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001784 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001888 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004424 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001108 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001922 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004398 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004538 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001098 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004526 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001142 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004566 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004372 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001944 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001168 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001854 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001788 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001908 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004574 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001816 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001838 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001864 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001878 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001950 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002012 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001928 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001776 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001984 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001930 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002000 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004528 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004350 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004506 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001900 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001162 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001794 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004514 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001986 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001086 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001170 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001138 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001122 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001806 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004346 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001048 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001912 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004542 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001164 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004404 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002002 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004570 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004416 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002016 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001770 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001906 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004406 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001934 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001836 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001992 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001802 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004448 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004562 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001844 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004494 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004508 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001046 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001774 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001818 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001180 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001982 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001952 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001160 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001184 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001810 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001850 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001972 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004342 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001918 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001820 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004500 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001914 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001870 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001946 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004344 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004534 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004546 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004524 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001790 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001902 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004340 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004490 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004446 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004462 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004532 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004464 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001936 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004328 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001994 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001966 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001948 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004564 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004332 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004450 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001996 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004480 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004456 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004334 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004400 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001998 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001068 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001874 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004354 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004392 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001814 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004522 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004376 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001856 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004348 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004554 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004440 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004540 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001822 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002014 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001942 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004512 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004442 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004482 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004458 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001782 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004460 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001074 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004414 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004530 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001146 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001064 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004466 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001980 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001178 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001920 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004492 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001804 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001772 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001890 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001882 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004382 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004336 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001130 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001022 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004360 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001186 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002006 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004426 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004408 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004468 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004556 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004560 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004470 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004444 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001954 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001898 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004352 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001834 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004472 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001974 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004536 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004402 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001976 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002008 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001964 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001078 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004572 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001018 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002018 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001808 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004384 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004412 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001852 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001886 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004476 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004394 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004410 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001884 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001792 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004362 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004374 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001072 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001016 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004438 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004368 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001148 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001868 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001970 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001812 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004510 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004558 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004380 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001778 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001080 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001916 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001152 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004386 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004552 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001978 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[001002,001004,001006,001016,001018,001020,001022,001026,001036,001046,001048,001052,001054,001056,001058,001064,001066,001068,001072,001074,001076,001078,001080,001082,001084,001086,001088,001090,001098,001102,001104,001106,001108,001110,001112,001114,001116,001118,001120,001122,001128,001130,001132,001136,001138,001140,001142,001144,001146,001148,001150,001152,001160,001162,001164,001168,001170,001178,001180,001184,001186,001770,001772,001774,001776,001778,001782,001784,001788,001790,001792,001794,001800,001802,001804,001806,001808,001810,001812,001814,001816,001818,001820,001822,001834,001836,001838,001844,001846,001850,001852,001854,001856,001864,001868,001870,001872,001874,001876,001878,001882,001884,001886,001888,001890,001896,001898,001900,001902,001906,001908,001910,001912,001914,001916,001918,001920,001922,001928,001930,001932,001934,001936,001938,001940,001942,001944,001946,001948,001950,001952,001954,001960,001964,001966,001968,001970,001972,001974,001976,001978,001980,001982,001984,001986,001992,001994,001996,001998,002000,002002,002006,002008,002010,002012,002014,002016,002018,004328,004332,004334,004336,004340,004342,004344,004346,004348,004350,004352,004354,004360,004362,004366,004368,004372,004374,004376,004380,004382,004384,004386,004392,004394,004398,004400,004402,004404,004406,004408,004410,004412,004414,004416,004424,004426,004438,004440,004442,004444,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004506,004508,004510,004512,004514,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004540,004542,004544,004546,004552,004554,004556,004558,004560,004562,004564,004566,004568,004570,004572,004574,004576] +Hostname: nid001002 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 1.74898 -[0:0] time: 1.7784 bw: 4.81128 -[0:1] time: 1.73091 bw: 4.94328 -[0:2] time: 1.70511 bw: 5.01809 -[0:3] time: 1.70569 bw: 5.01638 -[0:4] time: 1.70404 bw: 5.02123 -[1:0] time: 1.75407 bw: 4.87803 -[1:1] time: 1.70414 bw: 5.02092 -[1:2] time: 1.70442 bw: 5.02011 -[1:3] time: 1.7041 bw: 5.02104 -[1:4] time: 1.70319 bw: 5.02373 -[2:0] time: 1.7531 bw: 4.88071 -[2:1] time: 1.70602 bw: 5.01539 -[2:2] time: 1.70411 bw: 5.02104 -[2:3] time: 1.70437 bw: 5.02025 -[2:4] time: 1.70437 bw: 5.02026 -[3:0] time: 1.76096 bw: 4.85892 -[3:1] time: 1.70446 bw: 5.02001 -[3:2] time: 1.70352 bw: 5.02275 -[3:3] time: 1.70458 bw: 5.01965 -[3:4] time: 1.70351 bw: 5.02281 -[4:0] time: 1.759 bw: 4.86433 -[4:1] time: 1.70372 bw: 5.02218 -[4:2] time: 1.70359 bw: 5.02254 -[4:3] time: 1.70468 bw: 5.01935 -[4:4] time: 1.70464 bw: 5.01946 -[5:0] time: 1.7659 bw: 4.84535 -[5:1] time: 1.70327 bw: 5.02349 -[5:2] time: 1.70395 bw: 5.0215 -[5:3] time: 1.704 bw: 5.02134 -[5:4] time: 1.70362 bw: 5.02247 -[6:0] time: 1.76219 bw: 4.85554 -[6:1] time: 1.70664 bw: 5.01359 -[6:2] time: 1.70614 bw: 5.01506 -[6:3] time: 1.7062 bw: 5.01489 -[6:4] time: 1.70474 bw: 5.01916 -[7:0] time: 1.76915 bw: 4.83645 -[7:1] time: 1.70669 bw: 5.01344 -[7:2] time: 1.70496 bw: 5.01853 -[7:3] time: 1.70416 bw: 5.02087 -[7:4] time: 1.70457 bw: 5.01968 -[8:0] time: 1.76501 bw: 4.84779 -[8:1] time: 1.70435 bw: 5.02033 -[8:2] time: 1.70506 bw: 5.01824 -[8:3] time: 1.70395 bw: 5.02151 -[8:4] time: 1.70339 bw: 5.02314 -[9:0] time: 1.77053 bw: 4.83268 -[9:1] time: 1.70433 bw: 5.02039 -[9:2] time: 1.7042 bw: 5.02075 -[9:3] time: 1.70535 bw: 5.01738 -[9:4] time: 1.7048 bw: 5.01898 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +Found 1 devices for local rank 0 (1) -> setting device = 0 +[-1] time: 1.7448 +[0:0] time: 1.76525 bw: 4.84711 +[0:1] time: 1.73053 bw: 4.94438 +[0:2] time: 1.70213 bw: 5.02687 +[0:3] time: 1.70401 bw: 5.02132 +[0:4] time: 1.7025 bw: 5.02577 +[1:0] time: 1.74578 bw: 4.90118 +[1:1] time: 1.70444 bw: 5.02006 +[1:2] time: 1.70313 bw: 5.02392 +[1:3] time: 1.70378 bw: 5.02201 +[1:4] time: 1.70467 bw: 5.01936 +[2:0] time: 1.746 bw: 4.90056 +[2:1] time: 1.70456 bw: 5.01971 +[2:2] time: 1.70308 bw: 5.02406 +[2:3] time: 1.70358 bw: 5.02258 +[2:4] time: 1.7047 bw: 5.01929 +[3:0] time: 1.74875 bw: 4.89287 +[3:1] time: 1.70392 bw: 5.02157 +[3:2] time: 1.70465 bw: 5.01943 +[3:3] time: 1.70354 bw: 5.0227 +[3:4] time: 1.70341 bw: 5.02309 +[4:0] time: 1.75157 bw: 4.88496 +[4:1] time: 1.70398 bw: 5.0214 +[4:2] time: 1.7038 bw: 5.02195 +[4:3] time: 1.70481 bw: 5.01897 +[4:4] time: 1.7039 bw: 5.02165 +[5:0] time: 1.75138 bw: 4.8855 +[5:1] time: 1.70376 bw: 5.02206 +[5:2] time: 1.70302 bw: 5.02424 +[5:3] time: 1.70404 bw: 5.02123 +[5:4] time: 1.70545 bw: 5.01707 +[6:0] time: 1.75408 bw: 4.87799 +[6:1] time: 1.70512 bw: 5.01805 +[6:2] time: 1.70537 bw: 5.0173 +[6:3] time: 1.70361 bw: 5.0225 +[6:4] time: 1.70466 bw: 5.01941 +[7:0] time: 1.75448 bw: 4.87688 +[7:1] time: 1.70443 bw: 5.02008 +[7:2] time: 1.70437 bw: 5.02026 +[7:3] time: 1.70591 bw: 5.01574 +[7:4] time: 1.70388 bw: 5.0217 +[8:0] time: 1.7578 bw: 4.86767 +[8:1] time: 1.70522 bw: 5.01776 +[8:2] time: 1.70406 bw: 5.02118 +[8:3] time: 1.70393 bw: 5.02156 +[8:4] time: 1.70488 bw: 5.01876 +[9:0] time: 1.75695 bw: 4.87002 +[9:1] time: 1.70526 bw: 5.01763 +[9:2] time: 1.70558 bw: 5.01671 +[9:3] time: 1.70484 bw: 5.01889 +[9:4] time: 1.70378 bw: 5.022 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 1.70519 -time0: 1.76383 -bw: 5.01788 -bw0: 4.85111 -======================================= - -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 1.13626 -[0:0] time: 1.04262 bw: 8.20661 -[0:1] time: 1.03877 bw: 8.23702 -[0:2] time: 1.03879 bw: 8.23684 -[0:3] time: 1.03879 bw: 8.23687 -[0:4] time: 1.03882 bw: 8.23665 -[1:0] time: 1.03984 bw: 8.22859 -[1:1] time: 1.03908 bw: 8.23461 -[1:2] time: 1.03875 bw: 8.23722 -[1:3] time: 1.03852 bw: 8.23905 -[1:4] time: 1.03843 bw: 8.2397 -[2:0] time: 1.04285 bw: 8.20478 -[2:1] time: 1.03876 bw: 8.2371 -[2:2] time: 1.03884 bw: 8.23648 -[2:3] time: 1.03876 bw: 8.23709 -[2:4] time: 1.03935 bw: 8.23242 -[3:0] time: 1.04611 bw: 8.17922 -[3:1] time: 1.0435 bw: 8.19969 -[3:2] time: 1.04532 bw: 8.18541 -[3:3] time: 1.03846 bw: 8.23952 -[3:4] time: 1.03845 bw: 8.23955 -[4:0] time: 1.03935 bw: 8.23242 -[4:1] time: 1.03872 bw: 8.23742 -[4:2] time: 1.03877 bw: 8.23702 -[4:3] time: 1.03876 bw: 8.23708 -[4:4] time: 1.03876 bw: 8.23715 -[5:0] time: 1.04024 bw: 8.22541 -[5:1] time: 1.03847 bw: 8.23942 -[5:2] time: 1.03879 bw: 8.23687 -[5:3] time: 1.03851 bw: 8.23907 -[5:4] time: 1.03849 bw: 8.23926 -[6:0] time: 1.03951 bw: 8.23114 -[6:1] time: 1.03877 bw: 8.237 -[6:2] time: 1.03874 bw: 8.2373 -[6:3] time: 1.03871 bw: 8.2375 -[6:4] time: 1.03873 bw: 8.23738 -[7:0] time: 1.03913 bw: 8.23421 -[7:1] time: 1.03849 bw: 8.23924 -[7:2] time: 1.03855 bw: 8.23881 -[7:3] time: 1.03853 bw: 8.23897 -[7:4] time: 1.03859 bw: 8.23847 -[8:0] time: 1.04172 bw: 8.21372 -[8:1] time: 1.03873 bw: 8.23733 -[8:2] time: 1.03884 bw: 8.23645 -[8:3] time: 1.03874 bw: 8.23724 -[8:4] time: 1.0387 bw: 8.23757 -[9:0] time: 1.03926 bw: 8.23315 -[9:1] time: 1.03867 bw: 8.23782 -[9:2] time: 1.03842 bw: 8.23979 -[9:3] time: 1.03854 bw: 8.23887 -[9:4] time: 1.03859 bw: 8.23846 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 1.03897 -time0: 1.04106 -bw: 8.23542 -bw0: 8.21893 -======================================= - -Hostname: nid004202 Rank: 39, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 1, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 42, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 34, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 14, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 2, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 87, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 26, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 7, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 115, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 111, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004148 Rank: 27, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 123, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 234, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 38, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 242, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 110, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 46, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 95, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 57, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 255, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 9, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 63, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 105, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 51, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 207, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 21, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 102, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 162, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 90, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004092 Rank: 6, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 19, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 145, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 43, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 175, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 73, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 211, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 62, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 50, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 74, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 154, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 250, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 142, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 11, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 17, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 5, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 215, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 149, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 253, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 174, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 205, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 67, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 157, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 97, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 13, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 158, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 86, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 82, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 173, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 101, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 41, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 198, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 55, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 169, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 151, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 218, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 171, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 129, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004834 Rank: 165, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 69, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 15, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 141, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 135, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 33, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 190, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 130, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 118, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 70, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 238, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 233, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 245, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 89, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 167, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 227, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 71, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 75, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 183, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 199, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 131, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 210, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 126, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 150, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 246, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 222, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 223, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 121, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 241, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 139, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 127, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 31, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 254, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 122, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 134, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 177, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 94, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 178, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 214, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 155, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004824 Rank: 161, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 185, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 107, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 137, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 119, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 225, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 249, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 125, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 251, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 143, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 3, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 37, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 85, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 83, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 77, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 59, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 35, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 193, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 182, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 201, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 195, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 213, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 226, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 221, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 230, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 147, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 197, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 191, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 25, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 79, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 229, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 243, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 99, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004240 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 22, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 206, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 23, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 78, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 30, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 163, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 146, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 189, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 209, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 239, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 219, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 10, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 106, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 186, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 18, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 103, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 133, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 202, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 181, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 194, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 117, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004888 Rank: 187, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 159, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 247, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 113, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 237, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 91, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 45, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 58, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 231, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 47, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 81, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 66, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 203, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 93, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 235, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 153, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 65, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 53, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 217, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 98, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 166, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 49, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 179, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 109, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 61, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 54, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 29, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 138, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 1.19075 -[0:0] time: 1.08147 bw: 7.91177 -[0:1] time: 1.08162 bw: 7.91067 -[0:2] time: 1.07902 bw: 7.92979 -[0:3] time: 1.0727 bw: 7.97651 -[0:4] time: 1.0727 bw: 7.97648 -[1:0] time: 1.07312 bw: 7.97337 -[1:1] time: 1.07286 bw: 7.97531 -[1:2] time: 1.0729 bw: 7.97503 -[1:3] time: 1.07288 bw: 7.97512 -[1:4] time: 1.07288 bw: 7.97511 -[2:0] time: 1.07338 bw: 7.97144 -[2:1] time: 1.07269 bw: 7.97659 -[2:2] time: 1.07263 bw: 7.97701 -[2:3] time: 1.0727 bw: 7.9765 -[2:4] time: 1.07274 bw: 7.97618 -[3:0] time: 1.07323 bw: 7.97257 -[3:1] time: 1.07295 bw: 7.97466 -[3:2] time: 1.07295 bw: 7.97465 -[3:3] time: 1.07309 bw: 7.97356 -[3:4] time: 1.07296 bw: 7.97457 -[4:0] time: 1.07348 bw: 7.97068 -[4:1] time: 1.07279 bw: 7.97585 -[4:2] time: 1.07277 bw: 7.97598 -[4:3] time: 1.07289 bw: 7.97509 -[4:4] time: 1.07263 bw: 7.97704 -[5:0] time: 1.07332 bw: 7.97192 -[5:1] time: 1.07283 bw: 7.97551 -[5:2] time: 1.07319 bw: 7.97284 -[5:3] time: 1.07285 bw: 7.9754 -[5:4] time: 1.07294 bw: 7.97471 -[6:0] time: 1.07295 bw: 7.97461 -[6:1] time: 1.07268 bw: 7.97662 -[6:2] time: 1.07266 bw: 7.97682 -[6:3] time: 1.07273 bw: 7.97627 -[6:4] time: 1.07273 bw: 7.97627 -[7:0] time: 1.07338 bw: 7.97146 -[7:1] time: 1.07301 bw: 7.97415 -[7:2] time: 1.07294 bw: 7.97471 -[7:3] time: 1.07292 bw: 7.97482 -[7:4] time: 1.07289 bw: 7.97511 -[8:0] time: 1.07297 bw: 7.97451 -[8:1] time: 1.07272 bw: 7.97631 -[8:2] time: 1.07273 bw: 7.9763 -[8:3] time: 1.07271 bw: 7.97645 -[8:4] time: 1.07266 bw: 7.97682 -[9:0] time: 1.07311 bw: 7.97341 -[9:1] time: 1.07287 bw: 7.97521 -[9:2] time: 1.07291 bw: 7.97496 -[9:3] time: 1.07291 bw: 7.97495 -[9:4] time: 1.07285 bw: 7.97535 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 1.0732 -time0: 1.07404 -bw: 7.97278 -bw0: 7.96657 +time: 1.70486 +time0: 1.7532 +bw: 5.01886 +bw0: 4.88047 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out index ee92f00..5039f43 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_00512_N_0128_TPN_4.out @@ -1,2327 +1,2189 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001770 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001794 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005000 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001794 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005000 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004858 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001770 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001794 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001794 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001770 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001998 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001888 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Slurm Job Hostlist: nid[001770,001774,001776,001778,001788,001792,001794,001800,001802,001812,001814,001816,001818,001820,001836,001838,001876,001888,001910,001930,001938,001960,001976,001978,001998,002002,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid001770 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001876 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004856 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001802 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001998 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005058 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004854 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001820 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005000 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004858 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004920 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001876 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001978 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005000 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001876 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001888 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005032 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001998 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001876 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001888 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001888 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001930 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001838 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001802 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004846 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001800 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005048 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001802 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005042 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001998 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004858 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001820 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001802 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001818 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004858 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004894 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004898 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001976 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001978 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001812 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001816 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001978 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001976 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004926 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004942 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004946 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001978 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001800 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001976 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001800 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001800 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004956 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004942 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001838 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001976 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004956 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005058 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004894 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001838 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001788 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001930 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004942 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001838 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001820 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001788 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005006 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001938 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001930 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005046 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004942 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005036 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005026 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001788 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001816 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001818 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001788 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001820 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001816 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004862 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001816 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001812 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004946 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001818 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005042 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001938 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001812 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005042 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001910 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004950 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001938 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001812 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004842 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001818 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004940 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001776 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002002 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005048 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005034 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005048 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005048 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001774 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005010 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002002 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002002 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001774 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001776 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001960 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001776 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001776 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002002 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001774 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001774 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001910 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001910 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001910 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004936 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004936 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001960 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004908 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001960 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001960 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001792 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001836 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004958 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001814 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004918 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004890 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004978 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001792 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001792 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001792 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001836 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001836 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001836 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001814 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001814 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001814 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001778 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001778 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004912 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004918 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001778 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001778 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 naive algo: 0 -[-1] time: 0.758111 -[0:0] time: 0.398167 bw: 21.5316 -[0:1] time: 0.395453 bw: 21.6794 -[0:2] time: 0.411099 bw: 20.8542 -[0:3] time: 0.374675 bw: 22.8816 -[0:4] time: 0.355259 bw: 24.1321 -[1:0] time: 0.404274 bw: 21.2063 -[1:1] time: 0.341317 bw: 25.1179 -[1:2] time: 0.340978 bw: 25.1428 -[1:3] time: 0.341165 bw: 25.1291 -[1:4] time: 0.340623 bw: 25.169 -[2:0] time: 0.409561 bw: 20.9325 -[2:1] time: 0.339334 bw: 25.2646 -[2:2] time: 0.339636 bw: 25.2422 -[2:3] time: 0.340022 bw: 25.2135 -[2:4] time: 0.339633 bw: 25.2424 -[3:0] time: 0.407087 bw: 21.0598 -[3:1] time: 0.339931 bw: 25.2203 -[3:2] time: 0.339699 bw: 25.2375 -[3:3] time: 0.339515 bw: 25.2512 -[3:4] time: 0.339692 bw: 25.238 -[4:0] time: 0.406124 bw: 21.1097 -[4:1] time: 0.340403 bw: 25.1853 -[4:2] time: 0.339713 bw: 25.2364 -[4:3] time: 0.339017 bw: 25.2883 -[4:4] time: 0.339863 bw: 25.2254 -[5:0] time: 0.406543 bw: 21.0879 -[5:1] time: 0.340094 bw: 25.2082 -[5:2] time: 0.340103 bw: 25.2076 -[5:3] time: 0.339335 bw: 25.2646 -[5:4] time: 0.33958 bw: 25.2464 -[6:0] time: 0.40838 bw: 20.9931 -[6:1] time: 0.340577 bw: 25.1725 -[6:2] time: 0.341103 bw: 25.1336 -[6:3] time: 0.340724 bw: 25.1616 -[6:4] time: 0.340851 bw: 25.1522 -[7:0] time: 0.413934 bw: 20.7114 -[7:1] time: 0.340765 bw: 25.1586 -[7:2] time: 0.340765 bw: 25.1585 -[7:3] time: 0.340834 bw: 25.1534 -[7:4] time: 0.340645 bw: 25.1675 -[8:0] time: 0.408367 bw: 20.9938 -[8:1] time: 0.340751 bw: 25.1596 -[8:2] time: 0.340564 bw: 25.1734 -[8:3] time: 0.34087 bw: 25.1508 -[8:4] time: 0.340515 bw: 25.177 -[9:0] time: 0.413884 bw: 20.7139 -[9:1] time: 0.340744 bw: 25.1601 -[9:2] time: 0.341396 bw: 25.112 -[9:3] time: 0.340438 bw: 25.1827 -[9:4] time: 0.340947 bw: 25.1451 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 0.757714 +[0:0] time: 0.415158 bw: 20.6504 +[0:1] time: 0.421637 bw: 20.333 +[0:2] time: 0.343978 bw: 24.9236 +[0:3] time: 0.340466 bw: 25.1807 +[0:4] time: 0.338569 bw: 25.3217 +[1:0] time: 0.395436 bw: 21.6803 +[1:1] time: 0.338716 bw: 25.3107 +[1:2] time: 0.33939 bw: 25.2605 +[1:3] time: 0.338953 bw: 25.2931 +[1:4] time: 0.338624 bw: 25.3177 +[2:0] time: 0.403428 bw: 21.2508 +[2:1] time: 0.339949 bw: 25.219 +[2:2] time: 0.337982 bw: 25.3657 +[2:3] time: 0.338529 bw: 25.3248 +[2:4] time: 0.33845 bw: 25.3307 +[3:0] time: 0.410638 bw: 20.8777 +[3:1] time: 0.338658 bw: 25.3151 +[3:2] time: 0.338113 bw: 25.3559 +[3:3] time: 0.337697 bw: 25.3871 +[3:4] time: 0.338033 bw: 25.3619 +[4:0] time: 0.409422 bw: 20.9396 +[4:1] time: 0.33845 bw: 25.3307 +[4:2] time: 0.339748 bw: 25.2339 +[4:3] time: 0.338234 bw: 25.3468 +[4:4] time: 0.338428 bw: 25.3323 +[5:0] time: 0.407895 bw: 21.018 +[5:1] time: 0.338404 bw: 25.3341 +[5:2] time: 0.338409 bw: 25.3337 +[5:3] time: 0.33799 bw: 25.3651 +[5:4] time: 0.338362 bw: 25.3372 +[6:0] time: 0.410857 bw: 20.8665 +[6:1] time: 0.338808 bw: 25.3039 +[6:2] time: 0.337551 bw: 25.3981 +[6:3] time: 0.33823 bw: 25.3471 +[6:4] time: 0.337569 bw: 25.3967 +[7:0] time: 0.408888 bw: 20.967 +[7:1] time: 0.337873 bw: 25.3739 +[7:2] time: 0.33794 bw: 25.3689 +[7:3] time: 0.337946 bw: 25.3684 +[7:4] time: 0.337596 bw: 25.3948 +[8:0] time: 0.407418 bw: 21.0427 +[8:1] time: 0.338435 bw: 25.3317 +[8:2] time: 0.338475 bw: 25.3288 +[8:3] time: 0.336924 bw: 25.4454 +[8:4] time: 0.338588 bw: 25.3204 +[9:0] time: 0.406246 bw: 21.1034 +[9:1] time: 0.338315 bw: 25.3407 +[9:2] time: 0.338174 bw: 25.3513 +[9:3] time: 0.338099 bw: 25.3569 +[9:4] time: 0.338789 bw: 25.3053 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.344716 -time0: 0.407632 -bw: 24.9099 -bw0: 21.034 +time: 0.340627 +time0: 0.407539 +bw: 25.1987 +bw0: 21.0396 ======================================= -Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001794 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001770 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004858 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001770 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001770 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[001770,001774,001776,001778,001788,001792,001794,001800,001802,001812,001814,001816,001818,001820,001836,001838,001876,001888,001910,001930,001938,001960,001976,001978,001998,002002,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid001770 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004840 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001814 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005058 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004858 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001802 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001794 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001794 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001794 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001774 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004858 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001816 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004858 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001888 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004842 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001774 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001978 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004914 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001802 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001802 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001774 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001876 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001802 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004876 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001774 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001888 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004898 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004944 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001888 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001888 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001838 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005000 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005052 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001800 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005000 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001910 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001776 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001816 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004956 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001816 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005052 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001816 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004880 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001814 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001978 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005048 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001978 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004970 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004860 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005058 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004874 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004866 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001800 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001814 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001978 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004988 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001814 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001800 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001800 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005058 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001960 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005002 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 0 - -[-1] time: 15.6412 -[0:0] time: 15.6363 bw: 0.548285 -[0:1] time: 0.623145 bw: 13.7579 -[0:2] time: 0.619911 bw: 13.8297 -[0:3] time: 0.619609 bw: 13.8364 -[0:4] time: 0.617857 bw: 13.8756 -[1:0] time: 15.5781 bw: 0.550333 -[1:1] time: 0.617464 bw: 13.8845 -[1:2] time: 0.621994 bw: 13.7833 -[1:3] time: 0.619007 bw: 13.8499 -[1:4] time: 0.619483 bw: 13.8392 -[2:0] time: 15.5712 bw: 0.550579 -[2:1] time: 0.618117 bw: 13.8698 -[2:2] time: 0.618362 bw: 13.8643 -[2:3] time: 0.618695 bw: 13.8568 -[2:4] time: 0.618004 bw: 13.8723 -[3:0] time: 15.5992 bw: 0.54959 -[3:1] time: 0.617651 bw: 13.8803 -[3:2] time: 0.618077 bw: 13.8707 -[3:3] time: 0.617745 bw: 13.8782 -[3:4] time: 0.618475 bw: 13.8618 -[4:0] time: 15.6196 bw: 0.548873 -[4:1] time: 0.618061 bw: 13.871 -[4:2] time: 0.617581 bw: 13.8818 -[4:3] time: 0.619259 bw: 13.8442 -[4:4] time: 0.61807 bw: 13.8709 -[5:0] time: 15.5974 bw: 0.549653 -[5:1] time: 0.618309 bw: 13.8655 -[5:2] time: 0.61813 bw: 13.8695 -[5:3] time: 0.618115 bw: 13.8699 -[5:4] time: 0.618512 bw: 13.8609 -[6:0] time: 15.5842 bw: 0.55012 -[6:1] time: 0.622479 bw: 13.7726 -[6:2] time: 0.61994 bw: 13.829 -[6:3] time: 0.621177 bw: 13.8015 -[6:4] time: 0.619289 bw: 13.8435 -[7:0] time: 15.6367 bw: 0.548272 -[7:1] time: 0.618717 bw: 13.8564 -[7:2] time: 0.619462 bw: 13.8397 -[7:3] time: 0.619411 bw: 13.8408 -[7:4] time: 0.619985 bw: 13.828 -[8:0] time: 15.6577 bw: 0.547535 -[8:1] time: 0.619204 bw: 13.8454 -[8:2] time: 0.619008 bw: 13.8498 -[8:3] time: 0.619479 bw: 13.8393 -[8:4] time: 0.619244 bw: 13.8446 -[9:0] time: 15.6454 bw: 0.547968 -[9:1] time: 0.61864 bw: 13.8581 -[9:2] time: 0.620228 bw: 13.8226 -[9:3] time: 0.618714 bw: 13.8564 -[9:4] time: 0.619637 bw: 13.8358 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 0 -time: 0.619106 -time0: 15.6126 -bw: 13.8477 -bw0: 0.549121 -======================================= - -Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001820 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004978 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004942 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001976 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001838 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001776 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001976 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004892 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004994 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001976 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001838 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001976 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002002 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001876 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001776 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001776 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001838 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001876 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004920 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005048 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001876 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 2.21199 -[0:0] time: 2.07827 bw: 4.12515 -[0:1] time: 2.07861 bw: 4.12447 -[0:2] time: 2.07795 bw: 4.12578 -[0:3] time: 2.07792 bw: 4.12583 -[0:4] time: 2.07786 bw: 4.12596 -[1:0] time: 2.08027 bw: 4.12118 -[1:1] time: 2.07816 bw: 4.12537 -[1:2] time: 2.07811 bw: 4.12545 -[1:3] time: 2.07815 bw: 4.12537 -[1:4] time: 2.07829 bw: 4.1251 -[2:0] time: 2.07862 bw: 4.12445 -[2:1] time: 2.07792 bw: 4.12584 -[2:2] time: 2.07783 bw: 4.12601 -[2:3] time: 2.07787 bw: 4.12594 -[2:4] time: 2.07823 bw: 4.12522 -[3:0] time: 2.08091 bw: 4.11991 -[3:1] time: 2.07814 bw: 4.1254 -[3:2] time: 2.07815 bw: 4.12537 -[3:3] time: 2.0781 bw: 4.12548 -[3:4] time: 2.07818 bw: 4.12531 -[4:0] time: 2.07997 bw: 4.12177 -[4:1] time: 2.07789 bw: 4.1259 -[4:2] time: 2.07802 bw: 4.12563 -[4:3] time: 2.07809 bw: 4.12551 -[4:4] time: 2.07782 bw: 4.12604 -[5:0] time: 2.08055 bw: 4.12063 -[5:1] time: 2.0782 bw: 4.12528 -[5:2] time: 2.0783 bw: 4.12508 -[5:3] time: 2.07827 bw: 4.12515 -[5:4] time: 2.07832 bw: 4.12505 -[6:0] time: 2.08008 bw: 4.12155 -[6:1] time: 2.07834 bw: 4.12501 -[6:2] time: 2.07784 bw: 4.126 -[6:3] time: 2.07789 bw: 4.1259 -[6:4] time: 2.07786 bw: 4.12596 -[7:0] time: 2.08229 bw: 4.11718 -[7:1] time: 2.07821 bw: 4.12526 -[7:2] time: 2.0782 bw: 4.12529 -[7:3] time: 2.07814 bw: 4.1254 -[7:4] time: 2.07824 bw: 4.1252 -[8:0] time: 2.08106 bw: 4.11962 -[8:1] time: 2.07785 bw: 4.12597 -[8:2] time: 2.07783 bw: 4.12602 -[8:3] time: 2.07795 bw: 4.12579 -[8:4] time: 2.07789 bw: 4.12589 -[9:0] time: 2.08201 bw: 4.11773 -[9:1] time: 2.07825 bw: 4.12518 -[9:2] time: 2.07817 bw: 4.12535 -[9:3] time: 2.07824 bw: 4.12519 -[9:4] time: 2.07817 bw: 4.12535 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 2.07809 -time0: 2.0804 -bw: 4.12551 -bw0: 4.12092 -======================================= - -Hostname: nid004586 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004960 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004638 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004982 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004854 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005032 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005056 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005052 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004880 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005032 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004898 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004638 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004970 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005006 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004586 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004638 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004692 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004586 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004908 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005032 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004586 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004886 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004856 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004692 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004648 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005018 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005032 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004862 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004624 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004948 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004928 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004988 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004624 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004654 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004912 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004682 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004654 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004924 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004840 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004600 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004848 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004970 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004974 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005076 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005054 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004898 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004916 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004600 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004840 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004846 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004888 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004864 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004982 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004888 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005050 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004978 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005038 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004660 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004584 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004896 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004882 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004694 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004920 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004990 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004912 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005040 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004666 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004898 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004842 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005056 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004992 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004888 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004584 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004914 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004970 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005026 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004600 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004914 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005074 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005026 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004912 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004584 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005056 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004910 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004928 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004842 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004584,004586,004588,004592,004598,004600,004602,004604,004606,004608,004610,004616,004618,004622,004624,004628,004630,004634,004638,004648,004650,004654,004656,004658,004660,004662,004664,004666,004674,004682,004684,004686,004688,004690,004692,004694,004698,004700,004840,004842,004844,004846,004848,004850,004852,004854,004856,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004922,004924,004926,004928,004930,004938,004940,004944,004948,004950,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,005002,005004,005006,005008,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005044,005046,005050,005052,005054,005056,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] -Hostname: nid004876 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004584 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004912 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005026 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005068 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004970 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004666 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004648 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004924 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004898 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004928 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004880 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004880 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004914 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004960 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005006 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004688 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004982 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004862 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004928 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004600 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004630 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004862 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004862 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004960 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005006 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005018 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004886 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004648 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004886 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004960 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005016 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004856 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004886 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005074 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004854 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004648 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005018 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005038 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004980 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005038 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004924 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004700 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005006 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004690 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004930 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004926 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004606 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004924 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005038 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004856 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004682 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004992 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004864 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004684 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005074 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004854 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004938 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004616 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004650 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004864 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004660 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004974 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004968 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004938 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004908 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004874 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005034 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004948 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004664 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004936 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004942 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002002 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004874 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005070 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005014 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004942 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004994 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005034 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004928 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002002 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001820 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004942 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004918 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005036 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005084 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005068 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004700 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004698 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004948 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004874 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005076 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004876 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004660 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004972 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004660 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004974 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004938 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005070 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004608 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004878 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004698 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004908 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004864 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004616 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004876 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004948 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005070 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004874 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004698 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005076 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005040 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004616 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005046 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005012 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004972 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004920 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004938 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004974 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004876 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005036 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002002 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004890 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005040 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001910 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001960 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001812 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001910 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001910 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004956 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001788 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004936 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004956 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001820 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004936 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) Hostname: nid005084 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005012 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004852 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004610 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005052 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005040 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004606 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004940 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004628 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004910 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004684 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004916 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005054 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004606 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004976 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004616 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005012 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004608 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004602 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005084 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005050 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005068 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004608 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004846 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004896 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004882 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005050 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004602 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001938 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005084 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004922 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004920 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005050 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004860 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004882 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004608 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004890 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004606 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005052 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004972 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005052 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004896 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004978 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004598 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004972 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004980 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005068 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004882 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004852 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005046 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004978 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004978 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004650 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004980 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004922 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004630 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004872 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004650 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005072 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004844 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004604 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005008 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004688 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004980 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005072 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005044 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005008 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005046 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004976 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004630 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004872 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005072 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004604 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004650 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004990 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004968 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004694 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004958 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004968 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005036 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004930 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004688 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004694 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004852 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004926 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004976 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004968 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005036 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004930 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004872 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004690 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004940 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005004 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005034 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004976 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004844 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004940 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005016 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004926 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004990 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005034 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004926 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004690 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004592 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005016 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004690 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004986 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004892 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004940 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005016 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004664 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004950 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005002 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004610 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004664 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004628 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004588 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004890 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004658 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004890 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004588 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004988 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004598 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004950 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004986 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004988 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004658 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004962 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004958 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004628 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004986 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004598 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004958 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004658 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004662 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004950 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004962 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005002 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004986 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004610 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004950 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005014 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005002 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005002 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004958 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004892 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004662 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005004 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004884 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004850 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004962 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004610 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004884 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004850 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005078 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004662 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004662 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005044 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004592 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004884 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004850 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004592 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005086 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005044 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005004 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005004 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001938 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001820 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005078 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005078 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001960 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001960 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005078 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005014 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005014 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004634 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004634 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005086 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004866 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005086 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004918 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004918 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004904 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005086 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004634 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004918 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005022 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004656 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004866 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004622 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004866 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005022 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004618 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005022 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004622 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004674 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004674 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004618 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004656 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004622 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004674 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005080 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004656 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004618 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004904 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004656 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004904 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004944 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001788 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001792 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001788 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004926 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001788 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001812 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) Hostname: nid005080 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004944 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001812 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001812 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) Hostname: nid005080 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001930 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004946 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) Hostname: nid005080 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004686 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004686 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004686 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004686 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001818 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001778 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001836 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001792 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001778 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005008 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001778 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001930 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004946 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004946 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001778 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001930 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001792 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001792 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001930 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005042 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005042 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001818 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid001818 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001818 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004948 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001836 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001836 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001836 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid001998 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004894 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001998 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid001998 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004894 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004894 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004850 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004912 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004912 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004894 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid001998 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 -naive algo: 1 +naive algo: 0 -[-1] time: 2.26663 -[0:0] time: 2.15454 bw: 3.9791 -[0:1] time: 2.15178 bw: 3.98422 -[0:2] time: 2.15178 bw: 3.98421 -[0:3] time: 2.15186 bw: 3.98408 -[0:4] time: 2.15183 bw: 3.98412 -[1:0] time: 2.15656 bw: 3.97538 -[1:1] time: 2.14897 bw: 3.98943 -[1:2] time: 2.14905 bw: 3.98927 -[1:3] time: 2.14897 bw: 3.98942 -[1:4] time: 2.14898 bw: 3.9894 -[2:0] time: 2.15449 bw: 3.97921 -[2:1] time: 2.15189 bw: 3.98401 -[2:2] time: 2.15179 bw: 3.98419 -[2:3] time: 2.1519 bw: 3.98399 -[2:4] time: 2.15173 bw: 3.98431 -[3:0] time: 2.14985 bw: 3.9878 -[3:1] time: 2.14905 bw: 3.98929 -[3:2] time: 2.14906 bw: 3.98925 -[3:3] time: 2.14908 bw: 3.98922 -[3:4] time: 2.14906 bw: 3.98925 -[4:0] time: 2.15416 bw: 3.97981 -[4:1] time: 2.15193 bw: 3.98393 -[4:2] time: 2.15189 bw: 3.98401 -[4:3] time: 2.1518 bw: 3.98418 -[4:4] time: 2.15174 bw: 3.98429 -[5:0] time: 2.15071 bw: 3.98619 -[5:1] time: 2.14929 bw: 3.98882 -[5:2] time: 2.14917 bw: 3.98906 -[5:3] time: 2.14899 bw: 3.98939 -[5:4] time: 2.14905 bw: 3.98928 -[6:0] time: 2.16012 bw: 3.96883 -[6:1] time: 2.15197 bw: 3.98386 -[6:2] time: 2.1519 bw: 3.98399 -[6:3] time: 2.15198 bw: 3.98385 -[6:4] time: 2.15198 bw: 3.98385 -[7:0] time: 2.15243 bw: 3.98302 -[7:1] time: 2.14897 bw: 3.98943 -[7:2] time: 2.14906 bw: 3.98926 -[7:3] time: 2.14895 bw: 3.98947 -[7:4] time: 2.14898 bw: 3.98941 -[8:0] time: 2.15217 bw: 3.9835 -[8:1] time: 2.15188 bw: 3.98402 -[8:2] time: 2.15189 bw: 3.98401 -[8:3] time: 2.15189 bw: 3.98402 -[8:4] time: 2.15189 bw: 3.98401 -[9:0] time: 2.14922 bw: 3.98896 -[9:1] time: 2.14898 bw: 3.9894 -[9:2] time: 2.1492 bw: 3.98899 -[9:3] time: 2.14915 bw: 3.9891 -[9:4] time: 2.14888 bw: 3.9896 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 15.5808 +[0:0] time: 15.5996 bw: 0.549575 +[0:1] time: 0.612649 bw: 13.9936 +[0:2] time: 0.613283 bw: 13.9791 +[0:3] time: 0.61407 bw: 13.9612 +[0:4] time: 0.613624 bw: 13.9713 +[1:0] time: 15.5339 bw: 0.5519 +[1:1] time: 0.613449 bw: 13.9753 +[1:2] time: 0.614203 bw: 13.9582 +[1:3] time: 0.614104 bw: 13.9604 +[1:4] time: 0.615074 bw: 13.9384 +[2:0] time: 15.5466 bw: 0.551448 +[2:1] time: 0.614461 bw: 13.9523 +[2:2] time: 0.614156 bw: 13.9592 +[2:3] time: 0.614352 bw: 13.9548 +[2:4] time: 0.614598 bw: 13.9492 +[3:0] time: 15.5704 bw: 0.550606 +[3:1] time: 0.61336 bw: 13.9774 +[3:2] time: 0.613507 bw: 13.974 +[3:3] time: 0.61541 bw: 13.9308 +[3:4] time: 0.615011 bw: 13.9398 +[4:0] time: 15.5701 bw: 0.550617 +[4:1] time: 0.613704 bw: 13.9695 +[4:2] time: 0.6137 bw: 13.9696 +[4:3] time: 0.614561 bw: 13.9501 +[4:4] time: 0.614953 bw: 13.9411 +[5:0] time: 15.5785 bw: 0.550318 +[5:1] time: 0.61284 bw: 13.9892 +[5:2] time: 0.613991 bw: 13.963 +[5:3] time: 0.613829 bw: 13.9667 +[5:4] time: 0.614354 bw: 13.9547 +[6:0] time: 15.6148 bw: 0.549041 +[6:1] time: 0.613337 bw: 13.9779 +[6:2] time: 0.613359 bw: 13.9774 +[6:3] time: 0.614241 bw: 13.9573 +[6:4] time: 0.61485 bw: 13.9435 +[7:0] time: 15.5658 bw: 0.550768 +[7:1] time: 0.61316 bw: 13.9819 +[7:2] time: 0.614229 bw: 13.9576 +[7:3] time: 0.612857 bw: 13.9888 +[7:4] time: 0.612332 bw: 14.0008 +[8:0] time: 15.517 bw: 0.552502 +[8:1] time: 0.613036 bw: 13.9848 +[8:2] time: 0.613679 bw: 13.9701 +[8:3] time: 0.614119 bw: 13.9601 +[8:4] time: 0.613725 bw: 13.969 +[9:0] time: 15.5148 bw: 0.552581 +[9:1] time: 0.613133 bw: 13.9825 +[9:2] time: 0.613523 bw: 13.9736 +[9:3] time: 0.614006 bw: 13.9627 +[9:4] time: 0.613999 bw: 13.9628 ======================================= type: device size: 4294967296 pad: 4 -naive: 1 -time: 2.15046 -time0: 2.15343 -bw: 3.98667 -bw0: 3.98118 +naive: 0 +time: 0.613871 +time0: 15.5612 +bw: 13.9658 +bw0: 0.550936 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out b/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out index 6a3aaf4..15a0110 100644 --- a/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out +++ b/allreduce-perf/mpi-cpp/run/data/job_n_01024_N_0256_TPN_4.out @@ -1,4375 +1,4237 @@ All nodes and tasks are healthy. All ranks passed the checks. loading the view test_mpi:default with activation script /user-environment/env/default/activate.sh -Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004770 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002890 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002848 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002962 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004754 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002876 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004706 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005026 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002870 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004654 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002828 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002894 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002906 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002860 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004634 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002952 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004858 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002964 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002966 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002944 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004776 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002904 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002802 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002898 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004594 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005066 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004798 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002828 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004682 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004590 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004716 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004960 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002860 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002892 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002952 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002872 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002974 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002930 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004812 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002874 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002944 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002828 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002890 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002966 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004858 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002890 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004960 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004634 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002860 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004688 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002952 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002848 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002842 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002896 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002890 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002878 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002828 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002848 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002806 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004988 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002966 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002894 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004704 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002860 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002836 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002952 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004798 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004630 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002882 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002944 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002920 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004974 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004834 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004590 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002848 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004634 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002946 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004886 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004706 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002894 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002966 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002960 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002898 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004982 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002926 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002898 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004832 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002894 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004754 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004858 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004706 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002962 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002944 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002898 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002986 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005012 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004716 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004798 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005046 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005008 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004754 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004714 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004788 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002962 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002978 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004706 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004858 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002914 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004754 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004590 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002930 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004636 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002906 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004624 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004716 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004816 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002928 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002876 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004826 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002962 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004798 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002930 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002892 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005026 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004660 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002924 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002906 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002806 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002798 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004970 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004978 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002906 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002876 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002896 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002920 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002892 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002876 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004692 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005054 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002798 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004590 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002896 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004594 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002896 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004980 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004736 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002930 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002870 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002834 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004962 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002806 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004790 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002892 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002842 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002974 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002946 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004630 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002922 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002870 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002798 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002824 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002872 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002806 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002978 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002928 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002914 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004718 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002960 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004654 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004760 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004620 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002842 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002920 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002882 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004942 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004826 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002964 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004872 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002904 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004886 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005084 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004796 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004724 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002834 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004816 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002874 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002910 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004618 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004920 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004834 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004802 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004700 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004594 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004622 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002986 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004882 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004750 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004894 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004990 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004616 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004588 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005042 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004890 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004746 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002878 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005068 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004610 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002974 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002946 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002870 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Slurm Job Hostlist: nid[002798,002800,002802,002806,002808,002812,002824,002828,002834,002836,002838,002842,002844,002848,002850,002856,002860,002862,002870,002872,002874,002876,002878,002882,002888,002890,002892,002894,002896,002898,002904,002906,002908,002910,002914,002920,002922,002924,002926,002928,002930,002932,002934,002944,002946,002952,002960,002962,002964,002966,002974,002978,002984,002986,004584,004586,004588,004590,004592,004594,004596,004598,004600,004602,004604,004606,004608,004610,004616,004618,004620,004622,004624,004628,004630,004632,004634,004636,004638,004640,004648,004650,004654,004656,004658,004660,004662,004664,004666,004672,004674,004680,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004704,004706,004714,004716,004718,004720,004722,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004754,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004784,004786,004788,004790,004792,004794,004796,004798,004800,004802Hostname: nid002872 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004692 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004628 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002978 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002928 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004684 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002960 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004654 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004760 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002842 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002920 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002882 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004974 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004942 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004866 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004826 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004812 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002964 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002904 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004736 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004586 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002834 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004832 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002874 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004776 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004594 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002986 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004808 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004746 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002878 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002974 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002946 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +,004808,004812,004814,004816,004820,004824,004826,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid002872 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002978 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002928 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002960 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002882 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004942 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004826 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004812 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002964 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002904 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004876 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002908 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004700 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002878 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002798 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002834 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004882 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004658 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004844 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004632 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004942 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004860 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002874 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004808 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002824 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004800 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004664 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004802 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004910 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004752 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004730 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004842 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002986 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004660 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002838 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002844 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004978 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005072 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004800 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004714 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002824 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002802 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004802 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004622 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004584 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002838 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004672 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004730 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004584 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004786 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004980 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002824 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004718 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004816 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002924 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004930 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005018 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005084 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004816 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002888 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002924 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004584 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005086 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005076 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004660 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004650 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004956 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004802 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004786 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002924 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005072 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005012 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004956 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004896 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004786 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002856 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004874 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004686 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004672 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005036 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004704 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004956 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005076 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004790 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004714 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002922 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004758 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004672 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004894 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004722 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005086 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004704 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002802 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004790 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004704 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002802 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002926 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002922 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004672 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005018 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004664 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002926 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005054 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005048 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004796 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004664 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004894 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002926 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004872 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004790 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004718 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004752 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004796 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004636 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004674 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004718 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004734 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004762 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002922 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002910 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004618 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004732 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004620 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004796 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002910 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004608 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004674 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004620 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002910 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004636 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002932 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004620 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002908 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005050 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004734 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004636 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004890 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004762 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002856 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004724 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004878 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004846 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004650 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002856 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004604 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002808 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004936 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004756 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004680 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002856 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002908 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004972 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004936 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004782 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004640 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005042 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004640 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005042 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005000 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004640 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005042 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004828 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002844 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004756 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004874 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004640 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002908 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002984 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004680 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002844 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004756 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004912 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004782 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005004 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002812 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002844 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004662 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004662 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004680 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004686 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002888 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004986 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002800 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004632 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004606 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004632 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004986 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004848 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005040 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004720 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002850 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005078 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002888 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004680 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002932 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002932 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004992 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005048 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002850 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002812 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005048 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005078 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002888 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002932 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004632 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004720 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004722 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004946 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004722 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004946 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002934 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004814 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004856 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004830 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004702 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002808 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004722 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004784 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002984 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002808 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002984 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004784 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002808 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002984 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005010 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004784 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005056 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005010 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004784 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004702 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002800 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004702 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004976 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002800 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002800 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004994 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004950 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004850 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004766 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005058 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004656 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005058 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002934 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004916 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004778 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002934 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004694 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004738 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002934 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004856 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004856 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004852 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004898 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004944 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004820 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004820 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004926 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004744 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002862 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002862 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002862 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004694 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004726 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004924 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005016 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004682 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004726 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004592 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004682 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004738 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004592 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005044 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004592 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005034 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005090 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005034 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004698 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004698 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005080 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 0 naive algo: 0 -[-1] time: 0.93078 -[0:0] time: 0.573745 bw: 14.9571 -[0:1] time: 0.384237 bw: 22.334 -[0:2] time: 0.382073 bw: 22.4605 -[0:3] time: 0.370926 bw: 23.1355 -[0:4] time: 0.373009 bw: 23.0063 -[1:0] time: 0.417102 bw: 20.5742 -[1:1] time: 0.362265 bw: 23.6886 -[1:2] time: 0.362608 bw: 23.6662 -[1:3] time: 0.363429 bw: 23.6127 -[1:4] time: 0.363704 bw: 23.5948 -[2:0] time: 0.429774 bw: 19.9676 -[2:1] time: 0.362724 bw: 23.6586 -[2:2] time: 0.362866 bw: 23.6494 -[2:3] time: 0.363263 bw: 23.6235 -[2:4] time: 0.362008 bw: 23.7054 -[3:0] time: 0.429733 bw: 19.9695 -[3:1] time: 0.361766 bw: 23.7213 -[3:2] time: 0.362216 bw: 23.6918 -[3:3] time: 0.362651 bw: 23.6634 -[3:4] time: 0.362349 bw: 23.6831 -[4:0] time: 0.434529 bw: 19.7491 -[4:1] time: 0.362643 bw: 23.6639 -[4:2] time: 0.361182 bw: 23.7596 -[4:3] time: 0.363303 bw: 23.6209 -[4:4] time: 0.361469 bw: 23.7408 -[5:0] time: 0.43231 bw: 19.8504 -[5:1] time: 0.362727 bw: 23.6584 -[5:2] time: 0.361562 bw: 23.7347 -[5:3] time: 0.362921 bw: 23.6458 -[5:4] time: 0.36226 bw: 23.6889 -[6:0] time: 0.430324 bw: 19.942 -[6:1] time: 0.364444 bw: 23.547 -[6:2] time: 0.362648 bw: 23.6636 -[6:3] time: 0.363774 bw: 23.5903 -[6:4] time: 0.364418 bw: 23.5486 -[7:0] time: 0.434093 bw: 19.7689 -[7:1] time: 0.366189 bw: 23.4348 -[7:2] time: 0.36353 bw: 23.6062 -[7:3] time: 0.36432 bw: 23.555 -[7:4] time: 0.364396 bw: 23.5501 -[8:0] time: 0.431242 bw: 19.8996 -[8:1] time: 0.364277 bw: 23.5577 -[8:2] time: 0.362703 bw: 23.66 -[8:3] time: 0.363077 bw: 23.6356 -[8:4] time: 0.363684 bw: 23.5962 -[9:0] time: 0.469404 bw: 18.2818 -[9:1] time: 0.372588 bw: 23.0323 -[9:2] time: 0.362764 bw: 23.656 -[9:3] time: 0.367559 bw: 23.3474 -[9:4] time: 0.380095 bw: 22.5774 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 0.863192 +[0:0] time: 0.495978 bw: 17.3023 +[0:1] time: 0.434031 bw: 19.7717 +[0:2] time: 0.362128 bw: 23.6975 +[0:3] time: 0.363561 bw: 23.6042 +[0:4] time: 0.361611 bw: 23.7314 +[1:0] time: 0.409136 bw: 20.9748 +[1:1] time: 0.362965 bw: 23.6429 +[1:2] time: 0.360846 bw: 23.7817 +[1:3] time: 0.363447 bw: 23.6115 +[1:4] time: 0.3627 bw: 23.6602 +[2:0] time: 0.42693 bw: 20.1006 +[2:1] time: 0.362197 bw: 23.693 +[2:2] time: 0.361794 bw: 23.7194 +[2:3] time: 0.3623 bw: 23.6863 +[2:4] time: 0.361925 bw: 23.7108 +[3:0] time: 0.428 bw: 20.0503 +[3:1] time: 0.363141 bw: 23.6315 +[3:2] time: 0.361345 bw: 23.7489 +[3:3] time: 0.362534 bw: 23.671 +[3:4] time: 0.361316 bw: 23.7508 +[4:0] time: 0.429036 bw: 20.0019 +[4:1] time: 0.364485 bw: 23.5443 +[4:2] time: 0.361244 bw: 23.7555 +[4:3] time: 0.361852 bw: 23.7156 +[4:4] time: 0.363044 bw: 23.6377 +[5:0] time: 0.426024 bw: 20.1433 +[5:1] time: 0.36288 bw: 23.6485 +[5:2] time: 0.361355 bw: 23.7482 +[5:3] time: 0.361349 bw: 23.7486 +[5:4] time: 0.363858 bw: 23.5849 +[6:0] time: 0.428295 bw: 20.0365 +[6:1] time: 0.363371 bw: 23.6165 +[6:2] time: 0.363399 bw: 23.6147 +[6:3] time: 0.361554 bw: 23.7352 +[6:4] time: 0.362849 bw: 23.6504 +[7:0] time: 0.427652 bw: 20.0666 +[7:1] time: 0.362741 bw: 23.6575 +[7:2] time: 0.36263 bw: 23.6647 +[7:3] time: 0.360536 bw: 23.8022 +[7:4] time: 0.362213 bw: 23.692 +[8:0] time: 0.426468 bw: 20.1224 +[8:1] time: 0.360475 bw: 23.8062 +[8:2] time: 0.360933 bw: 23.776 +[8:3] time: 0.381153 bw: 22.5147 +[8:4] time: 0.368517 bw: 23.2867 +[9:0] time: 0.426466 bw: 20.1225 +[9:1] time: 0.362644 bw: 23.6638 +[9:2] time: 0.369665 bw: 23.2144 +[9:3] time: 0.370723 bw: 23.1481 +[9:4] time: 0.361501 bw: 23.7387 ======================================= type: device size: 4294967296 pad: 0 naive: 0 -time: 0.365266 -time0: 0.448226 -bw: 23.4991 -bw0: 19.296 +time: 0.36507 +time0: 0.432398 +bw: 23.527 +bw0: 19.8921 ======================================= -Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004716 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002802 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002890 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002848 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002962 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004754 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002876 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002952 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002964 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002874 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004798 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002828 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005006 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002836 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004716 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004738 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004788 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002878 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002906 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004692 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002908 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002860 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004798 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002934 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004666 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004834 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002898 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004880 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004692 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004594 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004888 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004588 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002798 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004600 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004798 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004590 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002896 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002828 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002806 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004706 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004730 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004624 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004858 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004692 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004738 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004586 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004634 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004982 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002878 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004790 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004632 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004798 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004648 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004840 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004592 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002928 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002798 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002952 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004920 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002828 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005044 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005038 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005012 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004852 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002798 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002802 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002878 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002872 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004920 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004982 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005006 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Slurm Job Hostlist: nid[002798,002800,002802,002806,002808,002812,002824,002828,002834,002836,002838,002842,002844,002848,002850,002856,002860,002862,002870,002872,002874,002876,002878,002882,002888,002890,002892,002894,002896,002898,002904,002906,002908,002910,002914,002920,002922,002924,002926,002928,002930,002932,002934,002944,002946,002952,002960,002962,002964,002966,002974,002978,002984,002986,004584,004586,004588,004590,004592,004594,004596,004598,004600,004602,004604,004606,004608,004610,004616,004618,004620,004622,004624,004628,004630,004632,004634,004636,004638,004640,004648,004650,004654,004656,004658,004660,004662,004664,004666,004672,004674,004680,004682,004684,004686,004688,004690,004692,004694,004698,004700,004702,004704,004706,004714,004716,004718,004720,004722,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004754,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004784,004786,004788,004790,004792,004794,004796,004798,004800,004802,004808,004812,004814,004816,004820,004824,004826,004828,004830,004832,004834,004840,004842,004844,004846,004848,004850,004852,004854,004856,004858,004860,004862,004864,004866,004872,004874,004876,004878,004880,004882,004884,004886,004888,004890,004892,004894,004896,004898,004904,004908,004910,004912,004914,004916,004918,004920,004924,004926,004928,004930,004936,004938,004940,004942,004944,004946,004948,004950,004956,004958,004960,004962,004968,004970,004972,004974,004976,004978,004980,004982,004986,004988,004990,004992,004994,005000,005002,005004,005006,005008,005010,005012,005014,005016,005018,005022,005026,005032,005034,005036,005038,005040,005042,005044,005046,005048,005050,005052,005054,005056,005058,005066,005068,005070,005072,005074,005076,005078,005080,005084,005086,005090] +Hostname: nid004634 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002798 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004816 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002802 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002934 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004782 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005044 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002952 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002802 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002930 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002908 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005040 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005004 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002828 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002934 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004600 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002974 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002898 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004738 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002870 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004802 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005006 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004852 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002824 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004896 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004992 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004594 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004982 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002934 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002842 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004916 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002922 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004656 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004634 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002898 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002878 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002860 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004782 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004704 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004826 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002952 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004744 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002838 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005044 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002908 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005074 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004682 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002896 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002926 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004790 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002806 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002944 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002890 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004852 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004834 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004876 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002896 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002896 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002808 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002836 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002898 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002890 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004790 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002860 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004702 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002806 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004834 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004878 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004656 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002908 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004912 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004590 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005032 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004854 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005004 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002806 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004858 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004714 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002882 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004876 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004854 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004790 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002874 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004656 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004914 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002890 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004882 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004926 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004592 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004596 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002860 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005008 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004776 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005058 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004796 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004648 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004834 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004620 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004730 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004754 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004988 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002964 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002964 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004992 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002894 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004894 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002874 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002874 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004854 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004754 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002842 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005068 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002986 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004882 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004766 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002962 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004594 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004748 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004592 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005004 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004754 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002964 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004816 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002872 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004730 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004914 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004968 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004592 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004898 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002848 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004702 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004590 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004832 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004884 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004860 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002978 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002930 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002848 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005038 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002842 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004666 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002962 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002924 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004628 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004744 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004940 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004914 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004648 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004594 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002962 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002842 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002848 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002872 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004802 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002930 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002876 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004958 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005038 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002928 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004752 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005026 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002906 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004770 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005090 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004744 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004662 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002922 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004702 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005026 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004596 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002824 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002946 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004590 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004730 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005038 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002928 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004802 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004680 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004988 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005068 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004666 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004992 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004778 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002930 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002960 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002812 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002824 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004958 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004648 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004770 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002984 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004824 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004874 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002906 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004878 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005052 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002922 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002928 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005046 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002974 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002932 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004824 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002906 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005074 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004752 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004802 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004988 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004596 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004746 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004944 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002872 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004820 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002824 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004632 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004974 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002944 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004638 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004848 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004628 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002974 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004958 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004596 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002974 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004874 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004770 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004824 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005072 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004732 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004816 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004588 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004686 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004848 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004632 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002862 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004840 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002922 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004688 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004816 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002978 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004732 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004858 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005068 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004974 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004912 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002876 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005056 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004638 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004588 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002978 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004848 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004746 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004874 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004860 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005090 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004620 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004588 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002876 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004620 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004990 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005072 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004704 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004878 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004912 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004942 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005052 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004684 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004858 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004620 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004894 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004962 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002850 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005056 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004844 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004844 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002932 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004628 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004918 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004756 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004916 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004786 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004888 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002882 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004844 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004632 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004750 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005070 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004756 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002932 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004894 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004904 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004918 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005090 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004828 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002986 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004826 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004766 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002978 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004748 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004724 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004888 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004704 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004660 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005074 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004722 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002986 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004904 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004732 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004704 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004756 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004916 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005054 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004718 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004872 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002924 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002882 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002882 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004896 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004918 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005040 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004826 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004826 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004872 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004766 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005032 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002932 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004888 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004940 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002924 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002944 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004880 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005012 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005054 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004748 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002838 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004904 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005034 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004732 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005008 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004788 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002914 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002986 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005040 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004970 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002812 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002812 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004880 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004884 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005012 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004896 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004872 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004706 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002924 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002926 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002984 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004950 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004962 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004970 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002926 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004942 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004780 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002926 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005008 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004862 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004750 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004750 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005040 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004618 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004980 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004766 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004910 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004950 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005074 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004980 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005010 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004654 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004820 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004884 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004706 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002984 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004862 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002946 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004884 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002960 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002856 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004786 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004672 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004606 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002944 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004960 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004942 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004910 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004840 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005010 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004940 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002946 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005048 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004662 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005018 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004658 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004778 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004786 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004778 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004960 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004962 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004640 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004690 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002960 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005048 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002850 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002946 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002856 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004968 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004976 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004786 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002984 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004694 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004686 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004680 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002850 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004658 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004690 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004828 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004928 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004944 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005010 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004778 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004606 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004796 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004862 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002844 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004662 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004706 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004968 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004942 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002862 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002856 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004972 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002856 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004640 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004718 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004910 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004928 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004658 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004926 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004828 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004680 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005046 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002960 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004690 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004660 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004968 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004686 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004660 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004640 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004796 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004886 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004694 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004886 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004976 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002894 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004944 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004796 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002892 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004640 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004832 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004630 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004602 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005014 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002910 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004898 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004624 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004828 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004586 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004680 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004602 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004636 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004672 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005034 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004856 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005058 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004624 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005066 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004586 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004718 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004700 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005034 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002808 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004976 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005002 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004890 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004718 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004830 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005042 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002808 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005034 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004672 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004972 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005058 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004976 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004946 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004630 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002894 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004794 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004682 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004890 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005032 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005058 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004682 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002920 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004700 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002800 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004688 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004610 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004672 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004892 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002808 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005002 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004724 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005070 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004892 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002894 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005046 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004892 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004780 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005032 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004990 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004946 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004784 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005018 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005050 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005018 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004684 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005002 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004812 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004780 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004602 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004724 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004602 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004812 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004688 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004946 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004792 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004688 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004684 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004946 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004650 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005046 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004724 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004856 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004780 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004762 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004598 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005018 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004812 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002910 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005070 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005070 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004734 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004856 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004598 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004736 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005086 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004762 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004650 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004866 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004598 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002920 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004990 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002920 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004856 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004830 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002888 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002834 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002920 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004990 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004726 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002892 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004830 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004736 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004784 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005050 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004616 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004674 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002910 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002910 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004616 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004720 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004722 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004784 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002892 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004722 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004726 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004994 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004846 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004736 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004720 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004722 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004616 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002892 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004720 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004784 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004636 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005014 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004736 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004764 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005086 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005014 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005078 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004994 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005066 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005050 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004764 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004866 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004978 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002844 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004610 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004734 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004636 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005078 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005014 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002844 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004930 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004956 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004866 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004978 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002800 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002870 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004764 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004636 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002800 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002844 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004608 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005036 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005086 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004956 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004734 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002800 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002870 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002870 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004978 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004776 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004930 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005066 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004734 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002888 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004674 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004956 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004800 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004930 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004674 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004800 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004622 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004622 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005066 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005042 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004622 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004794 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005042 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004674 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004948 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004584 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002888 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004584 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004908 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004794 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002834 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004948 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002834 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid002834 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004584 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004792 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002888 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004610 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005080 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004948 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004604 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004808 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004842 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004792 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004604 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004938 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004604 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004842 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005022 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004808 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004654 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004808 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004654 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004864 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004758 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005080 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005022 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004654 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004608 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005022 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004846 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005080 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004608 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004846 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005084 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004908 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002904 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004850 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005084 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004814 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004814 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004698 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004814 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004760 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002966 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004850 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004938 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004938 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004938 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004758 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004760 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004758 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004698 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004758 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004908 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005000 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004698 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004864 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004864 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004924 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004864 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002966 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002904 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002904 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid002904 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004986 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid002966 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004986 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004664 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid002966 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005000 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004664 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004924 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid004936 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004924 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005000 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005000 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid005076 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid004936 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) +Hostname: nid004936 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005076 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid004936 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Hostname: nid005016 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) +Hostname: nid005016 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) +Hostname: nid005016 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 mem_type: device size: 1073741824 padding: 1 naive algo: 0 -[-1] time: 15.7304 -[0:0] time: 15.6843 bw: 0.547144 -[0:1] time: 0.636695 bw: 13.4783 -[0:2] time: 0.637389 bw: 13.4636 -[0:3] time: 0.638144 bw: 13.4477 -[0:4] time: 0.639936 bw: 13.41 -[1:0] time: 15.5927 bw: 0.550356 -[1:1] time: 0.636032 bw: 13.4923 -[1:2] time: 0.646497 bw: 13.2739 -[1:3] time: 0.638227 bw: 13.4459 -[1:4] time: 0.636875 bw: 13.4745 -[2:0] time: 15.5966 bw: 0.55022 -[2:1] time: 0.635211 bw: 13.5098 -[2:2] time: 0.636567 bw: 13.481 -[2:3] time: 0.637265 bw: 13.4662 -[2:4] time: 0.636442 bw: 13.4836 -[3:0] time: 15.6246 bw: 0.549234 -[3:1] time: 0.634735 bw: 13.5199 -[3:2] time: 0.635924 bw: 13.4946 -[3:3] time: 0.635061 bw: 13.513 -[3:4] time: 0.635848 bw: 13.4962 -[4:0] time: 15.5765 bw: 0.55093 -[4:1] time: 0.635537 bw: 13.5028 -[4:2] time: 0.635678 bw: 13.4998 -[4:3] time: 0.63573 bw: 13.4987 -[4:4] time: 0.636334 bw: 13.4859 -[5:0] time: 15.6045 bw: 0.549939 -[5:1] time: 0.635054 bw: 13.5131 -[5:2] time: 0.636599 bw: 13.4803 -[5:3] time: 0.636055 bw: 13.4918 -[5:4] time: 0.63568 bw: 13.4998 -[6:0] time: 15.6929 bw: 0.546842 -[6:1] time: 0.636633 bw: 13.4796 -[6:2] time: 0.637347 bw: 13.4645 -[6:3] time: 0.636632 bw: 13.4796 -[6:4] time: 0.637188 bw: 13.4678 -[7:0] time: 15.6332 bw: 0.548931 -[7:1] time: 0.638007 bw: 13.4505 -[7:2] time: 0.638792 bw: 13.434 -[7:3] time: 0.639163 bw: 13.4262 -[7:4] time: 0.639133 bw: 13.4269 -[8:0] time: 15.6072 bw: 0.549845 -[8:1] time: 0.636126 bw: 13.4903 -[8:2] time: 0.636368 bw: 13.4852 -[8:3] time: 0.636972 bw: 13.4724 -[8:4] time: 0.636832 bw: 13.4754 -[9:0] time: 15.6575 bw: 0.548079 -[9:1] time: 0.636896 bw: 13.474 -[9:2] time: 0.636869 bw: 13.4746 -[9:3] time: 0.638079 bw: 13.449 -[9:4] time: 0.636434 bw: 13.4838 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 0 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 1 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 2 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +Found 1 devices for local rank 3 (4) -> setting device = 0 +[-1] time: 15.7201 +[0:0] time: 15.7224 bw: 0.545815 +[0:1] time: 0.632658 bw: 13.5643 +[0:2] time: 0.633965 bw: 13.5363 +[0:3] time: 0.635441 bw: 13.5049 +[0:4] time: 0.633903 bw: 13.5376 +[1:0] time: 15.621 bw: 0.549358 +[1:1] time: 0.634973 bw: 13.5148 +[1:2] time: 0.636029 bw: 13.4924 +[1:3] time: 0.635294 bw: 13.508 +[1:4] time: 0.636443 bw: 13.4836 +[2:0] time: 15.602 bw: 0.550029 +[2:1] time: 0.634308 bw: 13.529 +[2:2] time: 0.634643 bw: 13.5219 +[2:3] time: 0.635235 bw: 13.5093 +[2:4] time: 0.634892 bw: 13.5165 +[3:0] time: 15.6329 bw: 0.548943 +[3:1] time: 0.633757 bw: 13.5407 +[3:2] time: 0.634116 bw: 13.5331 +[3:3] time: 0.63466 bw: 13.5215 +[3:4] time: 0.634513 bw: 13.5246 +[4:0] time: 15.5972 bw: 0.550199 +[4:1] time: 0.634058 bw: 13.5343 +[4:2] time: 0.634884 bw: 13.5167 +[4:3] time: 0.63493 bw: 13.5157 +[4:4] time: 0.634604 bw: 13.5227 +[5:0] time: 15.6378 bw: 0.54877 +[5:1] time: 0.635107 bw: 13.512 +[5:2] time: 0.63435 bw: 13.5281 +[5:3] time: 0.635227 bw: 13.5094 +[5:4] time: 0.634821 bw: 13.5181 +[6:0] time: 15.6415 bw: 0.54864 +[6:1] time: 0.633756 bw: 13.5408 +[6:2] time: 0.634774 bw: 13.5191 +[6:3] time: 0.636981 bw: 13.4722 +[6:4] time: 0.634394 bw: 13.5272 +[7:0] time: 15.6445 bw: 0.548535 +[7:1] time: 0.634339 bw: 13.5283 +[7:2] time: 0.634186 bw: 13.5316 +[7:3] time: 0.634619 bw: 13.5224 +[7:4] time: 0.63429 bw: 13.5294 +[8:0] time: 15.6261 bw: 0.549182 +[8:1] time: 0.634136 bw: 13.5327 +[8:2] time: 0.634257 bw: 13.5301 +[8:3] time: 0.635429 bw: 13.5051 +[8:4] time: 0.634418 bw: 13.5266 +[9:0] time: 15.599 bw: 0.550136 +[9:1] time: 0.635438 bw: 13.5049 +[9:2] time: 0.634117 bw: 13.5331 +[9:3] time: 0.634546 bw: 13.5239 +[9:4] time: 0.635021 bw: 13.5138 ======================================= type: device size: 4294967296 pad: 4 naive: 0 -time: 0.637025 -time0: 15.627 -bw: 13.4714 -bw0: 0.549152 -======================================= - -Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 0 -naive algo: 1 - -[-1] time: 6.78062 -[0:0] time: 7.83042 bw: 1.09592 -[0:1] time: 7.95677 bw: 1.07852 -[0:2] time: 8.0796 bw: 1.06213 -[0:3] time: 8.22408 bw: 1.04347 -[0:4] time: 8.40486 bw: 1.02102 -[1:0] time: 8.36945 bw: 1.02534 -[1:1] time: 8.68631 bw: 0.987939 -[1:2] time: 8.74781 bw: 0.980993 -[1:3] time: 9.07966 bw: 0.94514 -[1:4] time: 8.93379 bw: 0.960571 -[2:0] time: 9.00591 bw: 0.95288 -[2:1] time: 9.02889 bw: 0.950454 -[2:2] time: 9.089 bw: 0.944168 -[2:3] time: 9.19958 bw: 0.932819 -[2:4] time: 9.24249 bw: 0.928489 -[3:0] time: 9.29612 bw: 0.923132 -[3:1] time: 9.38297 bw: 0.914587 -[3:2] time: 9.26814 bw: 0.925919 -[3:3] time: 9.2751 bw: 0.925224 -[3:4] time: 9.29843 bw: 0.922903 -[4:0] time: 8.91455 bw: 0.962645 -[4:1] time: 9.31935 bw: 0.920831 -[4:2] time: 9.35474 bw: 0.917348 -[4:3] time: 9.48555 bw: 0.904697 -[4:4] time: 9.59272 bw: 0.89459 -[5:0] time: 9.44209 bw: 0.908861 -[5:1] time: 9.38511 bw: 0.914379 -[5:2] time: 9.48948 bw: 0.904322 -[5:3] time: 9.27111 bw: 0.925622 -[5:4] time: 9.12718 bw: 0.940219 -[6:0] time: 9.34983 bw: 0.917829 -[6:1] time: 9.32988 bw: 0.919792 -[6:2] time: 9.34334 bw: 0.918466 -[6:3] time: 9.40047 bw: 0.912884 -[6:4] time: 9.51846 bw: 0.901569 -[7:0] time: 9.5379 bw: 0.899731 -[7:1] time: 9.63821 bw: 0.890367 -[7:2] time: 9.41054 bw: 0.911908 -[7:3] time: 9.4825 bw: 0.904988 -[7:4] time: 9.47691 bw: 0.905521 -[8:0] time: 9.41786 bw: 0.911199 -[8:1] time: 9.45006 bw: 0.908095 -[8:2] time: 9.38986 bw: 0.913916 -[8:3] time: 9.38717 bw: 0.914178 -[8:4] time: 9.42353 bw: 0.910651 -[9:0] time: 9.50029 bw: 0.903293 -[9:1] time: 9.54789 bw: 0.89879 -[9:2] time: 9.46385 bw: 0.906771 -[9:3] time: 9.49768 bw: 0.903542 -[9:4] time: 9.53211 bw: 0.900277 - -======================================= -type: device -size: 4294967296 -pad: 0 -naive: 1 -time: 9.20538 -time0: 9.06644 -bw: 0.934202 -bw0: 0.950084 -======================================= - -Hostname: nid005548 Rank: 953, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 465, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 293, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 681, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 661, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 153, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 357, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 3, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004746 Rank: 617, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 129, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 103, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 945, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 873, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004072 Rank: 1, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004470 Rank: 445, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005406 Rank: 799, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005492 Rank: 889, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 387, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 435, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005556 Rank: 965, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 18, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 450, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 921, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004800 Rank: 693, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 439, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004770 Rank: 657, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 107, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004822 Rank: 714, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004780 Rank: 669, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 575, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 783, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004312 Rank: 315, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004072 Rank: 2, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004176 Rank: 137, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 952, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Slurm Job Hostlist: nid[004072,004074,004076,004082,004084,004086,004092,004094,004096,004098,004104,004106,004108,004110,004112,004114,004116,004122,004124,004126,004128,004130,004136,004138,004140,004144,004146,004148,004156,004158,004160,004162,004168,004174,004176,004178,004180,004192,004200,004202,004206,004208,004210,004212,004214,004216,004218,004226,004232,004234,004236,004238,004240,004242,004244,004246,004248,004252,004254,004256,004258,004264,004270,004272,004274,004276,004278,004280,004282,004284,004286,004288,004290,004296,004300,004302,004304,004310,004312,004314,004316,004318,004320,004322,004328,004330,004350,004352,004360,004374,004380,004382,004384,004392,004394,004402,004412,004424,004426,004438,004440,004446,004448,004450,004456,004458,004460,004462,004464,004466,004468,004470,004472,004474,004476,004478,004480,004482,004488,004490,004492,004494,004496,004500,004502,004504,004508,004510,004512,004520,004522,004524,004526,004528,004530,004532,004534,004536,004538,004542,004544,004546,004702,004714,004716,004720,004724,004726,004730,004732,004734,004736,004738,004744,004746,004748,004750,004752,004756,004758,004760,004762,004764,004766,004770,004776,004778,004780,004782,004786,004788,004792,004794,004800,004808,004812,004814,004820,004822,004824,004828,004830,004832,005352,005356,005360,005362,005364,005366,005372,005376,005378,005384,005386,005388,005394,005398,005400,005402,005406,005418,005420,005422,005424,005426,005428,005432,005434,005442,005448,005450,005454,005458,005460,005462,005466,005468,005470,005480,005486,005488,005490,005492,005494,005496,005500,005502,005504,005506,005512,005520,005522,005524,005526,005528,005530,005536,005538,005548,005552,005554,005556,005558,005560,005566,005568,005570,005578,005580,005582,005584,005590,005596,005598,005600,005602] -Hostname: nid004072 Rank: 0, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004702 Rank: 571, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004524 Rank: 527, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005548 Rank: 955, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004144 Rank: 101, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005548 Rank: 954, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004744 Rank: 615, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004318 Rank: 325, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004482 Rank: 471, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004092 Rank: 27, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004502 Rank: 497, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 154, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004208 Rank: 167, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004412 Rank: 385, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 817, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004360 Rank: 355, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 311, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004748 Rank: 621, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 17, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 245, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 53, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005580 Rank: 993, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004206 Rank: 161, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004200 Rank: 155, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004788 Rank: 683, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005406 Rank: 797, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004526 Rank: 531, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005494 Rank: 895, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005470 Rank: 869, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 682, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004794 Rank: 689, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 461, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 102, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 849, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 437, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 555, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 375, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004200 Rank: 152, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004744 Rank: 613, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 405, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004762 Rank: 645, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004130 Rank: 85, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 911, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005528 Rank: 939, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004734 Rank: 601, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004472 Rank: 449, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 105, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004788 Rank: 680, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 697, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004144 Rank: 100, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004412 Rank: 386, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004714 Rank: 573, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005566 Rank: 979, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005394 Rank: 781, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004114 Rank: 61, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 466, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004466 Rank: 438, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005490 Rank: 885, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 294, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004280 Rank: 269, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 485, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 429, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004738 Rank: 611, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004464 Rank: 433, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004296 Rank: 295, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1021, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 663, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004480 Rank: 467, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005442 Rank: 833, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 42, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004226 Rank: 191, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004296 Rank: 292, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005496 Rank: 897, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004480 Rank: 464, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005406 Rank: 798, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 469, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005372 Rank: 757, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004724 Rank: 585, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 662, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004312 Rank: 313, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004314 Rank: 317, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004468 Rank: 441, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 106, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004300 Rank: 299, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005406 Rank: 796, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004464 Rank: 434, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 605, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004412 Rank: 384, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 809, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 989, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 513, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004212 Rank: 173, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004466 Rank: 436, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004084 Rank: 19, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004284 Rank: 277, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004746 Rank: 619, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005460 Rank: 855, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 493, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004478 Rank: 462, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 529, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004168 Rank: 130, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004108 Rank: 49, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004822 Rank: 713, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 883, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004538 Rank: 553, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 525, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004116 Rank: 65, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004726 Rank: 589, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004208 Rank: 165, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 614, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004472 Rank: 451, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004310 Rank: 309, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 909, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005384 Rank: 769, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004776 Rank: 660, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004478 Rank: 463, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004746 Rank: 618, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004312 Rank: 314, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004714 Rank: 574, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004464 Rank: 432, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 701, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004084 Rank: 16, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005502 Rank: 907, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005520 Rank: 923, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 917, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004146 Rank: 104, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004168 Rank: 131, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004800 Rank: 692, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004752 Rank: 629, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004258 Rank: 241, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004780 Rank: 670, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004782 Rank: 673, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004744 Rank: 612, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 874, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004232 Rank: 193, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005528 Rank: 937, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004136 Rank: 89, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004216 Rank: 182, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005520 Rank: 922, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 893, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 55, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 109, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 359, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004702 Rank: 569, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005568 Rank: 981, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 54, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 389, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005394 Rank: 782, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004780 Rank: 671, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004168 Rank: 128, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004746 Rank: 616, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004360 Rank: 353, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004476 Rank: 457, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004110 Rank: 52, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 46, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004478 Rank: 460, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005480 Rank: 875, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004472 Rank: 448, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004374 Rank: 358, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004524 Rank: 526, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004274 Rank: 257, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 870, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004482 Rank: 470, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004234 Rank: 197, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004714 Rank: 572, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 977, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005424 Rank: 815, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 850, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004208 Rank: 166, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005360 Rank: 741, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004780 Rank: 668, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 715, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004092 Rank: 25, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004300 Rank: 297, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005520 Rank: 920, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004312 Rank: 312, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004808 Rank: 698, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004252 Rank: 230, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004800 Rank: 695, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004290 Rank: 289, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 871, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 409, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005356 Rank: 737, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 498, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004440 Rank: 401, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004374 Rank: 356, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004526 Rank: 530, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004800 Rank: 694, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004702 Rank: 570, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005500 Rank: 901, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005480 Rank: 872, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005394 Rank: 780, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 221, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 499, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 51, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004128 Rank: 81, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005470 Rank: 868, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 349, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 138, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 310, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 913, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005460 Rank: 853, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004720 Rank: 581, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 881, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004538 Rank: 554, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 561, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 139, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 717, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004524 Rank: 524, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 659, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004808 Rank: 699, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004314 Rank: 318, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 425, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005504 Rank: 910, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004108 Rank: 50, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005494 Rank: 894, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004304 Rank: 305, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004736 Rank: 606, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005458 Rank: 851, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004208 Rank: 164, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 189, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 415, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004482 Rank: 468, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005492 Rank: 890, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 609, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004458 Rank: 421, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004158 Rank: 117, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004176 Rank: 136, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004822 Rank: 712, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005528 Rank: 938, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004218 Rank: 185, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004382 Rank: 365, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 967, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005426 Rank: 818, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004360 Rank: 354, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004310 Rank: 308, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 487, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005458 Rank: 848, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 775, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005490 Rank: 886, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 685, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004202 Rank: 157, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004702 Rank: 568, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 925, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005596 Rank: 1009, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004502 Rank: 496, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1017, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005512 Rank: 918, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004824 Rank: 718, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005566 Rank: 978, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004138 Rank: 93, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005502 Rank: 905, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 399, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 337, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005376 Rank: 763, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004300 Rank: 298, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004526 Rank: 528, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005580 Rank: 994, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004178 Rank: 143, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004808 Rank: 696, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004314 Rank: 319, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004770 Rank: 658, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004106 Rank: 47, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004092 Rank: 26, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005426 Rank: 819, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005536 Rank: 947, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 916, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 678, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005372 Rank: 759, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004736 Rank: 607, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004108 Rank: 48, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 646, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 443, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004794 Rank: 691, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004474 Rank: 454, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004538 Rank: 552, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 345, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004494 Rank: 486, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004490 Rank: 477, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005426 Rank: 816, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 969, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004762 Rank: 647, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005512 Rank: 919, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 961, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005556 Rank: 966, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005504 Rank: 908, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004794 Rank: 690, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 834, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 891, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004500 Rank: 495, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 369, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 865, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004446 Rank: 406, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004360 Rank: 352, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005494 Rank: 892, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 162, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004106 Rank: 45, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 801, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005496 Rank: 899, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005466 Rank: 861, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005536 Rank: 946, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005528 Rank: 936, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004278 Rank: 266, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005492 Rank: 888, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 887, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004124 Rank: 73, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004104 Rank: 41, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004314 Rank: 316, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005556 Rank: 964, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 821, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004426 Rank: 394, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004794 Rank: 688, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004494 Rank: 484, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 160, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004446 Rank: 407, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004300 Rank: 296, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004206 Rank: 163, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004254 Rank: 235, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004536 Rank: 551, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005372 Rank: 758, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004736 Rank: 604, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005536 Rank: 944, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004770 Rank: 656, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004106 Rank: 44, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004762 Rank: 644, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 702, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 630, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004092 Rank: 24, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005566 Rank: 976, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005490 Rank: 884, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 703, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005488 Rank: 882, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005372 Rank: 756, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004226 Rank: 190, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004446 Rank: 404, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005362 Rank: 745, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004500 Rank: 494, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004752 Rank: 631, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 285, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005602 Rank: 1023, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 567, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005496 Rank: 898, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004824 Rank: 719, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005554 Rank: 962, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004764 Rank: 651, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005352 Rank: 733, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 793, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 431, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005378 Rank: 765, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 430, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 442, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 995, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005568 Rank: 982, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005502 Rank: 906, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005580 Rank: 992, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 507, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004114 Rank: 63, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005602 Rank: 1022, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 610, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004468 Rank: 440, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004812 Rank: 700, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004500 Rank: 492, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004778 Rank: 665, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004528 Rank: 533, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004462 Rank: 428, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 963, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004252 Rank: 229, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004248 Rank: 227, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004732 Rank: 597, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004240 Rank: 211, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004824 Rank: 716, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005376 Rank: 761, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004258 Rank: 242, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004114 Rank: 62, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005496 Rank: 896, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 347, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004160 Rank: 121, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005602 Rank: 1020, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005568 Rank: 983, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 141, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004752 Rank: 628, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004258 Rank: 243, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004116 Rank: 66, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004234 Rank: 198, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004738 Rank: 608, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 810, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 151, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005376 Rank: 762, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004136 Rank: 90, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004758 Rank: 637, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004470 Rank: 446, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004116 Rank: 67, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004114 Rank: 60, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 95, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 111, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 739, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004148 Rank: 108, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005460 Rank: 854, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004148 Rank: 110, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005378 Rank: 764, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005554 Rank: 960, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004522 Rank: 521, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004098 Rank: 37, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004730 Rank: 593, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004280 Rank: 270, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004544 Rank: 562, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005568 Rank: 980, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 602, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004264 Rank: 246, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 835, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 830, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004258 Rank: 240, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004280 Rank: 268, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005422 Rank: 811, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004280 Rank: 271, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005422 Rank: 808, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004470 Rank: 447, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004470 Rank: 444, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004510 Rank: 509, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004350 Rank: 346, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004074 Rank: 5, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 778, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004128 Rank: 82, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004202 Rank: 156, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004116 Rank: 64, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005600 Rank: 1018, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005524 Rank: 929, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005488 Rank: 880, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004350 Rank: 344, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005460 Rank: 852, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004138 Rank: 94, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004734 Rank: 603, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004246 Rank: 222, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004226 Rank: 188, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004734 Rank: 600, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004424 Rank: 391, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 563, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004234 Rank: 199, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 393, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004274 Rank: 256, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004264 Rank: 247, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005448 Rank: 839, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1016, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 43, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005502 Rank: 904, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004246 Rank: 223, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004162 Rank: 125, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004122 Rank: 69, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004126 Rank: 79, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004138 Rank: 92, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 1010, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004202 Rank: 158, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005558 Rank: 971, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005600 Rank: 1019, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004178 Rank: 142, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004086 Rank: 21, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 766, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004720 Rank: 582, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005442 Rank: 832, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005596 Rank: 1008, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004212 Rank: 174, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 390, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005596 Rank: 1011, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 738, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005376 Rank: 760, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 625, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004440 Rank: 402, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004216 Rank: 181, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005558 Rank: 970, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004476 Rank: 458, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004216 Rank: 183, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004426 Rank: 395, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004160 Rank: 122, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004136 Rank: 91, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 195, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005356 Rank: 736, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004546 Rank: 565, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005582 Rank: 997, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005386 Rank: 773, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004212 Rank: 175, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 564, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004234 Rank: 196, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004440 Rank: 403, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004720 Rank: 583, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004252 Rank: 231, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004202 Rank: 159, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004546 Rank: 566, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004778 Rank: 667, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005388 Rank: 777, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004264 Rank: 244, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004786 Rank: 677, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005378 Rank: 767, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004544 Rank: 560, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004236 Rank: 202, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 587, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004392 Rank: 373, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004216 Rank: 180, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 743, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004232 Rank: 194, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004424 Rank: 388, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004476 Rank: 459, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004488 Rank: 475, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 426, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 473, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004136 Rank: 88, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004426 Rank: 392, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005558 Rank: 968, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004104 Rank: 40, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 541, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004440 Rank: 400, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004178 Rank: 140, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 422, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004236 Rank: 201, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004246 Rank: 220, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 599, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004212 Rank: 172, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004720 Rank: 580, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 813, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004488 Rank: 474, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004128 Rank: 83, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004786 Rank: 679, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 515, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004302 Rank: 301, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004530 Rank: 537, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004392 Rank: 374, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 586, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004160 Rank: 123, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005384 Rank: 770, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005522 Rank: 927, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004830 Rank: 725, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004232 Rank: 192, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004438 Rank: 397, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004236 Rank: 203, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004218 Rank: 186, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 914, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004460 Rank: 427, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004512 Rank: 512, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 805, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004512 Rank: 514, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004252 Rank: 228, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 735, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 802, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005384 Rank: 771, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005506 Rank: 915, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004218 Rank: 187, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004748 Rank: 623, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 666, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004732 Rank: 598, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004394 Rank: 377, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005524 Rank: 930, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004724 Rank: 584, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 926, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004488 Rank: 472, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 721, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004218 Rank: 184, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005388 Rank: 779, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004786 Rank: 676, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 214, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004128 Rank: 80, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005384 Rank: 768, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005466 Rank: 863, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004476 Rank: 456, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005352 Rank: 734, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004450 Rank: 413, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005418 Rank: 803, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004236 Rank: 200, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 505, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004160 Rank: 120, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004828 Rank: 723, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004458 Rank: 423, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004778 Rank: 664, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004382 Rank: 366, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005506 Rank: 912, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005524 Rank: 931, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004382 Rank: 367, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005418 Rank: 800, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 453, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004278 Rank: 267, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005434 Rank: 831, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004460 Rank: 424, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004748 Rank: 620, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 841, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004748 Rank: 622, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 829, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005466 Rank: 862, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004278 Rank: 264, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 740, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004304 Rank: 306, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004276 Rank: 261, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004278 Rank: 265, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005352 Rank: 732, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005360 Rank: 742, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 333, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004382 Rank: 364, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004458 Rank: 420, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005364 Rank: 749, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005468 Rank: 864, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 239, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004828 Rank: 722, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 545, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005388 Rank: 776, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004732 Rank: 596, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005522 Rank: 924, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004474 Rank: 455, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004288 Rank: 287, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004112 Rank: 57, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004274 Rank: 258, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004304 Rank: 307, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005524 Rank: 928, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004274 Rank: 259, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004490 Rank: 476, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004392 Rank: 372, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004352 Rank: 350, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005434 Rank: 828, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004758 Rank: 638, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004124 Rank: 74, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005466 Rank: 860, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 171, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 998, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004304 Rank: 304, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 814, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004352 Rank: 351, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005468 Rank: 866, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005582 Rank: 999, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004758 Rank: 639, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004522 Rank: 523, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004352 Rank: 348, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 535, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004130 Rank: 86, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 417, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005402 Rank: 794, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 338, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004474 Rank: 452, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005468 Rank: 867, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005362 Rank: 746, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004288 Rank: 286, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 291, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005582 Rank: 996, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 75, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005590 Rank: 1006, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004238 Rank: 205, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 877, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004288 Rank: 284, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005386 Rank: 774, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004290 Rank: 290, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 641, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004828 Rank: 720, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004792 Rank: 686, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004140 Rank: 97, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004074 Rank: 7, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005500 Rank: 902, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 522, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005362 Rank: 747, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005402 Rank: 795, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004328 Rank: 339, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 626, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004448 Rank: 411, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004290 Rank: 288, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005500 Rank: 903, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004758 Rank: 636, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005424 Rank: 812, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004508 Rank: 506, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004522 Rank: 520, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 532, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004124 Rank: 72, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004528 Rank: 534, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1005, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005500 Rank: 900, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004448 Rank: 410, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 837, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 935, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004448 Rank: 408, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004490 Rank: 478, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004792 Rank: 687, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004792 Rank: 684, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005402 Rank: 792, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004450 Rank: 414, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004438 Rank: 398, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004158 Rank: 119, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005362 Rank: 744, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004254 Rank: 233, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005590 Rank: 1007, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005432 Rank: 825, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004490 Rank: 479, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004082 Rank: 15, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005386 Rank: 772, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 543, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004074 Rank: 6, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 70, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004726 Rank: 590, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005590 Rank: 1004, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004158 Rank: 118, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004086 Rank: 22, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004328 Rank: 336, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 845, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004248 Rank: 225, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004764 Rank: 650, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004240 Rank: 209, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004764 Rank: 649, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004726 Rank: 591, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004126 Rank: 78, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004122 Rank: 71, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004508 Rank: 504, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004074 Rank: 4, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 149, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004530 Rank: 538, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004130 Rank: 87, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004098 Rank: 38, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004086 Rank: 23, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 511, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004130 Rank: 84, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 557, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004450 Rank: 412, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 77, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004726 Rank: 588, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004764 Rank: 648, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004122 Rank: 68, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004126 Rank: 76, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 10, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004158 Rank: 116, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004750 Rank: 627, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004750 Rank: 624, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 379, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004284 Rank: 278, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004536 Rank: 549, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004086 Rank: 20, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 134, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 973, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004438 Rank: 396, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 806, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 539, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004532 Rank: 542, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004098 Rank: 39, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005598 Rank: 1013, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004162 Rank: 126, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 483, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 787, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004394 Rank: 378, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004098 Rank: 36, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004394 Rank: 376, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004532 Rank: 540, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005420 Rank: 807, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004240 Rank: 210, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 145, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005420 Rank: 804, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 279, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004254 Rank: 234, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004248 Rank: 226, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004318 Rank: 327, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 579, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004094 Rank: 29, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004536 Rank: 550, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 974, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 838, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004192 Rank: 150, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004530 Rank: 536, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 517, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004162 Rank: 127, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 510, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004318 Rank: 326, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005560 Rank: 975, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004112 Rank: 58, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 489, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005560 Rank: 972, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004814 Rank: 705, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004242 Rank: 213, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004276 Rank: 263, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 594, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004318 Rank: 324, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004256 Rank: 237, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005578 Rank: 990, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004162 Rank: 124, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004112 Rank: 59, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004140 Rank: 99, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 133, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004302 Rank: 302, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004140 Rank: 98, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004782 Rank: 675, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005578 Rank: 988, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005578 Rank: 991, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004510 Rank: 508, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004284 Rank: 276, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005366 Rank: 753, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005526 Rank: 933, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004276 Rank: 262, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 843, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004254 Rank: 232, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004140 Rank: 96, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005450 Rank: 842, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004302 Rank: 303, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004730 Rank: 595, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004174 Rank: 135, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004112 Rank: 56, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004244 Rank: 219, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 934, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005450 Rank: 840, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 674, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 491, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004240 Rank: 208, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004192 Rank: 148, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004248 Rank: 224, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004536 Rank: 548, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004174 Rank: 132, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004782 Rank: 672, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004276 Rank: 260, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004830 Rank: 726, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004302 Rank: 300, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004496 Rank: 490, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004320 Rank: 329, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 179, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 169, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004730 Rank: 592, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 13, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004270 Rank: 249, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004830 Rank: 727, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 418, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004760 Rank: 643, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005526 Rank: 932, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 633, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004322 Rank: 334, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004496 Rank: 488, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005364 Rank: 750, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004830 Rank: 724, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 822, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004716 Rank: 577, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 826, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004542 Rank: 559, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004534 Rank: 544, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004542 Rank: 558, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005364 Rank: 751, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004238 Rank: 206, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004384 Rank: 371, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004256 Rank: 238, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005448 Rank: 836, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004504 Rank: 501, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004322 Rank: 335, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005364 Rank: 748, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005432 Rank: 827, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004384 Rank: 370, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004456 Rank: 419, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004760 Rank: 642, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004322 Rank: 332, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004238 Rank: 207, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004542 Rank: 556, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 823, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 846, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004492 Rank: 482, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004534 Rank: 547, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004760 Rank: 640, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004384 Rank: 368, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005454 Rank: 847, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004270 Rank: 250, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 785, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005432 Rank: 824, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004242 Rank: 215, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004456 Rank: 416, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 653, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005366 Rank: 755, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005454 Rank: 844, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004492 Rank: 481, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004238 Rank: 204, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005428 Rank: 820, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004534 Rank: 546, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004076 Rank: 8, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004270 Rank: 251, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005462 Rank: 857, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004256 Rank: 236, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005538 Rank: 949, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004492 Rank: 480, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004082 Rank: 14, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 754, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004380 Rank: 363, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004210 Rank: 170, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005598 Rank: 1015, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004814 Rank: 706, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 503, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004270 Rank: 248, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 323, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004242 Rank: 212, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004716 Rank: 578, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004814 Rank: 707, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005486 Rank: 879, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 518, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005366 Rank: 752, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004210 Rank: 168, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005486 Rank: 878, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004814 Rank: 704, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005598 Rank: 1014, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 502, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004504 Rank: 500, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 343, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004244 Rank: 217, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005598 Rank: 1012, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004096 Rank: 33, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005486 Rank: 876, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004094 Rank: 30, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004820 Rank: 709, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004082 Rank: 12, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004520 Rank: 519, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004272 Rank: 253, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005400 Rank: 789, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004094 Rank: 31, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004716 Rank: 576, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 11, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004520 Rank: 516, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004076 Rank: 9, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004756 Rank: 634, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004094 Rank: 28, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004214 Rank: 177, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 178, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005398 Rank: 786, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004156 Rank: 113, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004214 Rank: 176, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 331, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004282 Rank: 275, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004756 Rank: 635, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004180 Rank: 147, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005398 Rank: 784, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004316 Rank: 322, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004316 Rank: 321, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004316 Rank: 320, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004756 Rank: 632, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004180 Rank: 146, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004180 Rank: 144, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004320 Rank: 330, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004766 Rank: 655, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004766 Rank: 654, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004244 Rank: 218, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 791, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004832 Rank: 729, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004320 Rank: 328, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004766 Rank: 652, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 858, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005400 Rank: 790, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005462 Rank: 859, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005400 Rank: 788, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005462 Rank: 856, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 381, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004244 Rank: 216, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 985, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004380 Rank: 361, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005538 Rank: 951, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004820 Rank: 711, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005538 Rank: 950, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 957, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004820 Rank: 710, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005538 Rank: 948, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004272 Rank: 254, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004820 Rank: 708, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004156 Rank: 114, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 255, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004156 Rank: 115, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 362, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004156 Rank: 112, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 341, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005570 Rank: 986, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004272 Rank: 252, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005570 Rank: 987, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004380 Rank: 360, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 731, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005570 Rank: 984, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004402 Rank: 383, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004330 Rank: 342, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004832 Rank: 730, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004282 Rank: 273, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004402 Rank: 382, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004402 Rank: 380, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004832 Rank: 728, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004330 Rank: 340, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 958, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005584 Rank: 1001, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid004096 Rank: 35, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004096 Rank: 34, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004096 Rank: 32, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005552 Rank: 959, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004282 Rank: 274, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005552 Rank: 956, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004282 Rank: 272, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005530 Rank: 943, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004286 Rank: 281, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005584 Rank: 1000, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid005584 Rank: 1003, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid005584 Rank: 1002, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 941, Local 1, GPUs 1 (count=1), CPUs 72-143 (count=72) -Hostname: nid005530 Rank: 942, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid005530 Rank: 940, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -Hostname: nid004286 Rank: 282, Local 2, GPUs 2 (count=1), CPUs 144-215 (count=72) -Hostname: nid004286 Rank: 283, Local 3, GPUs 3 (count=1), CPUs 216-287 (count=72) -Hostname: nid004286 Rank: 280, Local 0, GPUs 0 (count=1), CPUs 0-71 (count=72) -mem_type: device -size: 1073741824 -padding: 1 -naive algo: 1 - -[-1] time: 8.20451 -[0:0] time: 9.61907 bw: 0.892139 -[0:1] time: 9.44606 bw: 0.908479 -[0:2] time: 9.4207 bw: 0.910924 -[0:3] time: 9.4264 bw: 0.910374 -[0:4] time: 9.45642 bw: 0.907483 -[1:0] time: 9.61165 bw: 0.892828 -[1:1] time: 9.47711 bw: 0.905502 -[1:2] time: 9.51776 bw: 0.901635 -[1:3] time: 9.54085 bw: 0.899453 -[1:4] time: 9.8328 bw: 0.872747 -[2:0] time: 9.60359 bw: 0.893577 -[2:1] time: 9.49282 bw: 0.904004 -[2:2] time: 9.51094 bw: 0.902282 -[2:3] time: 9.64736 bw: 0.889523 -[2:4] time: 9.73577 bw: 0.881445 -[3:0] time: 9.737 bw: 0.881334 -[3:1] time: 9.9104 bw: 0.865913 -[3:2] time: 10.0438 bw: 0.854412 -[3:3] time: 9.6104 bw: 0.892943 -[3:4] time: 9.73436 bw: 0.881573 -[4:0] time: 9.78161 bw: 0.877314 -[4:1] time: 9.64655 bw: 0.889597 -[4:2] time: 9.68356 bw: 0.886198 -[4:3] time: 9.69721 bw: 0.88495 -[4:4] time: 9.73303 bw: 0.881693 -[5:0] time: 9.65955 bw: 0.8884 -[5:1] time: 9.70502 bw: 0.884238 -[5:2] time: 9.75326 bw: 0.879864 -[5:3] time: 9.33703 bw: 0.919087 -[5:4] time: 9.57752 bw: 0.896009 -[6:0] time: 9.59446 bw: 0.894427 -[6:1] time: 9.48658 bw: 0.904598 -[6:2] time: 9.53572 bw: 0.899937 -[6:3] time: 9.55059 bw: 0.898536 -[6:4] time: 9.54488 bw: 0.899073 -[7:0] time: 9.45357 bw: 0.907757 -[7:1] time: 9.13632 bw: 0.939278 -[7:2] time: 9.61145 bw: 0.892846 -[7:3] time: 9.55951 bw: 0.897697 -[7:4] time: 9.52563 bw: 0.90089 -[8:0] time: 9.20274 bw: 0.932499 -[8:1] time: 9.55038 bw: 0.898556 -[8:2] time: 9.27045 bw: 0.925688 -[8:3] time: 9.39211 bw: 0.913697 -[8:4] time: 9.35636 bw: 0.917188 -[9:0] time: 9.17322 bw: 0.9355 -[9:1] time: 9.18198 bw: 0.934608 -[9:2] time: 9.34243 bw: 0.918556 -[9:3] time: 8.99746 bw: 0.953774 -[9:4] time: 9.24351 bw: 0.928386 - -======================================= -type: device -size: 4294967296 -pad: 4 -naive: 1 -time: 9.53056 -time0: 9.54365 -bw: 0.900841 -bw0: 0.899577 +time: 0.634688 +time0: 15.6324 +bw: 13.5209 +bw0: 0.548961 ======================================= diff --git a/allreduce-perf/mpi-cpp/run/plot.py b/allreduce-perf/mpi-cpp/run/plot.py index 30d31aa..96e9e1a 100644 --- a/allreduce-perf/mpi-cpp/run/plot.py +++ b/allreduce-perf/mpi-cpp/run/plot.py @@ -28,7 +28,7 @@ # Function to parse integers from the filename def parse_filename(file_path): filename = os.path.basename(file_path) - pattern = re.compile(r"job_n_(\d+)_N_(\d+)_TPN_(\d+)\.out") + pattern = re.compile(r".*?_n_(\d+)_N_(\d+)_TPN_(\d+)\.out") match = pattern.match(filename) if match: n = int(match.group(1)) @@ -136,12 +136,12 @@ def plot_data_2(grouped_lines, base_name, ns, tpn, pad, bw0, combined=False): idx = 0 l = 0 for small_block_name, data in grouped_lines.items(): - if combined or ( + if (not "naive" in small_block_name) and (combined or ( data["tpn"] == tpn and ( (pad and ("pad" in small_block_name)) or (not pad and (not "pad" in small_block_name)) - ) + )) ): color = colors[idx % len(colors)] marker = markers[idx % len(markers)] @@ -190,51 +190,52 @@ def plot_data_2(grouped_lines, base_name, ns, tpn, pad, bw0, combined=False): ) idx = idx + 1 - if combined or tpn == 1: - n_nccl = [4, 16, 32, 64, 128, 256] - bw_nccl = [22.84, 22.87, 22.89, 22.90, 22.90, 22.92] + if combined or tpn == 4: + n_nccl = [4, 16, 32, 64, 128, 256, 512, 1024] + bw_nccl = [343.360, 91.740, 91.673, 91.646, 91.442, 91.464, 91.449, 91.449] plt.plot( n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f"NCCL/uenv (ranks/node=1)", + label=f"NCCL/container (ranks/node=4)", alpha=alpha, markersize=markersize, ) idx = idx + 1 if combined or tpn == 4: - n_nccl = [4, 16, 32, 64] - bw_nccl = [343.360, 91.740, 91.673, 91.646] + n_nccl = [4, 16, 32, 64, 128, 256] + bw_nccl = [342.326, 91.305, 91.311, 91.331, 91.437, 91.462] plt.plot( n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f"NCCL/container (ranks/node=4)", + label=f"NCCL/uenv (ranks/node=4)", alpha=alpha, markersize=markersize, ) idx = idx + 1 - if combined or tpn == 4: + if combined or tpn == 1: n_nccl = [4, 16, 32, 64, 128, 256] - bw_nccl = [342.326, 91.305, 91.311, 91.331, 91.437, 91.462] + bw_nccl = [22.84, 22.87, 22.89, 22.90, 22.90, 22.92] plt.plot( n_nccl, bw_nccl, color=colors[idx % len(colors)], marker=markers[idx % len(markers)], linestyle=line_styles[idx % len(line_styles)], - label=f"NCCL/uenv (ranks/node=4)", + label=f"NCCL/uenv (ranks/node=1)", alpha=alpha, markersize=markersize, ) idx = idx + 1 + # Add custom legend entries handles, labels = plt.gca().get_legend_handles_labels() unique_labels = list( diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory.svg index b1b0b68..39b10bd 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:36.159442 + 2024-06-05T11:35:25.396010 image/svg+xml @@ -42,16 +42,16 @@ z +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -104,11 +104,11 @@ z +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -150,11 +150,11 @@ z +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -203,166 +203,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -654,16 +654,16 @@ z +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -679,11 +679,11 @@ L -3.5 0 +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -699,11 +699,11 @@ L 537.309916 220.521325 +" clip-path="url(#pd5d625e514)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -718,187 +718,187 @@ L 537.309916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1108,17 +1108,17 @@ z - + - - - - - - - - - - + + + + + + + + + - + - +" style="stroke: #ce9032; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + - +" style="stroke: #97a431; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + - + - +" style="stroke: #32b166; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + - + - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + + + - + - +" style="stroke: #39a7d0; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + + + - + - +" style="stroke: #a48cf4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +" clip-path="url(#pd5d625e514)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - +L 5 -5 +z +" style="stroke: #f561dd; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + @@ -1667,30 +1558,30 @@ z - - - + - + - + - - + + - + - + @@ -1762,88 +1653,16 @@ L 561.466916 146.860877 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #97a431; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - + @@ -1866,16 +1685,16 @@ L 561.466916 184.004867 - + +" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #32b166; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - + @@ -1902,90 +1721,18 @@ L 561.466916 196.231745 - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - + @@ -2150,18 +1859,18 @@ L 561.466916 257.829492 - - + + - + - + - + @@ -2188,11 +1897,49 @@ L 561.466916 270.05637 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg index 6bb5ea3..8bed990 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:32.883918 + 2024-06-05T11:35:22.390695 image/svg+xml @@ -42,16 +42,16 @@ z +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -104,11 +104,11 @@ z +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -150,11 +150,11 @@ z +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -203,166 +203,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -654,16 +654,16 @@ z +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -679,11 +679,11 @@ L -3.5 0 +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -699,11 +699,11 @@ L 556.299916 220.521325 +" clip-path="url(#p09046f72e2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -718,187 +718,187 @@ L 556.299916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1108,84 +1108,52 @@ z - + - - - - - - - - - - - - - - - +" style="stroke: #97a431; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - + +" clip-path="url(#p09046f72e2)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - +" style="stroke: #f561dd; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + @@ -1444,30 +1412,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg index af399e4..b02c82a 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:34.088510 + 2024-06-05T11:35:23.528976 image/svg+xml @@ -42,16 +42,16 @@ z +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -104,11 +104,11 @@ z +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -150,11 +150,11 @@ z +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -203,166 +203,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -654,16 +654,16 @@ z +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -679,11 +679,11 @@ L -3.5 0 +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -699,11 +699,11 @@ L 556.299916 220.521325 +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -718,187 +718,187 @@ L 556.299916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1108,84 +1108,52 @@ z - + - +" style="stroke: #32b166; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - - - - - - - - - - - - - - +" clip-path="url(#p7d0532eb29)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1506,30 +1474,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg index 366f644..88b389a 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_pad_startup.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:34.716763 + 2024-06-05T11:35:24.109871 image/svg+xml @@ -38,96 +38,64 @@ z " style="fill: #ffffff"/> - - - - - - - - - - - - - - + + + + + + +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -180,11 +148,11 @@ z +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -226,11 +194,11 @@ z +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -279,166 +247,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -730,16 +698,16 @@ z +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -755,11 +723,11 @@ L -3.5 0 +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -775,11 +743,11 @@ L 591.433395 220.521325 +" clip-path="url(#p83efbff540)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -794,187 +762,187 @@ L 591.433395 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1184,84 +1152,52 @@ z - + - +" style="stroke: #32b166; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - - - - - - - - - - - - - - +" clip-path="url(#p83efbff540)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1606,30 +1542,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg index c87042b..f3cf35d 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_1_startup.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:33.501859 + 2024-06-05T11:35:22.981730 image/svg+xml @@ -38,96 +38,64 @@ z " style="fill: #ffffff"/> - - - - - - - - - - - - - - + + + + + + +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -180,11 +148,11 @@ z +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -226,11 +194,11 @@ z +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -279,166 +247,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -730,16 +698,16 @@ z +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -755,11 +723,11 @@ L -3.5 0 +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -775,11 +743,11 @@ L 556.299916 220.521325 +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -794,187 +762,187 @@ L 556.299916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1184,84 +1152,52 @@ z - + - - - - - - - - - - - - - - - +" style="stroke: #97a431; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - + +" clip-path="url(#p8cf8e8eded)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1544,30 +1480,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg index 1b2a043..f43385c 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:30.344287 + 2024-06-05T11:35:20.008274 image/svg+xml @@ -42,16 +42,16 @@ z +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -104,11 +104,11 @@ z +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -150,11 +150,11 @@ z +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -203,166 +203,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -654,16 +654,16 @@ z +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -679,11 +679,11 @@ L -3.5 0 +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -699,11 +699,11 @@ L 537.309916 220.521325 +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -718,187 +718,187 @@ L 537.309916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1108,17 +1108,17 @@ z - + - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - +" clip-path="url(#p2da3b86378)" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + +L 342.528278 108.393297 +L 397.384805 108.381109 +L 452.241332 108.389418 +L 507.09786 108.389418 +" clip-path="url(#p2da3b86378)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #39a7d0; stroke-opacity: 0.7; stroke-width: 1.5"/> - +" style="stroke: #39a7d0; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - + + + + + + + + + - + +" clip-path="url(#p2da3b86378)" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #a48cf4; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1502,30 +1477,30 @@ z - - - + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - + - + @@ -1733,18 +1674,18 @@ L 561.466916 214.572062 - - + + - + - + - + @@ -1775,7 +1716,7 @@ L 561.466916 226.798941 - + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg index 860d6ec..820f1a8 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:31.626443 + 2024-06-05T11:35:21.214739 image/svg+xml @@ -42,16 +42,16 @@ z +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -104,11 +104,11 @@ z +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -150,11 +150,11 @@ z +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -203,166 +203,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -654,16 +654,16 @@ z +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -679,11 +679,11 @@ L -3.5 0 +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -699,11 +699,11 @@ L 537.309916 220.521325 +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -718,187 +718,187 @@ L 537.309916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1108,65 +1108,36 @@ z - + - +" style="stroke: #ce9032; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - +" clip-path="url(#pad720899ff)" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + +L 342.528278 108.393297 +L 397.384805 108.381109 +L 452.241332 108.389418 +L 507.09786 108.389418 +" clip-path="url(#pad720899ff)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #39a7d0; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - + + + + + + + + + - + +" clip-path="url(#pad720899ff)" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #a48cf4; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1560,30 +1535,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - + - + @@ -1799,18 +1736,18 @@ L 561.466916 214.803741 - - + + - + - + - + @@ -1841,7 +1778,7 @@ L 561.466916 227.030619 - + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg index 76cfe22..ef763c2 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_pad_startup.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:32.298486 + 2024-06-05T11:35:21.846849 image/svg+xml @@ -39,7 +39,7 @@ z - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -197,11 +139,11 @@ z +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -243,11 +185,11 @@ z +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -296,166 +238,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -747,16 +689,16 @@ z +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -772,11 +714,11 @@ L -3.5 0 +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -792,11 +734,11 @@ L 581.938395 220.521325 +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -811,187 +753,187 @@ L 581.938395 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1201,65 +1143,36 @@ z - + - +" style="stroke: #ce9032; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - +" clip-path="url(#p8b942e1910)" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + +L 387.156757 108.393297 +L 442.013285 108.381109 +L 496.869812 108.389418 +L 551.726339 108.389418 +" clip-path="url(#p8b942e1910)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #39a7d0; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - + + + + + + + + + - + +" clip-path="url(#p8b942e1910)" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #a48cf4; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1677,30 +1594,30 @@ z - - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - + - + @@ -1916,18 +1795,18 @@ L 606.095395 214.803741 - - + + - + - + - + @@ -1958,7 +1837,7 @@ L 606.095395 227.030619 - + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg index 0fd7f55..9e5726c 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_4_startup.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:31.007817 + 2024-06-05T11:35:20.631408 image/svg+xml @@ -39,7 +39,7 @@ z - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -201,11 +143,11 @@ z +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -247,11 +189,11 @@ z +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -300,166 +242,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -751,16 +693,16 @@ z +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -776,11 +718,11 @@ L -3.5 0 +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -796,11 +738,11 @@ L 537.309916 220.521325 +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -815,187 +757,187 @@ L 537.309916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1205,17 +1147,17 @@ z - + - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - +" clip-path="url(#p36d62abde7)" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + +L 342.528278 108.393297 +L 397.384805 108.381109 +L 452.241332 108.389418 +L 507.09786 108.389418 +" clip-path="url(#p36d62abde7)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #39a7d0; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - + + + + + + + + + - + +" clip-path="url(#p36d62abde7)" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #a48cf4; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - - - - - - - + + + + + + + @@ -1623,30 +1540,30 @@ z - - - + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - + - + @@ -1854,18 +1737,18 @@ L 561.466916 214.572062 - - + + - + - + - + @@ -1896,7 +1779,7 @@ L 561.466916 226.798941 - + diff --git a/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg b/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg index 71d9d7b..78f1812 100644 --- a/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg +++ b/allreduce-perf/mpi-cpp/run/plots/device_memory_startup.svg @@ -6,7 +6,7 @@ - 2024-06-03T10:00:35.420229 + 2024-06-05T11:35:24.739559 image/svg+xml @@ -39,7 +39,7 @@ z - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -446,11 +266,11 @@ z +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -492,11 +312,11 @@ z +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -545,166 +365,166 @@ z - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -996,16 +816,16 @@ z +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -1021,11 +841,11 @@ L -3.5 0 +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -1041,11 +861,11 @@ L 537.309916 220.521325 +" clip-path="url(#p480d836561)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -1060,187 +880,187 @@ L 537.309916 103.860525 - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1450,17 +1270,17 @@ z - + - - - - - - - - - - + + + + + + + + + - + - +" style="stroke: #ce9032; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + + + - + - +" style="stroke: #97a431; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + - + - +" style="stroke: #32b166; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - - - + + + + + + + - + - +" style="stroke: #36ada4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + + + - + - +" style="stroke: #39a7d0; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + + + - + - +" style="stroke: #a48cf4; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +" clip-path="url(#p480d836561)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #f561dd; stroke-opacity: 0.7; stroke-width: 1.5"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - +L 5 -5 +z +" style="stroke: #f561dd; stroke-opacity: 0.7; stroke-linejoin: miter"/> - - - - - - - + + + + + + + @@ -2042,30 +1753,30 @@ z - - - + - + - + - - + + - + - + @@ -2137,88 +1848,16 @@ L 561.466916 146.860877 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +" style="fill: none; stroke-dasharray: 9.6,2.4,1.5,2.4; stroke-dashoffset: 0; stroke: #97a431; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - + @@ -2241,16 +1880,16 @@ L 561.466916 184.004867 - + +" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #32b166; stroke-opacity: 0.7; stroke-width: 1.5"/> - + - + @@ -2277,90 +1916,18 @@ L 561.466916 196.231745 - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +" style="fill: none; stroke: #36ada4; stroke-opacity: 0.7; stroke-width: 1.5; stroke-linecap: square"/> - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - + @@ -2525,18 +2054,18 @@ L 561.466916 257.829492 - - + + - + - + - + @@ -2563,11 +2092,49 @@ L 561.466916 270.05637 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - +