Skip to content

Commit

Permalink
Add support for substring based device selection
Browse files Browse the repository at this point in the history 
Add support for pre-SYCL1.2.1R3 standard
Add support for custom dpcpp paths
Add move semantic handling workaround in field_summary for old ComputeCpp versions
Fixed COMPUTECPP_USER_FLAGS from being overwritten
Fixed CXX_EXTRA_FLAGS/CXX_EXTRA_LINKER_FLAGS space handling, closes #4
  • Loading branch information
@tom91136
tom91136 committed Jan 10, 2021
1 parent 545807f commit 64e3693
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 50 deletions.
79 changes: 62 additions & 17 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
project(cloverleaf_sycl)


set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

if (SYCL_RUNTIME)
Expand All @@ -15,29 +16,50 @@ if (SYCL_RUNTIME)
if (NOT HIPSYCL_INSTALL_DIR)
message(FATAL_ERROR "HIPSYCL_INSTALL_DIR is undefined")
endif ()

set(hipSYCL_DIR ${HIPSYCL_INSTALL_DIR}/lib/cmake)
find_package(hipSYCL CONFIG REQUIRED)
set(EXTRA_FLAGS -Wno-sign-compare -Wno-stringop-truncation)
elseif (${SYCL_RUNTIME} STREQUAL "HIPSYCL-NEXT")

if (NOT HIPSYCL_INSTALL_DIR)
message(FATAL_ERROR "HIPSYCL_INSTALL_DIR is undefined")
endif ()
set(CMAKE_CXX_STANDARD 17)
set(hipSYCL_DIR ${HIPSYCL_INSTALL_DIR}/lib/cmake/hipSYCL)
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
find_package(hipSYCL CONFIG REQUIRED)
set(EXTRA_FLAGS -Wno-sign-compare -Wno-stringop-truncation)
elseif (${SYCL_RUNTIME} STREQUAL "COMPUTECPP")

if (NOT ComputeCpp_DIR)
message(FATAL_ERROR "ComputeCpp_DIR is undefined")
endif ()
set(OpenCL_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include)
add_definitions(-DCL_TARGET_OPENCL_VERSION=220)
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
set(COMPUTECPP_USER_FLAGS -O3 -fsycl-split-modules=20 -mllvm -inline-threshold=10000 -no-serial-memop)
find_package(ComputeCpp REQUIRED)
# set(EXTRA_FLAGS -pedantic)
set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop)
elseif (${SYCL_RUNTIME} STREQUAL "DPCPP")

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_COMPILER "dpcpp")
if (NOT DPCPP_BIN)
message(STATUS "DPCPP_BIN unspecified, using dpcpp on path")
set(CMAKE_CXX_COMPILER "dpcpp")
else()
set(CMAKE_CXX_COMPILER ${DPCPP_BIN})
endif ()

if(DPCPP_INCLUDE)
include_directories(${DPCPP_INCLUDE})
endif()

add_definitions(-DCL_TARGET_OPENCL_VERSION=220)
set(CMAKE_CXX_STANDARD 14)
set(EXTRA_FLAGS -pedantic)
else ()
message(FATAL_ERROR "SYCL_RUNTIME unsupported, must be one of HIPSYCL|COMPUTECPP|DPCPP, got ${SYCL_RUNTIME}")
message(FATAL_ERROR "SYCL_RUNTIME unsupported, must be one of HIPSYCL|HIPSYCL-NEXT|COMPUTECPP|DPCPP, got ${SYCL_RUNTIME}")
endif ()
else ()
message(FATAL_ERROR "SYCL_RUNTIME not defined, must be one of HIPSYCL|COMPUTECPP|DPCPP")
message(FATAL_ERROR "SYCL_RUNTIME not defined, must be one of HIPSYCL|HIPSYCL-NEXT|COMPUTECPP|DPCPP")
endif ()


Expand Down Expand Up @@ -99,9 +121,12 @@ set(SOURCES
src/visit.cpp)

include_directories(src)
add_executable(clover_leaf ${SOURCES})
add_executable(cloverleaf ${SOURCES})

separate_arguments(CXX_EXTRA_FLAGS)
separate_arguments(CXX_EXTRA_LINKER_FLAGS)

target_compile_options(clover_leaf
target_compile_options(cloverleaf
PUBLIC
-Wall
-Wextra
Expand All @@ -115,20 +140,40 @@ target_compile_options(clover_leaf
${EXTRA_FLAGS}
)


if (DEFINED USE_PRE_SYCL121R3)
add_definitions(-DUSE_PRE_SYCL121R3)
endif ()


set(DEBUG_OPTIONS -O2 -fno-omit-frame-pointer ${CXX_EXTRA_FLAGS})
set(RELEASE_OPTIONS -O3 ${CXX_EXTRA_FLAGS})
set(RELEASE_OPTIONS -O3 ${CXX_EXTRA_FLAGS})

target_link_libraries(clover_leaf PUBLIC ${MPI_C_LIB})
target_link_libraries(cloverleaf PUBLIC ${MPI_C_LIB})

target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:RelWithDebInfo>:${RELEASE_OPTIONS}>")
target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:Release>:${RELEASE_OPTIONS}>")
target_compile_options(clover_leaf PUBLIC "$<$<CONFIG:Debug>:${DEBUG_OPTIONS}>")
target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:RelWithDebInfo>:${RELEASE_OPTIONS}>")
target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:Release>:${RELEASE_OPTIONS}>")
target_compile_options(cloverleaf PUBLIC "$<$<CONFIG:Debug>:${DEBUG_OPTIONS}>")

if (${CMAKE_VERSION} VERSION_LESS "3.13.0")
message(WARNING "target_link_options is only available in CMake >= 3.13.0, using fallback target_link_libraries, this may cause issues with some compilers")
message(WARNING "whitespaces are not supported for CXX_EXTRA_LINKER_FLAGS/CXX_EXTRA_FLAGS in this mode as they are treated as libraries arguments (CMake splits them)")
if (DEFINED CXX_EXTRA_LINKER_FLAGS)
list(APPEND EXTRA_LINK_FLAGS "-Wl,${CXX_EXTRA_LINKER_FLAGS}")
endif ()

target_link_libraries(cloverleaf PUBLIC ${EXTRA_LINK_FLAGS})
target_link_libraries(cloverleaf PUBLIC ${CXX_EXTRA_FLAGS})

else ()
target_link_options(cloverleaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})
target_link_options(cloverleaf PUBLIC ${CXX_EXTRA_FLAGS})
endif ()

target_link_options(clover_leaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})

if (NOT ${SYCL_RUNTIME} STREQUAL "DPCPP")
add_sycl_to_target(
TARGET clover_leaf
TARGET cloverleaf
SOURCES ${SOURCES}) # must be the last
endif ()

4 changes: 3 additions & 1 deletion src/field_summary.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ struct captures {
};

struct value_type {
double vol = 0.0, mass = 0.0, ie = 0.0, ke = 0.0, press = 0.0;
double vol, mass, ie, ke, press;
value_type(double vol = 0.0, double mass = 0.0, double ie = 0.0, double ke = 0.0, double press = 0.0) :
vol(vol), mass(mass), ie(ie), ke(ke), press(press) {}
};

typedef clover::local_reducer<value_type, value_type, captures> ctx;
Expand Down
80 changes: 48 additions & 32 deletions src/initialise.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

#include <fstream>
#include <iomanip>
#include <algorithm>

extern std::ostream g_out;
std::ofstream of;
Expand Down Expand Up @@ -69,35 +70,35 @@ void printDetailed(const cl::sycl::device &device, size_t index) {
std::cout << " + Device : " << device.get_info<cl::sycl::info::device::name>() << "\n";
std::cout << " - Index : " << index << "\n";
std::cout << " - Type : " << deviceName(type) << "\n";
std::cout << " - Vendor : " << device.get_info<cl::sycl::info::device::vendor>()<< "\n";
std::cout << " - Vendor : " << device.get_info<cl::sycl::info::device::vendor>() << "\n";
std::cout << " - Extensions : " << extensions.str() << "\n";
std::cout << " + Platform : " << platform.get_info<cl::sycl::info::platform::name>()<< "\n";
std::cout << " - Vendor : " << platform.get_info<cl::sycl::info::platform::vendor>()<< "\n";
std::cout << " - Version : " << platform.get_info<cl::sycl::info::platform::version>()<< "\n";
std::cout << " - Profile : " << platform.get_info<cl::sycl::info::platform::profile>()<< "\n";
std::cout << " + Platform : " << platform.get_info<cl::sycl::info::platform::name>() << "\n";
std::cout << " - Vendor : " << platform.get_info<cl::sycl::info::platform::vendor>() << "\n";
std::cout << " - Version : " << platform.get_info<cl::sycl::info::platform::version>() << "\n";
std::cout << " - Profile : " << platform.get_info<cl::sycl::info::platform::profile>() << "\n";
}

void printSimple(const cl::sycl::device &device, size_t index) {
std::cout << std::setw(3) << index << ". "
<< device.get_info<cl::sycl::info::device::name>()
<< "(" << deviceName(device.get_info<cl::sycl::info::device::device_type>()) << ")"
<< std::endl;
<< device.get_info<cl::sycl::info::device::name>()
<< "(" << deviceName(device.get_info<cl::sycl::info::device::device_type>()) << ")"
<< std::endl;
}

void printHelp(const std::string &name) {
std::cout << std::endl;
std::cout << "Usage: " << name << " [OPTIONS]\n\n"
<< "Options:\n"
<< " -h --help Print the message\n"
<< " --list List available devices\n"
<< " --list-detailed List available devices and capabilities\n"
<< " --device <INDEX> Select device at INDEX from output of --list\n"
<< " --file Custom clover.in file (defaults to clover.in if unspecified)\n"
<< std::endl;
<< "Options:\n"
<< " -h --help Print the message\n"
<< " --list List available devices\n"
<< " --list-detailed List available devices and capabilities\n"
<< " --device <INDEX> Select device at INDEX from output of --list\n"
<< " --file Custom clover.in file (defaults to clover.in if unspecified)\n"
<< std::endl;
}

RunConfig parseArgs(const std::vector<cl::sycl::device> &devices,
const std::vector<std::string> &args) {
const std::vector<std::string> &args) {

const auto readParam = [&args](size_t current, const std::string &emptyMessage, auto map) {
if (current + 1 < args.size()) {
Expand All @@ -109,7 +110,7 @@ RunConfig parseArgs(const std::vector<cl::sycl::device> &devices,
}
};

auto config = RunConfig{ "clover.in",devices[0]};
auto config = RunConfig{"clover.in", devices[0]};
for (size_t i = 0; i < args.size(); ++i) {
const auto arg = args[i];

Expand All @@ -124,12 +125,27 @@ RunConfig parseArgs(const std::vector<cl::sycl::device> &devices,
std::exit(EXIT_SUCCESS);
} else if (arg == "--device") {
readParam(i, "--device specified but no size was given", [&config](const auto &param) {
try { config.device = cl::sycl::device::get_devices().at(std::stoul(param)); }
auto devices = cl::sycl::device::get_devices();

try { config.device = devices.at(std::stoul(param)); }
catch (const std::exception &e) {
std::cerr << "failed to parse/select device index `" << param << "`:"
<< e.what() << std::endl;
std::exit(EXIT_FAILURE);
std::cout << "Unable to parse/select device index `" << param << "`:" << e.what() << std::endl;
std::cout << "Attempting to match device with substring `" << param << "`" << std::endl;

auto matching = std::find_if(devices.begin(), devices.end(), [param](const cl::sycl::device &device) {
return device.get_info<cl::sycl::info::device::name>().find(param) != std::string::npos;
});
if (matching != devices.end()) {
config.device = *matching;
std::cout << "Using first device matching substring `" << param << "`" << std::endl;
} else if (devices.size() == 1)
std::cerr << "No matching device but there's only one device, will be using that anyway" << std::endl;
else {
std::cerr << "No matching devices" << std::endl;
std::exit(EXIT_FAILURE);
}
}

});
} else if (arg == "--file") {
readParam(i, "--file specified but no file was given", [&config](const auto &param) {
Expand Down Expand Up @@ -158,9 +174,9 @@ initialise(parallel_ &parallel, const std::vector<std::string> &args) {

if (parallel.boss) {
g_out << "Clover Version " << g_version << std::endl
<< "Kokkos Version" << std::endl
<< "Task Count " << parallel.max_task << std::endl
<< std::endl;
<< "Kokkos Version" << std::endl
<< "Task Count " << parallel.max_task << std::endl
<< std::endl;

std::cout << "Output file clover.out opened. All output will go there." << std::endl;
}
Expand All @@ -181,16 +197,16 @@ initialise(parallel_ &parallel, const std::vector<std::string> &args) {
for (size_t i = 0; i < devices.size(); ++i) printSimple(devices[i], i);

std::cout << "Using SYCL device: "
<< selectedDevice.get_info<cl::sycl::info::device::name>()
<< "("
<< deviceName(selectedDevice.get_info<cl::sycl::info::device::device_type>())
<< ")"
<< std::endl;
<< selectedDevice.get_info<cl::sycl::info::device::name>()
<< "("
<< deviceName(selectedDevice.get_info<cl::sycl::info::device::device_type>())
<< ")"
<< std::endl;

std::ifstream g_in;
if (parallel.boss) {
g_out << "Clover will run from the following input:-" << std::endl
<< std::endl;
<< std::endl;

if (!args.empty()) {
std::cout << "Args:";
Expand Down Expand Up @@ -231,8 +247,8 @@ initialise(parallel_ &parallel, const std::vector<std::string> &args) {

if (parallel.boss) {
g_out << std::endl
<< "Initialising and generating" << std::endl
<< std::endl;
<< "Initialising and generating" << std::endl
<< std::endl;
}

read_input(g_in, parallel, config);
Expand Down
1 change: 1 addition & 0 deletions src/start.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include "cxx14_compat.hpp"
#include <string>
#include <sstream>
#include <algorithm>

extern std::ostream g_out;

Expand Down
6 changes: 6 additions & 0 deletions src/sycl_reduction.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,14 @@ namespace clover {
cl::sycl::nd_range<1>(dot_num_groups * dot_wgsize, dot_wgsize),
[=](cl::sycl::nd_item<1> item) {

#ifdef USE_PRE_SYCL121R3
size_t i = item.get_global(0);
size_t li = item.get_local(0);
#else
size_t i = item.get_global_id(0);
size_t li = item.get_local_id(0);
#endif

size_t global_size = item.get_global_range()[0];

empty(ctx, cl::sycl::id<1>(li));
Expand Down

0 comments on commit 64e3693

Please sign in to comment.