From 64e36935051b372289e8794fd713af4213340fc5 Mon Sep 17 00:00:00 2001 From: Tom Lin Date: Sun, 10 Jan 2021 22:59:43 +0000 Subject: [PATCH] Add support for substring based device selection Add support for pre-SYCL1.2.1R3 standard Add support for custom dpcpp paths Add move semantic handling workaround in field_summary for old ComputeCpp versions Fixed COMPUTECPP_USER_FLAGS from being overwritten Fixed CXX_EXTRA_FLAGS/CXX_EXTRA_LINKER_FLAGS space handling, closes #4 --- CMakeLists.txt | 79 ++++++++++++++++++++++++++++++++--------- src/field_summary.cpp | 4 ++- src/initialise.cpp | 80 +++++++++++++++++++++++++----------------- src/start.cpp | 1 + src/sycl_reduction.hpp | 6 ++++ 5 files changed, 120 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9635546..3e05770 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,8 @@ cmake_minimum_required(VERSION 3.12 FATAL_ERROR) project(cloverleaf_sycl) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) if (SYCL_RUNTIME) @@ -15,29 +16,50 @@ if (SYCL_RUNTIME) if (NOT HIPSYCL_INSTALL_DIR) message(FATAL_ERROR "HIPSYCL_INSTALL_DIR is undefined") endif () - set(hipSYCL_DIR ${HIPSYCL_INSTALL_DIR}/lib/cmake) find_package(hipSYCL CONFIG REQUIRED) set(EXTRA_FLAGS -Wno-sign-compare -Wno-stringop-truncation) + elseif (${SYCL_RUNTIME} STREQUAL "HIPSYCL-NEXT") + + if (NOT HIPSYCL_INSTALL_DIR) + message(FATAL_ERROR "HIPSYCL_INSTALL_DIR is undefined") + endif () + set(CMAKE_CXX_STANDARD 17) + set(hipSYCL_DIR ${HIPSYCL_INSTALL_DIR}/lib/cmake/hipSYCL) + add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) + find_package(hipSYCL CONFIG REQUIRED) + set(EXTRA_FLAGS -Wno-sign-compare -Wno-stringop-truncation) elseif (${SYCL_RUNTIME} STREQUAL "COMPUTECPP") if (NOT ComputeCpp_DIR) message(FATAL_ERROR "ComputeCpp_DIR is undefined") endif () + set(OpenCL_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include) + add_definitions(-DCL_TARGET_OPENCL_VERSION=220) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) - set(COMPUTECPP_USER_FLAGS -O3 -fsycl-split-modules=20 -mllvm -inline-threshold=10000 -no-serial-memop) find_package(ComputeCpp REQUIRED) - # set(EXTRA_FLAGS -pedantic) + set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop) elseif (${SYCL_RUNTIME} STREQUAL "DPCPP") - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_COMPILER "dpcpp") + if (NOT DPCPP_BIN) + message(STATUS "DPCPP_BIN unspecified, using dpcpp on path") + set(CMAKE_CXX_COMPILER "dpcpp") + else() + set(CMAKE_CXX_COMPILER ${DPCPP_BIN}) + endif () + + if(DPCPP_INCLUDE) + include_directories(${DPCPP_INCLUDE}) + endif() + + add_definitions(-DCL_TARGET_OPENCL_VERSION=220) + set(CMAKE_CXX_STANDARD 14) set(EXTRA_FLAGS -pedantic) else () - message(FATAL_ERROR "SYCL_RUNTIME unsupported, must be one of HIPSYCL|COMPUTECPP|DPCPP, got ${SYCL_RUNTIME}") + message(FATAL_ERROR "SYCL_RUNTIME unsupported, must be one of HIPSYCL|HIPSYCL-NEXT|COMPUTECPP|DPCPP, got ${SYCL_RUNTIME}") endif () else () - message(FATAL_ERROR "SYCL_RUNTIME not defined, must be one of HIPSYCL|COMPUTECPP|DPCPP") + message(FATAL_ERROR "SYCL_RUNTIME not defined, must be one of HIPSYCL|HIPSYCL-NEXT|COMPUTECPP|DPCPP") endif () @@ -99,9 +121,12 @@ set(SOURCES src/visit.cpp) include_directories(src) -add_executable(clover_leaf ${SOURCES}) +add_executable(cloverleaf ${SOURCES}) + +separate_arguments(CXX_EXTRA_FLAGS) +separate_arguments(CXX_EXTRA_LINKER_FLAGS) -target_compile_options(clover_leaf +target_compile_options(cloverleaf PUBLIC -Wall -Wextra @@ -115,20 +140,40 @@ target_compile_options(clover_leaf ${EXTRA_FLAGS} ) + +if (DEFINED USE_PRE_SYCL121R3) + add_definitions(-DUSE_PRE_SYCL121R3) +endif () + + set(DEBUG_OPTIONS -O2 -fno-omit-frame-pointer ${CXX_EXTRA_FLAGS}) -set(RELEASE_OPTIONS -O3 ${CXX_EXTRA_FLAGS}) +set(RELEASE_OPTIONS -O3 ${CXX_EXTRA_FLAGS}) -target_link_libraries(clover_leaf PUBLIC ${MPI_C_LIB}) +target_link_libraries(cloverleaf PUBLIC ${MPI_C_LIB}) -target_compile_options(clover_leaf PUBLIC "$<$:${RELEASE_OPTIONS}>") -target_compile_options(clover_leaf PUBLIC "$<$:${RELEASE_OPTIONS}>") -target_compile_options(clover_leaf PUBLIC "$<$:${DEBUG_OPTIONS}>") +target_compile_options(cloverleaf PUBLIC "$<$:${RELEASE_OPTIONS}>") +target_compile_options(cloverleaf PUBLIC "$<$:${RELEASE_OPTIONS}>") +target_compile_options(cloverleaf PUBLIC "$<$:${DEBUG_OPTIONS}>") + +if (${CMAKE_VERSION} VERSION_LESS "3.13.0") + message(WARNING "target_link_options is only available in CMake >= 3.13.0, using fallback target_link_libraries, this may cause issues with some compilers") + message(WARNING "whitespaces are not supported for CXX_EXTRA_LINKER_FLAGS/CXX_EXTRA_FLAGS in this mode as they are treated as libraries arguments (CMake splits them)") + if (DEFINED CXX_EXTRA_LINKER_FLAGS) + list(APPEND EXTRA_LINK_FLAGS "-Wl,${CXX_EXTRA_LINKER_FLAGS}") + endif () + + target_link_libraries(cloverleaf PUBLIC ${EXTRA_LINK_FLAGS}) + target_link_libraries(cloverleaf PUBLIC ${CXX_EXTRA_FLAGS}) + +else () + target_link_options(cloverleaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS}) + target_link_options(cloverleaf PUBLIC ${CXX_EXTRA_FLAGS}) +endif () -target_link_options(clover_leaf PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS}) if (NOT ${SYCL_RUNTIME} STREQUAL "DPCPP") add_sycl_to_target( - TARGET clover_leaf + TARGET cloverleaf SOURCES ${SOURCES}) # must be the last endif () diff --git a/src/field_summary.cpp b/src/field_summary.cpp index 3378dbc..41ce718 100644 --- a/src/field_summary.cpp +++ b/src/field_summary.cpp @@ -52,7 +52,9 @@ struct captures { }; struct value_type { - double vol = 0.0, mass = 0.0, ie = 0.0, ke = 0.0, press = 0.0; + double vol, mass, ie, ke, press; + value_type(double vol = 0.0, double mass = 0.0, double ie = 0.0, double ke = 0.0, double press = 0.0) : + vol(vol), mass(mass), ie(ie), ke(ke), press(press) {} }; typedef clover::local_reducer ctx; diff --git a/src/initialise.cpp b/src/initialise.cpp index fa61e38..99a42ee 100644 --- a/src/initialise.cpp +++ b/src/initialise.cpp @@ -33,6 +33,7 @@ #include #include +#include extern std::ostream g_out; std::ofstream of; @@ -69,35 +70,35 @@ void printDetailed(const cl::sycl::device &device, size_t index) { std::cout << " + Device : " << device.get_info() << "\n"; std::cout << " - Index : " << index << "\n"; std::cout << " - Type : " << deviceName(type) << "\n"; - std::cout << " - Vendor : " << device.get_info()<< "\n"; + std::cout << " - Vendor : " << device.get_info() << "\n"; std::cout << " - Extensions : " << extensions.str() << "\n"; - std::cout << " + Platform : " << platform.get_info()<< "\n"; - std::cout << " - Vendor : " << platform.get_info()<< "\n"; - std::cout << " - Version : " << platform.get_info()<< "\n"; - std::cout << " - Profile : " << platform.get_info()<< "\n"; + std::cout << " + Platform : " << platform.get_info() << "\n"; + std::cout << " - Vendor : " << platform.get_info() << "\n"; + std::cout << " - Version : " << platform.get_info() << "\n"; + std::cout << " - Profile : " << platform.get_info() << "\n"; } void printSimple(const cl::sycl::device &device, size_t index) { std::cout << std::setw(3) << index << ". " - << device.get_info() - << "(" << deviceName(device.get_info()) << ")" - << std::endl; + << device.get_info() + << "(" << deviceName(device.get_info()) << ")" + << std::endl; } void printHelp(const std::string &name) { std::cout << std::endl; std::cout << "Usage: " << name << " [OPTIONS]\n\n" - << "Options:\n" - << " -h --help Print the message\n" - << " --list List available devices\n" - << " --list-detailed List available devices and capabilities\n" - << " --device Select device at INDEX from output of --list\n" - << " --file Custom clover.in file (defaults to clover.in if unspecified)\n" - << std::endl; + << "Options:\n" + << " -h --help Print the message\n" + << " --list List available devices\n" + << " --list-detailed List available devices and capabilities\n" + << " --device Select device at INDEX from output of --list\n" + << " --file Custom clover.in file (defaults to clover.in if unspecified)\n" + << std::endl; } RunConfig parseArgs(const std::vector &devices, - const std::vector &args) { + const std::vector &args) { const auto readParam = [&args](size_t current, const std::string &emptyMessage, auto map) { if (current + 1 < args.size()) { @@ -109,7 +110,7 @@ RunConfig parseArgs(const std::vector &devices, } }; - auto config = RunConfig{ "clover.in",devices[0]}; + auto config = RunConfig{"clover.in", devices[0]}; for (size_t i = 0; i < args.size(); ++i) { const auto arg = args[i]; @@ -124,12 +125,27 @@ RunConfig parseArgs(const std::vector &devices, std::exit(EXIT_SUCCESS); } else if (arg == "--device") { readParam(i, "--device specified but no size was given", [&config](const auto ¶m) { - try { config.device = cl::sycl::device::get_devices().at(std::stoul(param)); } + auto devices = cl::sycl::device::get_devices(); + + try { config.device = devices.at(std::stoul(param)); } catch (const std::exception &e) { - std::cerr << "failed to parse/select device index `" << param << "`:" - << e.what() << std::endl; - std::exit(EXIT_FAILURE); + std::cout << "Unable to parse/select device index `" << param << "`:" << e.what() << std::endl; + std::cout << "Attempting to match device with substring `" << param << "`" << std::endl; + + auto matching = std::find_if(devices.begin(), devices.end(), [param](const cl::sycl::device &device) { + return device.get_info().find(param) != std::string::npos; + }); + if (matching != devices.end()) { + config.device = *matching; + std::cout << "Using first device matching substring `" << param << "`" << std::endl; + } else if (devices.size() == 1) + std::cerr << "No matching device but there's only one device, will be using that anyway" << std::endl; + else { + std::cerr << "No matching devices" << std::endl; + std::exit(EXIT_FAILURE); + } } + }); } else if (arg == "--file") { readParam(i, "--file specified but no file was given", [&config](const auto ¶m) { @@ -158,9 +174,9 @@ initialise(parallel_ ¶llel, const std::vector &args) { if (parallel.boss) { g_out << "Clover Version " << g_version << std::endl - << "Kokkos Version" << std::endl - << "Task Count " << parallel.max_task << std::endl - << std::endl; + << "Kokkos Version" << std::endl + << "Task Count " << parallel.max_task << std::endl + << std::endl; std::cout << "Output file clover.out opened. All output will go there." << std::endl; } @@ -181,16 +197,16 @@ initialise(parallel_ ¶llel, const std::vector &args) { for (size_t i = 0; i < devices.size(); ++i) printSimple(devices[i], i); std::cout << "Using SYCL device: " - << selectedDevice.get_info() - << "(" - << deviceName(selectedDevice.get_info()) - << ")" - << std::endl; + << selectedDevice.get_info() + << "(" + << deviceName(selectedDevice.get_info()) + << ")" + << std::endl; std::ifstream g_in; if (parallel.boss) { g_out << "Clover will run from the following input:-" << std::endl - << std::endl; + << std::endl; if (!args.empty()) { std::cout << "Args:"; @@ -231,8 +247,8 @@ initialise(parallel_ ¶llel, const std::vector &args) { if (parallel.boss) { g_out << std::endl - << "Initialising and generating" << std::endl - << std::endl; + << "Initialising and generating" << std::endl + << std::endl; } read_input(g_in, parallel, config); diff --git a/src/start.cpp b/src/start.cpp index da792e3..620c28e 100644 --- a/src/start.cpp +++ b/src/start.cpp @@ -37,6 +37,7 @@ #include "cxx14_compat.hpp" #include #include +#include extern std::ostream g_out; diff --git a/src/sycl_reduction.hpp b/src/sycl_reduction.hpp index 6d8679a..7880829 100644 --- a/src/sycl_reduction.hpp +++ b/src/sycl_reduction.hpp @@ -93,8 +93,14 @@ namespace clover { cl::sycl::nd_range<1>(dot_num_groups * dot_wgsize, dot_wgsize), [=](cl::sycl::nd_item<1> item) { + #ifdef USE_PRE_SYCL121R3 + size_t i = item.get_global(0); + size_t li = item.get_local(0); + #else size_t i = item.get_global_id(0); size_t li = item.get_local_id(0); + #endif + size_t global_size = item.get_global_range()[0]; empty(ctx, cl::sycl::id<1>(li));