[SYCL] Impl of fallback group sort in device code

clang/include/clang/Driver/Options.td

@@ -6900,14 +6900,14 @@ def fno_sycl_dead_args_optimization : Flag<["-"], "fno-sycl-dead-args-optimizati
   HelpText<"Disables elimination of DPC++ dead kernel arguments">;
 def fsycl_device_lib_EQ : CommaJoined<["-"], "fsycl-device-lib=">,
   Flags<[NoXarchOption]>,
-  Values<"libc,libm-fp32,libm-fp64,libimf-fp32,libimf-fp64,libimf-bf16,all">,
+  Values<"libc,libm-fp32,libm-fp64,libimf-fp32,libimf-fp64,libimf-bf16,libgsort-fp32,all">,
   HelpText<"Control inclusion of device libraries into device binary linkage. "
            "Valid arguments are libc, libm-fp32, libm-fp64, libimf-fp32, "
-           "libimf-fp64, libimf-bf16, all">;
+           "libimf-fp64, libimf-bf16, libgsort-fp32, all">;
 def fno_sycl_device_lib_EQ : CommaJoined<["-"], "fno-sycl-device-lib=">,
-  Flags<[NoXarchOption]>, Values<"libc, libm-fp32, libm-fp64, all">,
+  Flags<[NoXarchOption]>, Values<"libc, libm-fp32, libm-fp64, libgsort-fp32, all">,
   HelpText<"Control exclusion of device libraries from device binary linkage. "
-           "Valid arguments are libc, libm-fp32, libm-fp64, all">;
+           "Valid arguments are libc, libm-fp32, libm-fp64, libgsort-fp32, all">;
 def fsycl_device_lib_jit_link : Flag<["-"], "fsycl-device-lib-jit-link">,
   HelpText<"Enables sycl device library jit link (experimental)">;
 def fno_sycl_device_lib_jit_link : Flag<["-"], "fno-sycl-device-lib-jit-link">,

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -452,9 +452,9 @@ SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
 
   // Currently, all SYCL device libraries will be linked by default.
   llvm::StringMap<bool> DeviceLibLinkInfo = {
-      {"libc", true},          {"libm-fp32", true},   {"libm-fp64", true},
-      {"libimf-fp32", true},   {"libimf-fp64", true}, {"libimf-bf16", true},
-      {"libm-bfloat16", true}, {"internal", true}};
+      {"libc", true},          {"libm-fp32", true},     {"libm-fp64", true},
+      {"libimf-fp32", true},   {"libimf-fp64", true},   {"libimf-bf16", true},
+      {"libm-bfloat16", true}, {"libgsort-fp32", true}, {"internal", true}};
 
   // If -fno-sycl-device-lib is specified, its values will be used to exclude
   // linkage of libraries specified by DeviceLibLinkInfo. Linkage of "internal"
@@ -536,6 +536,7 @@ SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
       {"libsycl-fallback-complex-fp64", "libm-fp64"},
       {"libsycl-fallback-cmath", "libm-fp32"},
       {"libsycl-fallback-cmath-fp64", "libm-fp64"},
+      {"libsycl-fallback-gsort", "libgsort-fp32"},
       {"libsycl-fallback-imf", "libimf-fp32"},
       {"libsycl-fallback-imf-fp64", "libimf-fp64"},
       {"libsycl-fallback-imf-bf16", "libimf-bf16"}};
@@ -851,6 +852,7 @@ static llvm::SmallVector<StringRef, 16> SYCLDeviceLibList{
     "fallback-cmath-fp64",
     "fallback-complex",
     "fallback-complex-fp64",
+    "fallback-gsort",
     "fallback-imf",
     "fallback-imf-fp64",
     "fallback-imf-bf16",

clang/test/Driver/sycl-device-lib-win.cpp

@@ -33,6 +33,7 @@
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-complex-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-cmath.bc"
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-cmath-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-gsort.bc"
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf.bc"
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"
@@ -70,6 +71,7 @@
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-complex-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-cmath.bc"
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-cmath-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-gsort.bc"
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-imf.bc"
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"
@@ -93,6 +95,7 @@
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-complex-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-cmath.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-cmath-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-gsort.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-imf.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"
@@ -115,6 +118,7 @@
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: "{{.*}}libsycl-fallback-complex-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: "{{.*}}libsycl-fallback-cmath.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: "{{.*}}libsycl-fallback-cmath-fp64.bc"
+// SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: "{{.*}}libsycl-fallback-gsort.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: "{{.*}}libsycl-fallback-imf.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"
@@ -172,6 +176,7 @@
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-complex-fp64.bc"
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-cmath.bc"
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-cmath-fp64.bc"
+// SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-gsort.bc"
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-imf.bc"
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-imf-fp64.bc"
 // SYCL_LLVM_LINK_DEVICE_LIB-SAME: "{{.*}}libsycl-fallback-imf-bf16.bc"

clang/test/Driver/sycl-device-lib.cpp

@@ -33,6 +33,7 @@
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-complex-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-cmath.new.o
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-cmath-fp64.new.o
+// SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-gsort.new.o
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf.new.o
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_DEFAULT-SAME: {{.*}}libsycl-fallback-imf-bf16.new.o
@@ -70,11 +71,12 @@
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-complex-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-cmath.new.o
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-cmath-fp64.new.o
+// SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-gsort.new.o
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-imf.new.o
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-imf-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_WITH_FP64-SAME: {{.*}}libsycl-fallback-imf-bf16.new.o
-/// ###########################################################################
 
+/// ###########################################################################
 /// test behavior of -fno-sycl-device-lib=libc
 // RUN: %clangxx -fsycl --offload-new-driver %s -fno-sycl-device-lib=libc --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_LIB_LINK_NO_LIBC
@@ -93,11 +95,12 @@
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-complex-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-cmath.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-cmath-fp64.new.o
+// SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-gsort.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-imf.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-imf-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBC-SAME: {{.*}}libsycl-fallback-imf-bf16.new.o
-/// ###########################################################################
 
+/// ###########################################################################
 /// test behavior of -fno-sycl-device-lib=libm-fp32,libm-fp64
 // RUN: %clangxx -fsycl --offload-new-driver %s -fno-sycl-device-lib=libm-fp32,libm-fp64 --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_LIB_LINK_NO_LIBM
@@ -116,11 +119,12 @@
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: {{.*}}libsycl-fallback-complex-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: {{.*}}libsycl-fallback-cmath.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-NOT: {{.*}}libsycl-fallback-cmath-fp64.new.o
+// SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: {{.*}}libsycl-fallback-gsort.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: {{.*}}libsycl-fallback-imf.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: {{.*}}libsycl-fallback-imf-fp64.new.o
 // SYCL_DEVICE_LIB_LINK_NO_LIBM-SAME: {{.*}}libsycl-fallback-imf-bf16.new.o
-/// ###########################################################################
 
+/// ###########################################################################
 /// test behavior of disabling all device libraries
 // RUN: %clangxx -fsycl --offload-new-driver %s -fno-sycl-device-lib=libc,libm-fp32 --sysroot=%S/Inputs/SYCL -### 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_LIB_LINK_NO_DEVICE_LIB
@@ -152,6 +156,7 @@
 // SYCL_DEVICE_LIB_INVALID_VALUE: error: unsupported argument '[[Val]]' to option '-fsycl-device-lib='
 // SYCL_NO_DEVICE_LIB_INVALID_VALUE: error: unsupported argument '[[Val]]' to option '-fno-sycl-device-lib='
 
+
 /// ###########################################################################
 /// test behavior of libsycl-asan.o linking when -fsanitize=address is available
 // RUN: %clangxx -fsycl --offload-new-driver %s --sysroot=%S/Inputs/SYCL -fsanitize=address -### 2>&1 \
@@ -168,8 +173,8 @@
 // RUN:   | FileCheck %s -check-prefix=SYCL_DEVICE_ASAN_MACRO
 // SYCL_DEVICE_LIB_ASAN: clang-linker-wrapper{{.*}} "-sycl-device-libraries
 // SYCL_DEVICE_LIB_ASAN: {{.*}}libsycl-crt.new.o
-// SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-complex.
-// SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-complex-fp64.
+// SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-complex.new.o
+// SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-complex-fp64.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-cmath.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-cmath-fp64.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-imf.new.o
@@ -181,6 +186,7 @@
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-complex-fp64.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-cmath.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-cmath-fp64.new.o
+// SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-gsort.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-imf.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-imf-fp64.new.o
 // SYCL_DEVICE_LIB_ASAN-SAME: {{.*}}libsycl-fallback-imf-bf16.new.o

clang/test/Driver/sycl-offload-new-driver.c

@@ -34,7 +34,7 @@
 // RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver \
 // RUN:          --sysroot=%S/Inputs/SYCL -### %s 2>&1 \
 // RUN:   | FileCheck -check-prefix WRAPPER_OPTIONS %s
-// WRAPPER_OPTIONS: clang-linker-wrapper{{.*}} "-sycl-device-libraries=libsycl-crt.new.o,libsycl-complex.new.o,libsycl-complex-fp64.new.o,libsycl-cmath.new.o,libsycl-cmath-fp64.new.o,libsycl-imf.new.o,libsycl-imf-fp64.new.o,libsycl-imf-bf16.new.o,libsycl-fallback-cassert.new.o,libsycl-fallback-cstring.new.o,libsycl-fallback-complex.new.o,libsycl-fallback-complex-fp64.new.o,libsycl-fallback-cmath.new.o,libsycl-fallback-cmath-fp64.new.o,libsycl-fallback-imf.new.o,libsycl-fallback-imf-fp64.new.o,libsycl-fallback-imf-bf16.new.o,libsycl-itt-user-wrappers.new.o,libsycl-itt-compiler-wrappers.new.o,libsycl-itt-stubs.new.o"
+// WRAPPER_OPTIONS: clang-linker-wrapper{{.*}} "-sycl-device-libraries=libsycl-crt.new.o,libsycl-complex.new.o,libsycl-complex-fp64.new.o,libsycl-cmath.new.o,libsycl-cmath-fp64.new.o,libsycl-imf.new.o,libsycl-imf-fp64.new.o,libsycl-imf-bf16.new.o,libsycl-fallback-cassert.new.o,libsycl-fallback-cstring.new.o,libsycl-fallback-complex.new.o,libsycl-fallback-complex-fp64.new.o,libsycl-fallback-cmath.new.o,libsycl-fallback-cmath-fp64.new.o,libsycl-fallback-gsort.new.o,libsycl-fallback-imf.new.o,libsycl-fallback-imf-fp64.new.o,libsycl-fallback-imf-bf16.new.o,libsycl-itt-user-wrappers.new.o,libsycl-itt-compiler-wrappers.new.o,libsycl-itt-stubs.new.o"
 // WRAPPER_OPTIONS-SAME: "-sycl-device-library-location={{.*}}/lib"
 
 /// Verify phases used to generate SPIR-V instead of LLVM-IR

libdevice/atomic.hpp

@@ -7,78 +7,9 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <cstdint>
-
-#include "device.h"
+#include "spirv_decls.hpp"
 
 #if defined(__SPIR__) || defined(__SPIRV__)
-
-#define SPIR_GLOBAL __attribute__((opencl_global))
-
-namespace __spv {
-struct Scope {
-
-  enum Flag : uint32_t {
-    CrossDevice = 0,
-    Device = 1,
-    Workgroup = 2,
-    Subgroup = 3,
-    Invocation = 4,
-  };
-
-  constexpr Scope(Flag flag) : flag_value(flag) {}
-
-  constexpr operator uint32_t() const { return flag_value; }
-
-  Flag flag_value;
-};
-
-struct MemorySemanticsMask {
-
-  enum Flag : uint32_t {
-    None = 0x0,
-    Acquire = 0x2,
-    Release = 0x4,
-    AcquireRelease = 0x8,
-    SequentiallyConsistent = 0x10,
-    UniformMemory = 0x40,
-    SubgroupMemory = 0x80,
-    WorkgroupMemory = 0x100,
-    CrossWorkgroupMemory = 0x200,
-    AtomicCounterMemory = 0x400,
-    ImageMemory = 0x800,
-  };
-
-  constexpr MemorySemanticsMask(Flag flag) : flag_value(flag) {}
-
-  constexpr operator uint32_t() const { return flag_value; }
-
-  Flag flag_value;
-};
-} // namespace __spv
-
-extern DEVICE_EXTERNAL int
-__spirv_AtomicCompareExchange(int SPIR_GLOBAL *, __spv::Scope::Flag,
-                              __spv::MemorySemanticsMask::Flag,
-                              __spv::MemorySemanticsMask::Flag, int, int);
-
-extern DEVICE_EXTERNAL int
-__spirv_AtomicCompareExchange(int *, __spv::Scope::Flag,
-                              __spv::MemorySemanticsMask::Flag,
-                              __spv::MemorySemanticsMask::Flag, int, int);
-
-extern DEVICE_EXTERNAL int __spirv_AtomicLoad(const int SPIR_GLOBAL *,
-                                              __spv::Scope::Flag,
-                                              __spv::MemorySemanticsMask::Flag);
-
-extern DEVICE_EXTERNAL void
-__spirv_AtomicStore(int SPIR_GLOBAL *, __spv::Scope::Flag,
-                    __spv::MemorySemanticsMask::Flag, int);
-
-extern DEVICE_EXTERNAL void
-__spirv_AtomicStore(int *, __spv::Scope::Flag, __spv::MemorySemanticsMask::Flag,
-                    int);
-
 /// Atomically set the value in *Ptr with Desired if and only if it is Expected
 /// Return the value which already was in *Ptr
 static inline int atomicCompareAndSet(SPIR_GLOBAL int *Ptr, int Desired,

libdevice/cmake/modules/SYCLLibdevice.cmake

@@ -218,9 +218,8 @@ function(add_devicelibs filename)
       DEPENDENCIES ${ARG_DEPENDENCIES}
       EXTRA_OPTS ${ARG_EXTRA_OPTS} ${bc_device_compile_opts}
                  ${compile_opts_${arch}})
-
-    append_to_property(${bc_binary_dir}/${filename}-${arch}.bc
-      PROPERTY_NAME BC_DEVICE_LIBS_${arch})
+      append_to_property(${bc_binary_dir}/${filename}-${arch}.bc
+        PROPERTY_NAME BC_DEVICE_LIBS_${arch})
   endforeach()
 endfunction()
 
@@ -233,7 +232,7 @@ set(itt_obj_deps device_itt.h spirv_vars.h device.h sycl-compiler)
 set(bfloat16_obj_deps sycl-headers sycl-compiler)
 if (NOT MSVC AND UR_SANITIZER_INCLUDE_DIR)
   set(asan_obj_deps
-    device.h atomic.hpp spirv_vars.h
+    device.h atomic.hpp spirv_vars.h spirv_decls.hpp
     ${UR_SANITIZER_INCLUDE_DIR}/asan/asan_libdevice.hpp
     include/asan_rtl.hpp
     include/spir_global_var.hpp
@@ -296,6 +295,7 @@ if (NOT MSVC AND UR_SANITIZER_INCLUDE_DIR)
     include/spir_global_var.hpp
     sycl-compiler)
 endif()
+set(gsort_obj_deps device.h spirv_decls.hpp spirv_vars.h group_helper.hpp sort_helper.hpp sycl-compiler)
 
 if("native_cpu" IN_LIST SYCL_ENABLE_BACKENDS)
   if (NOT DEFINED NATIVE_CPU_DIR)
@@ -416,6 +416,11 @@ add_devicelibs(libsycl-fallback-bfloat16
 add_devicelibs(libsycl-native-bfloat16
   SRC bfloat16_wrapper.cpp
   DEPENDENCIES ${bfloat16_obj_deps})
+add_devicelibs(libsycl-fallback-gsort
+  SRC fallback-gsort.cpp
+  DEPENDENCIES ${gsort_obj_deps}
+  EXTRA_OPTS -fno-sycl-instrument-device-code
+  SKIP_ARCHS amdgcn-amd-amdhsa)
 
 # Create dependency and source lists for Intel math function libraries.
 file(MAKE_DIRECTORY ${obj_binary_dir}/libdevice)