Tidy and move common adapter code to ur_common

callumfare · Jan 9, 2024 · d867d48 · d867d48
1 parent 2d42d77
commit d867d48
Show file tree

Hide file tree

Showing 99 changed files with 641 additions and 672 deletions.
diff --git a/source/adapters/cuda/CMakeLists.txt b/source/adapters/cuda/CMakeLists.txt
@@ -41,8 +41,6 @@ add_ur_adapter(${TARGET_NAME}
     ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp
 )
 
 set_target_properties(${TARGET_NAME} PROPERTIES
@@ -83,7 +81,3 @@ target_link_libraries(${TARGET_NAME} PRIVATE
     Threads::Threads
     cudadrv
 )
-
-target_include_directories(${TARGET_NAME} PRIVATE
-    "${CMAKE_CURRENT_SOURCE_DIR}/../../"
-)
diff --git a/source/adapters/cuda/adapter.cpp b/source/adapters/cuda/adapter.cpp
@@ -67,7 +67,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGetInfo(ur_adapter_handle_t,
                                                      size_t propSize,
                                                      void *pPropValue,
                                                      size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_ADAPTER_INFO_BACKEND:

diff --git a/source/adapters/cuda/command_buffer.hpp b/source/adapters/cuda/command_buffer.hpp
@@ -8,7 +8,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <ur/ur.hpp>
 #include <ur_api.h>
 
 #include "context.hpp"

diff --git a/source/adapters/cuda/common.hpp b/source/adapters/cuda/common.hpp
@@ -10,7 +10,7 @@
 #pragma once
 
 #include <cuda.h>
-#include <ur/ur.hpp>
+#include <ur_util.hpp>
 
 ur_result_t mapErrorUR(CUresult Result);
 

diff --git a/source/adapters/cuda/context.cpp b/source/adapters/cuda/context.cpp
@@ -68,7 +68,7 @@ urContextCreate(uint32_t DeviceCount, const ur_device_handle_t *phDevices,
 UR_APIEXPORT ur_result_t UR_APICALL urContextGetInfo(
     ur_context_handle_t hContext, ur_context_info_t ContextInfoType,
     size_t propSize, void *pContextInfo, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pContextInfo, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pContextInfo, pPropSizeRet);
 
   switch (static_cast<uint32_t>(ContextInfoType)) {
   case UR_CONTEXT_INFO_NUM_DEVICES:

diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp
@@ -42,7 +42,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
                                                     size_t propSize,
                                                     void *pPropValue,
                                                     size_t *pPropSizeRet) try {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   static constexpr uint32_t MaxWorkItemDimensions = 3u;
 

diff --git a/source/adapters/cuda/device.hpp b/source/adapters/cuda/device.hpp
@@ -9,7 +9,7 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 #include "common.hpp"
 

diff --git a/source/adapters/cuda/event.cpp b/source/adapters/cuda/event.cpp
@@ -162,7 +162,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent,
                                                    size_t propValueSize,
                                                    void *pPropValue,
                                                    size_t *pPropValueSizeRet) {
-  UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
+  ur::ReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
 
   switch (propName) {
   case UR_EVENT_INFO_COMMAND_QUEUE:
@@ -187,7 +187,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent,
 UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
     ur_event_handle_t hEvent, ur_profiling_info_t propName,
     size_t propValueSize, void *pPropValue, size_t *pPropValueSizeRet) {
-  UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
+  ur::ReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
 
   ur_queue_handle_t Queue = hEvent->getQueue();
   if (Queue == nullptr || !(Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE)) {

diff --git a/source/adapters/cuda/event.hpp b/source/adapters/cuda/event.hpp
@@ -10,7 +10,7 @@
 #pragma once
 
 #include <cuda.h>
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 #include "common.hpp"
 #include "queue.hpp"

diff --git a/source/adapters/cuda/image.cpp b/source/adapters/cuda/image.cpp
@@ -20,7 +20,6 @@
 #include "memory.hpp"
 #include "queue.hpp"
 #include "sampler.hpp"
-#include "ur/ur.hpp"
 #include "ur_api.h"
 
 ur_result_t urCalculateNumChannels(ur_image_channel_order_t order,

diff --git a/source/adapters/cuda/kernel.cpp b/source/adapters/cuda/kernel.cpp
@@ -62,7 +62,7 @@ UR_APIEXPORT ur_result_t UR_APICALL
 urKernelGetGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice,
                      ur_kernel_group_info_t propName, size_t propSize,
                      void *pPropValue, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_KERNEL_GROUP_INFO_GLOBAL_WORK_SIZE: {
@@ -205,7 +205,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetInfo(ur_kernel_handle_t hKernel,
                                                     size_t propSize,
                                                     void *pKernelInfo,
                                                     size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pKernelInfo, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pKernelInfo, pPropSizeRet);
 
   switch (propName) {
   case UR_KERNEL_INFO_FUNCTION_NAME:
@@ -237,7 +237,7 @@ UR_APIEXPORT ur_result_t UR_APICALL
 urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice,
                         ur_kernel_sub_group_info_t propName, size_t propSize,
                         void *pPropValue, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
   switch (propName) {
   case UR_KERNEL_SUB_GROUP_INFO_MAX_SUB_GROUP_SIZE: {
     // Sub-group size is equivalent to warp size

diff --git a/source/adapters/cuda/memory.cpp b/source/adapters/cuda/memory.cpp
@@ -175,7 +175,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemGetInfo(ur_mem_handle_t hMemory,
                                                  size_t *pPropSizeRet) {
   UR_ASSERT(hMemory->isBuffer(), UR_RESULT_ERROR_INVALID_MEM_OBJECT);
 
-  UrReturnHelper ReturnValue(propSize, pMemInfo, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pMemInfo, pPropSizeRet);
 
   ScopedContext Active(hMemory->getContext());
 

diff --git a/source/adapters/cuda/physical_mem.hpp b/source/adapters/cuda/physical_mem.hpp
@@ -9,7 +9,7 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 #include <cuda.h>
 

diff --git a/source/adapters/cuda/platform.cpp b/source/adapters/cuda/platform.cpp
@@ -22,7 +22,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformGetInfo(
     size_t Size, void *pPlatformInfo, size_t *pSizeRet) {
 
   UR_ASSERT(hPlatform, UR_RESULT_ERROR_INVALID_NULL_HANDLE);
-  UrReturnHelper ReturnValue(Size, pPlatformInfo, pSizeRet);
+  ur::ReturnHelper ReturnValue(Size, pPlatformInfo, pSizeRet);
 
   switch (PlatformInfoType) {
   case UR_PLATFORM_INFO_NAME:

diff --git a/source/adapters/cuda/platform.hpp b/source/adapters/cuda/platform.hpp
@@ -9,7 +9,8 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <ur/ur.hpp>
+#include <memory>
+#include <ur_api.h>
 #include <vector>
 
 struct ur_platform_handle_t_ {

diff --git a/source/adapters/cuda/program.cpp b/source/adapters/cuda/program.cpp
@@ -338,7 +338,7 @@ urProgramGetBuildInfo(ur_program_handle_t hProgram, ur_device_handle_t hDevice,
                       void *pPropValue, size_t *pPropSizeRet) {
   std::ignore = hDevice;
 
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_PROGRAM_BUILD_INFO_STATUS: {
@@ -360,7 +360,7 @@ urProgramGetBuildInfo(ur_program_handle_t hProgram, ur_device_handle_t hDevice,
 UR_APIEXPORT ur_result_t UR_APICALL
 urProgramGetInfo(ur_program_handle_t hProgram, ur_program_info_t propName,
                  size_t propSize, void *pProgramInfo, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pProgramInfo, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pProgramInfo, pPropSizeRet);
 
   switch (propName) {
   case UR_PROGRAM_INFO_REFERENCE_COUNT:

diff --git a/source/adapters/cuda/queue.cpp b/source/adapters/cuda/queue.cpp
@@ -291,7 +291,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue,
                                                    size_t propValueSize,
                                                    void *pPropValue,
                                                    size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propValueSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_QUEUE_INFO_CONTEXT:

diff --git a/source/adapters/cuda/queue.hpp b/source/adapters/cuda/queue.hpp
@@ -9,10 +9,12 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 #include <algorithm>
+#include <atomic>
 #include <cuda.h>
+#include <mutex>
 #include <vector>
 
 using ur_stream_guard_ = std::unique_lock<std::mutex>;

diff --git a/source/adapters/cuda/sampler.cpp b/source/adapters/cuda/sampler.cpp
@@ -49,7 +49,7 @@ urSamplerCreate(ur_context_handle_t hContext, const ur_sampler_desc_t *pDesc,
 UR_APIEXPORT ur_result_t UR_APICALL
 urSamplerGetInfo(ur_sampler_handle_t hSampler, ur_sampler_info_t propName,
                  size_t propValueSize, void *pPropValue, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propValueSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_SAMPLER_INFO_REFERENCE_COUNT:

diff --git a/source/adapters/cuda/sampler.hpp b/source/adapters/cuda/sampler.hpp
@@ -8,7 +8,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
+
+#include <atomic>
 
 /// Implementation of samplers for CUDA
 ///

diff --git a/source/adapters/cuda/usm.cpp b/source/adapters/cuda/usm.cpp
@@ -195,7 +195,7 @@ urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem,
                      void *pPropValue, size_t *pPropValueSizeRet) {
   ur_result_t Result = UR_RESULT_SUCCESS;
 
-  UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
+  ur::ReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet);
 
   try {
     ScopedContext Active(hContext);
@@ -489,7 +489,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMPoolGetInfo(
     size_t *pPropSizeRet ///< [out][optional] size in bytes returned in pool
                          ///< property value
 ) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_USM_POOL_INFO_REFERENCE_COUNT: {

diff --git a/source/adapters/cuda/usm_p2p.cpp b/source/adapters/cuda/usm_p2p.cpp
@@ -42,7 +42,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp(
     ur_exp_peer_info_t propName, size_t propSize, void *pPropValue,
     size_t *pPropSizeRet) {
 
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   int value;
   CUdevice_P2PAttribute cu_attr;

diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp
@@ -20,7 +20,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo(
     ur_context_handle_t hContext, ur_device_handle_t hDevice,
     ur_virtual_mem_granularity_info_t propName, size_t propSize,
     void *pPropValue, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   ScopedContext Active(hContext);
   switch (propName) {
@@ -108,7 +108,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(
     ur_context_handle_t hContext, const void *pStart,
     [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName,
     size_t propSize, void *pPropValue, size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   ScopedContext Active(hContext);
   switch (propName) {

diff --git a/source/adapters/hip/CMakeLists.txt b/source/adapters/hip/CMakeLists.txt
@@ -74,8 +74,6 @@ add_ur_adapter(${TARGET_NAME}
     ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp
 )
 
 if(NOT MSVC)
@@ -175,7 +173,3 @@ elseif("${UR_HIP_PLATFORM}" STREQUAL "NVIDIA")
 else()
     message(FATAL_ERROR "Unspecified UR HIP platform please set UR_HIP_PLATFORM to 'AMD' or 'NVIDIA'")
 endif()
-
-target_include_directories(${TARGET_NAME} PRIVATE
-    "${CMAKE_CURRENT_SOURCE_DIR}/../../"
-)
diff --git a/source/adapters/hip/adapter.cpp b/source/adapters/hip/adapter.cpp
@@ -56,7 +56,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGetInfo(ur_adapter_handle_t,
                                                      size_t propSize,
                                                      void *pPropValue,
                                                      size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (propName) {
   case UR_ADAPTER_INFO_BACKEND:

diff --git a/source/adapters/hip/command_buffer.hpp b/source/adapters/hip/command_buffer.hpp
@@ -8,7 +8,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 /// Stub implementation of command-buffers for HIP
 

diff --git a/source/adapters/hip/common.hpp b/source/adapters/hip/common.hpp
@@ -17,7 +17,8 @@
 #endif
 #endif
 #include <hip/hip_runtime.h>
-#include <ur/ur.hpp>
+#include <ur_api.h>
+#include <ur_util.hpp>
 
 // Before ROCm 6, hipify doesn't support cuArrayGetDescriptor, on AMD the
 // hipArray can just be indexed, but on NVidia it is an opaque type and needs to

diff --git a/source/adapters/hip/context.cpp b/source/adapters/hip/context.cpp
@@ -72,7 +72,7 @@ UR_APIEXPORT ur_result_t UR_APICALL
 urContextGetInfo(ur_context_handle_t hContext, ur_context_info_t propName,
                  size_t propSize, void *pPropValue, size_t *pPropSizeRet) {
 
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   switch (uint32_t{propName}) {
   case UR_CONTEXT_INFO_NUM_DEVICES:

diff --git a/source/adapters/hip/device.cpp b/source/adapters/hip/device.cpp
@@ -25,7 +25,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
                                                     size_t propSize,
                                                     void *pPropValue,
                                                     size_t *pPropSizeRet) {
-  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
+  ur::ReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
 
   static constexpr uint32_t MaxWorkItemDimensions = 3u;
 

diff --git a/source/adapters/hip/device.hpp b/source/adapters/hip/device.hpp
@@ -11,7 +11,7 @@
 
 #include "common.hpp"
 
-#include <ur/ur.hpp>
+#include <ur_api.h>
 
 /// UR device mapping to a hipDevice_t.
 /// Includes an observer pointer to the platform,

diff --git a/source/adapters/hip/enqueue.cpp b/source/adapters/hip/enqueue.cpp
@@ -146,7 +146,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferRead(
 
   std::unique_ptr<ur_event_handle_t_> RetImplEvent{nullptr};
 
-  ur_lock MemoryMigrationLock{hBuffer->MemoryMigrationMutex};
+  ur::Lock MemoryMigrationLock{hBuffer->MemoryMigrationMutex};
   auto Device = hQueue->getDevice();
   hipStream_t HIPStream = hQueue->getNextTransferStream();
 
@@ -215,7 +215,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunch(
 
   std::vector<ur_event_handle_t> DepEvents(
       phEventWaitList, phEventWaitList + numEventsInWaitList);
-  std::vector<std::pair<ur_mem_handle_t, ur_lock>> MemMigrationLocks;
+  std::vector<std::pair<ur_mem_handle_t, ur::Lock>> MemMigrationLocks;
 
   // phEventWaitList only contains events that are handed to UR by the SYCL
   // runtime. However since UR handles memory dependencies within a context
@@ -238,8 +238,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunch(
                          [MemArg](auto &Lock) {
                            return Lock.first == MemArg.Mem;
                          }) == MemMigrationLocks.end())
-          MemMigrationLocks.emplace_back(
-              std::pair{MemArg.Mem, ur_lock{MemArg.Mem->MemoryMigrationMutex}});
+          MemMigrationLocks.emplace_back(std::pair{
+              MemArg.Mem, ur::Lock{MemArg.Mem->MemoryMigrationMutex}});
       }
     }
   }
@@ -593,7 +593,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferReadRect(
   std::unique_ptr<ur_event_handle_t_> RetImplEvent{nullptr};
 
   ur_result_t Result = UR_RESULT_SUCCESS;
-  ur_lock MemoryMigrationLock(hBuffer->MemoryMigrationMutex);
+  ur::Lock MemoryMigrationLock(hBuffer->MemoryMigrationMutex);
   auto Device = hQueue->getDevice();
   hipStream_t HIPStream = hQueue->getNextTransferStream();
 
@@ -987,7 +987,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageRead(
     const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) {
   UR_ASSERT(hImage->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT);
 
-  ur_lock MemoryMigrationLock{hImage->MemoryMigrationMutex};
+  ur::Lock MemoryMigrationLock{hImage->MemoryMigrationMutex};
   auto Device = hQueue->getDevice();
   hipStream_t HIPStream = hQueue->getNextTransferStream();