Merge pull request #1785 from callumfare/v0.9.7rc

Candidate for the v0.9.7 release tag

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR)
-project(unified-runtime VERSION 0.9.6)
+project(unified-runtime VERSION 0.9.7)
 
 include(GNUInstallDirs)
 include(CheckCXXSourceCompiles)

source/adapters/level_zero/adapter.cpp

@@ -33,6 +33,9 @@ ur_result_t initPlatforms(PlatformVec &platforms) noexcept try {
   for (uint32_t I = 0; I < ZeDriverCount; ++I) {
     auto platform = std::make_unique<ur_platform_handle_t_>(ZeDrivers[I]);
     UR_CALL(platform->initialize());
+    ZE2UR_CALL(zelLoaderTranslateHandle,
+               (ZEL_HANDLE_DRIVER, platform->ZeDriver,
+                (void **)&platform->ZeDriverHandleExpTranslated));
 
     // Save a copy in the cache for future uses.
     platforms.push_back(std::move(platform));

source/adapters/level_zero/adapter.hpp

@@ -10,6 +10,7 @@
 #pragma once
 
 #include <atomic>
+#include <loader/ze_loader.h>
 #include <mutex>
 #include <optional>
 #include <ur/ur.hpp>

source/adapters/level_zero/command_buffer.cpp

@@ -17,10 +17,12 @@
 ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_(
     ur_context_handle_t Context, ur_device_handle_t Device,
     ze_command_list_handle_t CommandList,
+    ze_command_list_handle_t CommandListTranslated,
     ze_command_list_handle_t CommandListResetEvents,
     ZeStruct<ze_command_list_desc_t> ZeDesc,
     const ur_exp_command_buffer_desc_t *Desc)
     : Context(Context), Device(Device), ZeCommandList(CommandList),
+      ZeComputeCommandListTranslated(CommandListTranslated),
       ZeCommandListResetEvents(CommandListResetEvents),
       ZeCommandListDesc(ZeDesc), ZeFencesList(), QueueProperties(),
       SyncPoints(), NextSyncPoint(0),
@@ -420,10 +422,15 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
   // command-buffers, then reusing them.
   ZE2UR_CALL(zeCommandListCreate, (Context->ZeContext, Device->ZeDevice,
                                    &ZeCommandListDesc, &ZeCommandList));
+
+  ze_command_list_handle_t ZeCommandListTranslated = nullptr;
+  ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_COMMAND_LIST, ZeCommandList,
+                                        (void **)&ZeCommandListTranslated));
+
   try {
     *CommandBuffer = new ur_exp_command_buffer_handle_t_(
-        Context, Device, ZeCommandList, ZeCommandListResetEvents,
-        ZeCommandListDesc, CommandBufferDesc);
+        Context, Device, ZeCommandList, ZeCommandListTranslated,
+        ZeCommandListResetEvents, ZeCommandListDesc, CommandBufferDesc);
   } catch (const std::bad_alloc &) {
     return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
   } catch (...) {
@@ -587,9 +594,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp(
     auto Plt = CommandBuffer->Context->getPlatform();
     UR_ASSERT(Plt->ZeMutableCmdListExt.Supported,
               UR_RESULT_ERROR_UNSUPPORTED_FEATURE);
-    ZE2UR_CALL(
-        Plt->ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp,
-        (CommandBuffer->ZeCommandList, &ZeMutableCommandDesc, &CommandId));
+    ZE2UR_CALL(Plt->ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp,
+               (CommandBuffer->ZeComputeCommandListTranslated,
+                &ZeMutableCommandDesc, &CommandId));
   }
   try {
     if (Command)
@@ -1296,8 +1303,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp(
   auto Plt = Command->CommandBuffer->Context->getPlatform();
   UR_ASSERT(Plt->ZeMutableCmdListExt.Supported,
             UR_RESULT_ERROR_UNSUPPORTED_FEATURE);
-  ZE2UR_CALL(Plt->ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp,
-             (CommandBuffer->ZeCommandList, &MutableCommandDesc));
+  ZE2UR_CALL(
+      Plt->ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp,
+      (CommandBuffer->ZeComputeCommandListTranslated, &MutableCommandDesc));
   ZE2UR_CALL(zeCommandListClose, (CommandBuffer->ZeCommandList));
 
   return UR_RESULT_SUCCESS;

source/adapters/level_zero/command_buffer.hpp

@@ -28,6 +28,7 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   ur_exp_command_buffer_handle_t_(
       ur_context_handle_t Context, ur_device_handle_t Device,
       ze_command_list_handle_t CommandList,
+      ze_command_list_handle_t CommandListTranslated,
       ze_command_list_handle_t CommandListResetEvents,
       ZeStruct<ze_command_list_desc_t> ZeDesc,
       const ur_exp_command_buffer_desc_t *Desc);
@@ -50,6 +51,9 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   ur_device_handle_t Device;
   // Level Zero command list handle
   ze_command_list_handle_t ZeCommandList;
+  // Given a multi driver scenario, the driver handle must be translated to the
+  // internal driver handle to allow calls to driver experimental apis.
+  ze_command_list_handle_t ZeComputeCommandListTranslated;
   // Level Zero command list handle
   ze_command_list_handle_t ZeCommandListResetEvents;
   // Level Zero command list descriptor

source/adapters/level_zero/image.cpp

@@ -574,8 +574,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMPitchedAllocExp(
 
   size_t Width = widthInBytes / elementSizeBytes;
   size_t RowPitch;
+  ze_device_handle_t ZeDeviceTranslated;
+  ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_DEVICE, hDevice->ZeDevice,
+                                        (void **)&ZeDeviceTranslated));
   ZE2UR_CALL(zeMemGetPitchFor2dImageFunctionPtr,
-             (hContext->ZeContext, hDevice->ZeDevice, Width, height,
+             (hContext->ZeContext, ZeDeviceTranslated, Width, height,
               elementSizeBytes, &RowPitch));
   *pResultPitch = RowPitch;
 
@@ -722,7 +725,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp(
     return UR_RESULT_ERROR_INVALID_OPERATION;
 
   uint64_t DeviceOffset{};
-  ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, (ZeImage, &DeviceOffset));
+  ze_image_handle_t ZeImageTranslated;
+  ZE2UR_CALL(zelLoaderTranslateHandle,
+             (ZEL_HANDLE_IMAGE, ZeImage, (void **)&ZeImageTranslated));
+  ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr,
+             (ZeImageTranslated, &DeviceOffset));
   *phImage = reinterpret_cast<ur_exp_image_handle_t>(DeviceOffset);
 
   return UR_RESULT_SUCCESS;

source/adapters/level_zero/memory.cpp

@@ -1623,7 +1623,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreate(
     // If not shared of any type, we can import the ptr
     if (ZeMemoryAllocationProperties.type == ZE_MEMORY_TYPE_UNKNOWN) {
       // Promote the host ptr to USM host memory
-      ze_driver_handle_t driverHandle = Context->getPlatform()->ZeDriver;
+      ze_driver_handle_t driverHandle =
+          Context->getPlatform()->ZeDriverHandleExpTranslated;
       ZeUSMImport.doZeUSMImport(driverHandle, Host, Size);
       HostPtrImported = true;
     }
@@ -2206,7 +2207,8 @@ ur_result_t _ur_buffer::free() {
       UR_CALL(ZeMemFreeHelper(UrContext, ZeHandle));
       break;
     case allocation_t::unimport:
-      ZeUSMImport.doZeUSMRelease(UrContext->getPlatform()->ZeDriver, ZeHandle);
+      ZeUSMImport.doZeUSMRelease(
+          UrContext->getPlatform()->ZeDriverHandleExpTranslated, ZeHandle);
       break;
     default:
       die("_ur_buffer::free(): Unhandled release action");

source/adapters/level_zero/platform.hpp

@@ -24,6 +24,10 @@ struct ur_platform_handle_t_ : public _ur_platform {
   // a pretty good fit to keep here.
   ze_driver_handle_t ZeDriver;
 
+  // Given a multi driver scenario, the driver handle must be translated to the
+  // internal driver handle to allow calls to driver experimental apis.
+  ze_driver_handle_t ZeDriverHandleExpTranslated;
+
   // Cache versions info from zeDriverGetProperties.
   std::string ZeDriverVersion;
   std::string ZeDriverApiVersion;

source/adapters/level_zero/ur_level_zero.hpp

@@ -18,6 +18,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include <loader/ze_loader.h>
 #include <ur/ur.hpp>
 #include <ur_api.h>
 #include <ze_api.h>

source/adapters/level_zero/usm.cpp

@@ -1037,7 +1037,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMImportExp(ur_context_handle_t Context,
     // If not shared of any type, we can import the ptr
     if (ZeMemoryAllocationProperties.type == ZE_MEMORY_TYPE_UNKNOWN) {
       // Promote the host ptr to USM host memory
-      ze_driver_handle_t driverHandle = Context->getPlatform()->ZeDriver;
+      ze_driver_handle_t driverHandle =
+          Context->getPlatform()->ZeDriverHandleExpTranslated;
       ZeUSMImport.doZeUSMImport(driverHandle, HostPtr, Size);
     }
   }
@@ -1050,6 +1051,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMReleaseExp(ur_context_handle_t Context,
 
   // Release the imported memory.
   if (ZeUSMImport.Supported && HostPtr != nullptr)
-    ZeUSMImport.doZeUSMRelease(Context->getPlatform()->ZeDriver, HostPtr);
+    ZeUSMImport.doZeUSMRelease(
+        Context->getPlatform()->ZeDriverHandleExpTranslated, HostPtr);
   return UR_RESULT_SUCCESS;
 }