Implement VK_MSFT_layered_driver extension support

This extension reorders physical devices enumerated through the windows
EnumerateAdapterPhysicalDevices whenever multiple drivers exist for the same
LUID. This is so that if a driver is considered a 'layered implementation',
eg Dozen's Vulkan on Dx12, the physical device corresponding to the native
vulkan driver is preferred.

loader/loader_common.h

@@ -473,6 +473,9 @@ struct loader_phys_dev_per_icd {
     VkPhysicalDevice *physical_devices;
     uint32_t icd_index;
     struct loader_icd_term *icd_term;
+#if defined(WIN32)
+    LUID luid;
+#endif
 };
 
 struct loader_msg_callback_map_entry {

loader/loader_windows.c

@@ -884,37 +884,106 @@ VkResult windows_read_sorted_physical_devices(struct loader_instance *inst, uint
                 } while ((vkres = icd_term->scanned_icd->EnumerateAdapterPhysicalDevices(
                               icd_term->instance, description.AdapterLuid, &count,
                               sorted_array[*sorted_devices_count].physical_devices)) == VK_INCOMPLETE);
-            }
 
-            if (vkres != VK_SUCCESS) {
-                loader_instance_heap_free(inst, sorted_array[*sorted_devices_count].physical_devices);
-                sorted_array[*sorted_devices_count].physical_devices = NULL;
-                if (vkres == VK_ERROR_OUT_OF_HOST_MEMORY) {
-                    res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                    goto out;
-                } else {
-                    loader_log(inst, VULKAN_LOADER_WARN_BIT, 0, "Failed to convert DXGI adapter into Vulkan physical device");
-                    continue;
+                if (vkres != VK_SUCCESS) {
+                    loader_instance_heap_free(inst, sorted_array[*sorted_devices_count].physical_devices);
+                    sorted_array[*sorted_devices_count].physical_devices = NULL;
+                    if (vkres == VK_ERROR_OUT_OF_HOST_MEMORY) {
+                        res = VK_ERROR_OUT_OF_HOST_MEMORY;
+                        goto out;
+                    } else {
+                        loader_log(inst, VULKAN_LOADER_WARN_BIT, 0, "Failed to convert DXGI adapter into Vulkan physical device");
+                        continue;
+                    }
+                }
+
+                // Check if we have already added these physical devices - in case multiple adapters report the same LUID
+                bool already_has_phys_dev_per_icd = false;
+                for (uint32_t j = 0; j < *sorted_devices_count; j++) {
+                    if (count == sorted_array[j].device_count) {
+                        bool matches = true;
+                        for (uint32_t k = 0; k < sorted_array[j].device_count; k++) {
+                            if (sorted_array[j].physical_devices[k] != sorted_array[*sorted_devices_count].physical_devices[k]) {
+                                matches = false;
+
+                                break;
+                            }
+                        }
+                        if (matches) {
+                            already_has_phys_dev_per_icd = true;
+                        }
+                    }
+                }
+                if (!already_has_phys_dev_per_icd) {
+                    sorted_array[*sorted_devices_count].device_count = count;
+                    sorted_array[*sorted_devices_count].icd_index = icd_idx;
+                    sorted_array[*sorted_devices_count].icd_term = icd_term;
+                    sorted_array[*sorted_devices_count].luid = description.AdapterLuid;
+                    (*sorted_devices_count)++;
                 }
             }
-            sorted_array[*sorted_devices_count].device_count = count;
-            sorted_array[*sorted_devices_count].icd_index = icd_idx;
-            sorted_array[*sorted_devices_count].icd_term = icd_term;
-            (*sorted_devices_count)++;
         }
 
         adapter->lpVtbl->Release(adapter);
     }
 
     dxgi_factory->lpVtbl->Release(dxgi_factory);
 
+    bool app_is_vulkan_1_1 = loader_check_version_meets_required(LOADER_VERSION_1_1_0, inst->app_api_version);
+
+    // Iterate over all devices and make sure any ICD's with matching LUID's are sorted such that drivers with a underlyingAPI of
+    // VK_LAYERED_DRIVER_UNDERLYING_API_D3D12_MSFT are ordered after drivers without it
+    struct loader_phys_dev_per_icd *sorted_array = *sorted_devices;
+    for (uint32_t i = 0; *sorted_devices_count > 1 && i < *sorted_devices_count - 1; i++) {
+        for (uint32_t j = i + 1; j < *sorted_devices_count; j++) {
+            // Skip consideration for reordering if LUID's dont match
+            if ((sorted_array[i].luid.HighPart != sorted_array[j].luid.HighPart) &&
+                (sorted_array[i].luid.LowPart != sorted_array[j].luid.LowPart)) {
+                continue;
+            }
+
+            VkLayeredDriverUnderlyingApiMSFT underlyingAPI = VK_LAYERED_DRIVER_UNDERLYING_API_NONE_MSFT;
+            VkPhysicalDeviceLayeredDriverPropertiesMSFT layered_driver_properties_msft = {0};
+            layered_driver_properties_msft.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LAYERED_DRIVER_PROPERTIES_MSFT;
+            VkPhysicalDeviceProperties2 props2 = {0};
+            props2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
+            props2.pNext = (void *)&layered_driver_properties_msft;
+
+            for (uint32_t k = 0; k < sorted_array[i].device_count; k++) {
+                VkPhysicalDeviceProperties dev_props = {0};
+                sorted_array[i].icd_term->dispatch.GetPhysicalDeviceProperties(sorted_array[i].physical_devices[k], &dev_props);
+
+                bool device_is_1_1_capable =
+                    loader_check_version_meets_required(LOADER_VERSION_1_1_0, loader_make_version(dev_props.apiVersion));
+
+                PFN_vkGetPhysicalDeviceProperties2 GetPhysDevProps2 = NULL;
+                if (app_is_vulkan_1_1 && device_is_1_1_capable) {
+                    GetPhysDevProps2 = sorted_array[i].icd_term->dispatch.GetPhysicalDeviceProperties2;
+                } else {
+                    GetPhysDevProps2 =
+                        (PFN_vkGetPhysicalDeviceProperties2)sorted_array[i].icd_term->dispatch.GetPhysicalDeviceProperties2KHR;
+                }
+                if (GetPhysDevProps2) {
+                    GetPhysDevProps2(sorted_array[i].physical_devices[k], &props2);
+                    if (layered_driver_properties_msft.underlyingAPI != VK_LAYERED_DRIVER_UNDERLYING_API_NONE_MSFT) {
+                        underlyingAPI = layered_driver_properties_msft.underlyingAPI;
+                        break;
+                    }
+                }
+            }
+            if (underlyingAPI == VK_LAYERED_DRIVER_UNDERLYING_API_D3D12_MSFT) {
+                struct loader_phys_dev_per_icd swap_icd = sorted_array[i];
+                sorted_array[i] = sorted_array[j];
+                sorted_array[j] = swap_icd;
+            }
+        }
+    }
+
 out:
     if (*sorted_devices_count == 0 && *sorted_devices != NULL) {
         loader_instance_heap_free(inst, *sorted_devices);
         *sorted_devices = NULL;
     }
-    *sorted_devices_count = *sorted_devices_count;
-    *sorted_devices = *sorted_devices;
     return res;
 }
 

tests/framework/icd/physical_device.h

@@ -63,6 +63,9 @@ struct PhysicalDevice {
     BUILDER_VALUE(PhysicalDevice, VkDisplayModeKHR, display_mode, {})
     BUILDER_VALUE(PhysicalDevice, VkDisplayPlaneCapabilitiesKHR, display_plane_capabilities, {})
 
+    BUILDER_VALUE(PhysicalDevice, VkLayeredDriverUnderlyingApiMSFT, layered_driver_underlying_api,
+                  VK_LAYERED_DRIVER_UNDERLYING_API_NONE_MSFT)
+
     PhysicalDevice& set_api_version(uint32_t version) {
         properties.apiVersion = version;
         return *this;

tests/framework/icd/test_icd.cpp

@@ -988,10 +988,13 @@ VKAPI_ATTR void VKAPI_CALL test_vkGetPhysicalDeviceProperties2(VkPhysicalDevice
         VkBaseInStructure* pNext = reinterpret_cast<VkBaseInStructure*>(pProperties->pNext);
         while (pNext) {
             if (pNext->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT) {
-                VkPhysicalDevicePCIBusInfoPropertiesEXT* bus_info =
-                    reinterpret_cast<VkPhysicalDevicePCIBusInfoPropertiesEXT*>(pNext);
+                auto* bus_info = reinterpret_cast<VkPhysicalDevicePCIBusInfoPropertiesEXT*>(pNext);
                 bus_info->pciBus = phys_dev.pci_bus;
             }
+            if (pNext->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LAYERED_DRIVER_PROPERTIES_MSFT) {
+                auto* layered_driver_props = reinterpret_cast<VkPhysicalDeviceLayeredDriverPropertiesMSFT*>(pNext);
+                layered_driver_props->underlyingAPI = phys_dev.layered_driver_underlying_api;
+            }
             pNext = reinterpret_cast<VkBaseInStructure*>(const_cast<VkBaseInStructure*>(pNext->pNext));
         }
     }
@@ -1110,7 +1113,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vk_icdEnumerateAdapterPhysicalDevices(VkInst
                                                                           VkPhysicalDevice* pPhysicalDevices) {
     if (adapterLUID.LowPart != icd.adapterLUID.LowPart || adapterLUID.HighPart != icd.adapterLUID.HighPart) {
         *pPhysicalDeviceCount = 0;
-        return VK_SUCCESS;
+        return VK_ERROR_INCOMPATIBLE_DRIVER;
     }
     icd.called_enumerate_adapter_physical_devices = true;
     VkResult res = test_vkEnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices);

tests/framework/shim/shim.h

@@ -89,6 +89,8 @@ struct DXGIAdapter {
     GpuType gpu_preference = GpuType::unspecified;
     DXGI_ADAPTER_DESC1 desc1{};
     uint32_t adapter_index = 0;
+    IDXGIAdapter1 adapter_instance{};
+    IDXGIAdapter1Vtbl adapter_vtbl_instance{};
 };
 
 struct D3DKMT_Adapter {
@@ -156,10 +158,8 @@ struct PlatformShim {
     void add_d3dkmt_adapter(D3DKMT_Adapter const& adapter);
     void set_app_package_path(fs::path const& path);
 
-    uint32_t next_adapter_handle = 1;  // increment everytime add_dxgi_adapter is called
-    std::vector<DXGIAdapter> dxgi_adapters;
-    std::unordered_map<IDXGIAdapter1*, uint32_t> dxgi_adapter_map;
-    // next two are a pair
+    std::unordered_map<uint32_t, DXGIAdapter> dxgi_adapters;
+
     std::vector<D3DKMT_Adapter> d3dkmt_adapters;
 
     // TODO:

tests/framework/shim/shim_common.cpp

@@ -131,7 +131,8 @@ void PlatformShim::add_unsecured_manifest(ManifestCategory category, fs::path co
 }
 
 void PlatformShim::add_dxgi_adapter(GpuType gpu_preference, DXGI_ADAPTER_DESC1 desc1) {
-    dxgi_adapters.push_back({gpu_preference, desc1, next_adapter_handle++});
+    uint32_t next_index = static_cast<uint32_t>(dxgi_adapters.size());
+    dxgi_adapters.emplace(next_index, DXGIAdapter{gpu_preference, desc1, next_index});
 }
 
 void PlatformShim::add_d3dkmt_adapter(D3DKMT_Adapter const& adapter) { d3dkmt_adapters.push_back(adapter); }

tests/framework/shim/windows_shim.cpp

@@ -168,49 +168,24 @@ HRESULT __stdcall ShimGetDesc1(IDXGIAdapter1 *pAdapter,
                                /* [annotation][out] */
                                _Out_ DXGI_ADAPTER_DESC1 *pDesc) {
     if (pAdapter == nullptr || pDesc == nullptr) return DXGI_ERROR_INVALID_CALL;
-    auto it = platform_shim.dxgi_adapter_map.find(pAdapter);
-    if (it == platform_shim.dxgi_adapter_map.end()) {
-        return DXGI_ERROR_INVALID_CALL;
-    }
-    *pDesc = platform_shim.dxgi_adapters[it->second].desc1;
-    return S_OK;
-}
-ULONG __stdcall ShimIDXGIFactory1Release(IDXGIFactory1 *factory) {
-    if (factory != nullptr) {
-        if (factory->lpVtbl != nullptr) {
-            delete factory->lpVtbl;
-        }
-        delete factory;
-    }
-    return S_OK;
-}
-ULONG __stdcall ShimIDXGIFactory6Release(IDXGIFactory6 *factory) {
-    if (factory != nullptr) {
-        if (factory->lpVtbl != nullptr) {
-            delete factory->lpVtbl;
-        }
-        delete factory;
-    }
-    return S_OK;
-}
 
-ULONG __stdcall ShimRelease(IDXGIAdapter1 *pAdapter) {
-    if (pAdapter != nullptr) {
-        if (pAdapter->lpVtbl != nullptr) {
-            delete pAdapter->lpVtbl;
+    for (const auto &[index, adapter] : platform_shim.dxgi_adapters) {
+        if (&adapter.adapter_instance == pAdapter) {
+            *pDesc = adapter.desc1;
+            return S_OK;
         }
-        delete pAdapter;
     }
-    return S_OK;
+    return DXGI_ERROR_INVALID_CALL;
 }
-
-IDXGIAdapter1 *create_IDXGIAdapter1() {
-    IDXGIAdapter1Vtbl *vtbl = new IDXGIAdapter1Vtbl();
-    vtbl->GetDesc1 = ShimGetDesc1;
-    vtbl->Release = ShimRelease;
-    IDXGIAdapter1 *adapter = new IDXGIAdapter1();
-    adapter->lpVtbl = vtbl;
-    return adapter;
+ULONG __stdcall ShimIDXGIFactory1Release(IDXGIFactory1 *) { return S_OK; }
+ULONG __stdcall ShimIDXGIFactory6Release(IDXGIFactory6 *) { return S_OK; }
+ULONG __stdcall ShimRelease(IDXGIAdapter1 *) { return S_OK; }
+
+IDXGIAdapter1 *setup_and_get_IDXGIAdapter1(DXGIAdapter &adapter) {
+    adapter.adapter_vtbl_instance.GetDesc1 = ShimGetDesc1;
+    adapter.adapter_vtbl_instance.Release = ShimRelease;
+    adapter.adapter_instance.lpVtbl = &adapter.adapter_vtbl_instance;
+    return &adapter.adapter_instance;
 }
 
 HRESULT __stdcall ShimEnumAdapters1_1([[maybe_unused]] IDXGIFactory1 *This,
@@ -221,9 +196,7 @@ HRESULT __stdcall ShimEnumAdapters1_1([[maybe_unused]] IDXGIFactory1 *This,
         return DXGI_ERROR_INVALID_CALL;
     }
     if (ppAdapter != nullptr) {
-        auto *pAdapter = create_IDXGIAdapter1();
-        *ppAdapter = pAdapter;
-        platform_shim.dxgi_adapter_map[pAdapter] = Adapter;
+        *ppAdapter = setup_and_get_IDXGIAdapter1(platform_shim.dxgi_adapters.at(Adapter));
     }
     return S_OK;
 }
@@ -236,9 +209,7 @@ HRESULT __stdcall ShimEnumAdapters1_6([[maybe_unused]] IDXGIFactory6 *This,
         return DXGI_ERROR_INVALID_CALL;
     }
     if (ppAdapter != nullptr) {
-        auto *pAdapter = create_IDXGIAdapter1();
-        *ppAdapter = pAdapter;
-        platform_shim.dxgi_adapter_map[pAdapter] = Adapter;
+        *ppAdapter = setup_and_get_IDXGIAdapter1(platform_shim.dxgi_adapters.at(Adapter));
     }
     return S_OK;
 }
@@ -254,40 +225,38 @@ HRESULT __stdcall ShimEnumAdapterByGpuPreference([[maybe_unused]] IDXGIFactory6
     assert(GpuPreference == DXGI_GPU_PREFERENCE::DXGI_GPU_PREFERENCE_UNSPECIFIED &&
            "Test shim assumes the GpuPreference is unspecified.");
     if (ppvAdapter != nullptr) {
-        auto *pAdapter = create_IDXGIAdapter1();
-        *ppvAdapter = pAdapter;
-        platform_shim.dxgi_adapter_map[pAdapter] = Adapter;
+        *ppvAdapter = setup_and_get_IDXGIAdapter1(platform_shim.dxgi_adapters.at(Adapter));
     }
     return S_OK;
 }
 
-IDXGIFactory1 *create_IDXGIFactory1() {
-    IDXGIFactory1Vtbl *vtbl = new IDXGIFactory1Vtbl();
-    vtbl->EnumAdapters1 = ShimEnumAdapters1_1;
-    vtbl->Release = ShimIDXGIFactory1Release;
-    IDXGIFactory1 *factory = new IDXGIFactory1();
-    factory->lpVtbl = vtbl;
-    return factory;
+static IDXGIFactory1 *get_IDXGIFactory1() {
+    static IDXGIFactory1Vtbl vtbl{};
+    vtbl.EnumAdapters1 = ShimEnumAdapters1_1;
+    vtbl.Release = ShimIDXGIFactory1Release;
+    static IDXGIFactory1 factory{};
+    factory.lpVtbl = &vtbl;
+    return &factory;
 }
 
-IDXGIFactory6 *create_IDXGIFactory6() {
-    IDXGIFactory6Vtbl *vtbl = new IDXGIFactory6Vtbl();
-    vtbl->EnumAdapters1 = ShimEnumAdapters1_6;
-    vtbl->EnumAdapterByGpuPreference = ShimEnumAdapterByGpuPreference;
-    vtbl->Release = ShimIDXGIFactory6Release;
-    IDXGIFactory6 *factory = new IDXGIFactory6();
-    factory->lpVtbl = vtbl;
-    return factory;
+static IDXGIFactory6 *get_IDXGIFactory6() {
+    static IDXGIFactory6Vtbl vtbl{};
+    vtbl.EnumAdapters1 = ShimEnumAdapters1_6;
+    vtbl.EnumAdapterByGpuPreference = ShimEnumAdapterByGpuPreference;
+    vtbl.Release = ShimIDXGIFactory6Release;
+    static IDXGIFactory6 factory{};
+    factory.lpVtbl = &vtbl;
+    return &factory;
 }
 
 HRESULT __stdcall ShimCreateDXGIFactory1(REFIID riid, void **ppFactory) {
     if (riid == IID_IDXGIFactory1) {
-        auto *factory = create_IDXGIFactory1();
+        auto *factory = get_IDXGIFactory1();
         *ppFactory = factory;
         return S_OK;
     }
     if (riid == IID_IDXGIFactory6) {
-        auto *factory = create_IDXGIFactory6();
+        auto *factory = get_IDXGIFactory6();
         *ppFactory = factory;
         return S_OK;
     }