From 187ac04a5b43248bc841c1841f01d07510c2711d Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 6 Dec 2024 00:02:49 +0000 Subject: [PATCH 1/3] [Tests] replace zeCallsMap with zelTracer --- source/adapters/level_zero/common.cpp | 5 --- test/adapters/level_zero/CMakeLists.txt | 16 ++------ .../adapters/level_zero/event_cache_tests.cpp | 38 ++++++++++++++----- .../multi_device_event_cache_tests.cpp | 24 +++++++++--- test/adapters/level_zero/zeCallMap.cpp | 13 ------- test/adapters/level_zero/ze_tracer_common.hpp | 31 +++++++++++++++ 6 files changed, 83 insertions(+), 44 deletions(-) delete mode 100644 test/adapters/level_zero/zeCallMap.cpp create mode 100644 test/adapters/level_zero/ze_tracer_common.hpp diff --git a/source/adapters/level_zero/common.cpp b/source/adapters/level_zero/common.cpp index da7f624013..f2ea325202 100644 --- a/source/adapters/level_zero/common.cpp +++ b/source/adapters/level_zero/common.cpp @@ -86,12 +86,7 @@ bool setEnvVar(const char *name, const char *value) { ZeUSMImportExtension ZeUSMImport; -// This will count the calls to Level-Zero -// TODO: remove the ifdef once -// https://github.com/oneapi-src/unified-runtime/issues/1454 is implemented -#ifndef UR_L0_CALL_COUNT_IN_TESTS std::map *ZeCallCount = nullptr; -#endif inline void zeParseError(ze_result_t ZeError, const char *&ErrorString) { switch (ZeError) { diff --git a/test/adapters/level_zero/CMakeLists.txt b/test/adapters/level_zero/CMakeLists.txt index 8fe062b38b..d74d08311b 100644 --- a/test/adapters/level_zero/CMakeLists.txt +++ b/test/adapters/level_zero/CMakeLists.txt @@ -43,24 +43,16 @@ if(UR_BUILD_ADAPTER_L0) endif() if(NOT WIN32 AND NOT UR_STATIC_ADAPTER_L0) - # Make L0 use CallMap from a seprate shared lib so that we can access the map - # from the tests. This only seems to work on linux - add_library(zeCallMap SHARED zeCallMap.cpp) - install_ur_library(zeCallMap) - target_compile_definitions(ur_adapter_level_zero PRIVATE UR_L0_CALL_COUNT_IN_TESTS) - # TODO: stop exporting internals like this for tests... - target_link_libraries(ur_adapter_level_zero PRIVATE zeCallMap) - add_adapter_test(level_zero_ze_calls FIXTURE DEVICES SOURCES event_cache_tests.cpp ENVIRONMENT "UR_ADAPTERS_FORCE_LOAD=\"$\"" - "UR_L0_LEAKS_DEBUG=1" + "ZE_ENABLE_TRACING_LAYER=1" ) - target_link_libraries(test-adapter-level_zero_ze_calls PRIVATE zeCallMap) + target_link_libraries(test-adapter-level_zero_ze_calls PRIVATE LevelZeroLoader LevelZeroLoader-Headers) add_adapter_test(level_zero_multi_queue FIXTURE DEVICES @@ -68,10 +60,10 @@ if(UR_BUILD_ADAPTER_L0) multi_device_event_cache_tests.cpp ENVIRONMENT "UR_ADAPTERS_FORCE_LOAD=\"$\"" - "UR_L0_LEAKS_DEBUG=1" + "ZE_ENABLE_TRACING_LAYER=1" ) - target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE zeCallMap) + target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE LevelZeroLoader LevelZeroLoader-Headers) endif() add_adapter_test(level_zero_ipc diff --git a/test/adapters/level_zero/event_cache_tests.cpp b/test/adapters/level_zero/event_cache_tests.cpp index 5ad970bad1..e62e70ff9c 100644 --- a/test/adapters/level_zero/event_cache_tests.cpp +++ b/test/adapters/level_zero/event_cache_tests.cpp @@ -10,12 +10,32 @@ #include #include +#include "ze_tracer_common.hpp" + +std::size_t eventCreateCount = 0; +std::size_t eventDestroyCount = 0; + +void OnEnterEventCreate(ze_event_create_params_t *, ze_result_t, void *, + void **) { + eventCreateCount++; +} + +void OnEnterEventDestroy(ze_event_destroy_params_t *, ze_result_t, void *, + void **) { + eventDestroyCount++; +} + +static std::shared_ptr<_zel_tracer_handle_t> tracer = [] { + zel_core_callbacks_t prologue_callbacks{}; + prologue_callbacks.Event.pfnCreateCb = OnEnterEventCreate; + prologue_callbacks.Event.pfnDestroyCb = OnEnterEventDestroy; + return enableTracing(prologue_callbacks, {}); +}(); + template auto combineFlags(std::tuple tuple) { return std::apply([](auto... args) { return (... |= args); }, tuple); } -extern std::map *ZeCallCount; - using FlagsTupleType = std::tuple; @@ -43,8 +63,8 @@ struct urEventCacheTest : uur::urContextTestWithParam { ASSERT_SUCCESS(urMemBufferCreate(context, UR_MEM_FLAG_WRITE_ONLY, size, nullptr, &buffer)); - (*ZeCallCount)["zeEventCreate"] = 0; - (*ZeCallCount)["zeEventDestroy"] = 0; + eventCreateCount = 0; + eventDestroyCount = 0; } void TearDown() override { @@ -96,9 +116,9 @@ TEST_P(urEventCacheTest, eventsReuseNoVisibleEvent) { // TODO: why events are not reused for UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE? if ((flags & UR_QUEUE_FLAG_DISCARD_EVENTS) && !(flags & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE)) { - ASSERT_EQ((*ZeCallCount)["zeEventCreate"], 2); + ASSERT_EQ(eventCreateCount, 2); } else { - ASSERT_GE((*ZeCallCount)["zeEventCreate"], numIters * numEnqueues); + ASSERT_GE(eventCreateCount, numIters * numEnqueues); } } @@ -115,7 +135,7 @@ TEST_P(urEventCacheTest, eventsReuseWithVisibleEvent) { verifyData(); } - ASSERT_LT((*ZeCallCount)["zeEventCreate"], numIters * numEnqueues); + ASSERT_LT(eventCreateCount, numIters * numEnqueues); } TEST_P(urEventCacheTest, eventsReuseWithVisibleEventAndWait) { @@ -139,9 +159,9 @@ TEST_P(urEventCacheTest, eventsReuseWithVisibleEventAndWait) { UUR_ASSERT_SUCCESS_OR_EXIT_IF_UNSUPPORTED(urQueueFinish(queue)); } - ASSERT_GE((*ZeCallCount)["zeEventCreate"], waitEveryN); + ASSERT_GE(eventCreateCount, waitEveryN); // TODO: why there are more events than this? - // ASSERT_LE((*ZeCallCount)["zeEventCreate"], waitEveryN * 2 + 2); + // ASSERT_LE(eventCreateCount, waitEveryN * 2 + 2); } template diff --git a/test/adapters/level_zero/multi_device_event_cache_tests.cpp b/test/adapters/level_zero/multi_device_event_cache_tests.cpp index c30100557c..fd0320547d 100644 --- a/test/adapters/level_zero/multi_device_event_cache_tests.cpp +++ b/test/adapters/level_zero/multi_device_event_cache_tests.cpp @@ -10,7 +10,21 @@ #include #include -extern std::map *ZeCallCount; +#include "ze_tracer_common.hpp" + +size_t zeCommandListAppendWaitOnEventsCount = 0; + +void OnAppendWaitOnEventsCb(ze_command_list_append_wait_on_events_params_t *, + ze_result_t, void *, void **) { + zeCommandListAppendWaitOnEventsCount++; +} + +static std::shared_ptr<_zel_tracer_handle_t> tracer = [] { + zel_core_callbacks_t prologue_callbacks{}; + prologue_callbacks.CommandList.pfnAppendWaitOnEventsCb = + OnAppendWaitOnEventsCb; + return enableTracing(prologue_callbacks, {}); +}(); using urMultiQueueMultiDeviceEventCacheTest = uur::urAllDevicesTest; TEST_F(urMultiQueueMultiDeviceEventCacheTest, @@ -54,7 +68,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, uur::raii::Event event = nullptr; uur::raii::Event eventWait = nullptr; uur::raii::Event eventWaitDummy = nullptr; - (*ZeCallCount)["zeCommandListAppendWaitOnEvents"] = 0; + zeCommandListAppendWaitOnEventsCount = 0; EXPECT_SUCCESS( urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr())); EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr, @@ -63,7 +77,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, urEnqueueEventsWait(queue1, 1, eventWaitDummy.ptr(), eventWait.ptr())); EXPECT_SUCCESS( urEnqueueEventsWait(queue2, 1, eventWait.ptr(), event.ptr())); - EXPECT_EQ((*ZeCallCount)["zeCommandListAppendWaitOnEvents"], 2); + EXPECT_EQ(zeCommandListAppendWaitOnEventsCount, 2); ASSERT_SUCCESS(urEventRelease(eventWaitDummy.get())); ASSERT_SUCCESS(urEventRelease(eventWait.get())); ASSERT_SUCCESS(urEventRelease(event.get())); @@ -89,7 +103,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, uur::raii::Event event = nullptr; uur::raii::Event eventWait = nullptr; uur::raii::Event eventWaitDummy = nullptr; - (*ZeCallCount)["zeCommandListAppendWaitOnEvents"] = 0; + zeCommandListAppendWaitOnEventsCount = 0; EXPECT_SUCCESS( urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr())); EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr, @@ -98,7 +112,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, urEnqueueEventsWait(queue1, 1, eventWaitDummy.ptr(), eventWait.ptr())); EXPECT_SUCCESS( urEnqueueEventsWait(queue2, 1, eventWait.ptr(), event.ptr())); - EXPECT_EQ((*ZeCallCount)["zeCommandListAppendWaitOnEvents"], 3); + EXPECT_EQ(zeCommandListAppendWaitOnEventsCount, 3); ASSERT_SUCCESS(urEventRelease(eventWaitDummy.get())); ASSERT_SUCCESS(urEventRelease(eventWait.get())); ASSERT_SUCCESS(urEventRelease(event.get())); diff --git a/test/adapters/level_zero/zeCallMap.cpp b/test/adapters/level_zero/zeCallMap.cpp deleted file mode 100644 index c2e47b856d..0000000000 --- a/test/adapters/level_zero/zeCallMap.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (C) 2024 Intel Corporation -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -// See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include -#include - -// Map used by L0 adapter to count the number of calls to each L0 function -// Lifetime is managed by the adapter, this variable is defined here -// only so that we can read it from the tests. -__attribute__((visibility("default"))) std::map *ZeCallCount = - nullptr; diff --git a/test/adapters/level_zero/ze_tracer_common.hpp b/test/adapters/level_zero/ze_tracer_common.hpp new file mode 100644 index 0000000000..ed33eb30a5 --- /dev/null +++ b/test/adapters/level_zero/ze_tracer_common.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2024 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "uur/fixtures.h" + +#include +#include + +#include + +std::shared_ptr<_zel_tracer_handle_t> +enableTracing(zel_core_callbacks_t prologueCallbacks, + zel_core_callbacks_t epilogueCallbacks) { + EXPECT_EQ(zeInit(ZE_INIT_FLAG_GPU_ONLY), ZE_RESULT_SUCCESS); + + zel_tracer_desc_t tracer_desc = {ZEL_STRUCTURE_TYPE_TRACER_EXP_DESC, + nullptr, nullptr}; + zel_tracer_handle_t tracer = nullptr; + EXPECT_EQ(zelTracerCreate(&tracer_desc, &tracer), ZE_RESULT_SUCCESS); + + EXPECT_EQ(zelTracerSetPrologues(tracer, &prologueCallbacks), + ZE_RESULT_SUCCESS); + EXPECT_EQ(zelTracerSetEpilogues(tracer, &epilogueCallbacks), + ZE_RESULT_SUCCESS); + EXPECT_EQ(zelTracerSetEnabled(tracer, true), ZE_RESULT_SUCCESS); + + return std::shared_ptr<_zel_tracer_handle_t>( + tracer, [](zel_tracer_handle_t tracer) { zelTracerDestroy(tracer); }); +} From a99c36963b7f037770c90d108c1857af5b2e91ee Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 6 Dec 2024 17:43:48 +0000 Subject: [PATCH 2/3] [Tests] fix segfault in multi_device_event_cache_tests Using event created on a context associated with device1 on a queue associated with a different device was causing as segault on urEnqueueEventsWait --- .../level_zero/multi_device_event_cache_tests.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/adapters/level_zero/multi_device_event_cache_tests.cpp b/test/adapters/level_zero/multi_device_event_cache_tests.cpp index fd0320547d..6ad5b0a9ef 100644 --- a/test/adapters/level_zero/multi_device_event_cache_tests.cpp +++ b/test/adapters/level_zero/multi_device_event_cache_tests.cpp @@ -54,12 +54,12 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, ASSERT_SUCCESS(urDevicePartition(devices[0], &properties, numSubDevices, sub_devices.data(), nullptr)); uur::raii::Context context1 = nullptr; - ASSERT_SUCCESS( - urContextCreate(1, &sub_devices[0], nullptr, context1.ptr())); + ASSERT_SUCCESS(urContextCreate(sub_devices.size(), &sub_devices[0], nullptr, + context1.ptr())); ASSERT_NE(nullptr, context1); uur::raii::Context context2 = nullptr; - ASSERT_SUCCESS( - urContextCreate(1, &sub_devices[1], nullptr, context2.ptr())); + ASSERT_SUCCESS(urContextCreate(sub_devices.size(), &sub_devices[0], nullptr, + context2.ptr())); ASSERT_NE(nullptr, context2); ur_queue_handle_t queue1 = nullptr; ASSERT_SUCCESS(urQueueCreate(context1, sub_devices[0], 0, &queue1)); @@ -91,10 +91,12 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, GTEST_SKIP(); } uur::raii::Context context1 = nullptr; - ASSERT_SUCCESS(urContextCreate(1, &devices[0], nullptr, context1.ptr())); + ASSERT_SUCCESS( + urContextCreate(devices.size(), &devices[0], nullptr, context1.ptr())); ASSERT_NE(nullptr, context1); uur::raii::Context context2 = nullptr; - ASSERT_SUCCESS(urContextCreate(1, &devices[1], nullptr, context2.ptr())); + ASSERT_SUCCESS( + urContextCreate(devices.size(), &devices[0], nullptr, context2.ptr())); ASSERT_NE(nullptr, context2); ur_queue_handle_t queue1 = nullptr; ASSERT_SUCCESS(urQueueCreate(context1, devices[0], 0, &queue1)); From 03c81a7c53f5af7fd0c85970f938f70b1d6eab75 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 6 Dec 2024 17:46:26 +0000 Subject: [PATCH 3/3] [Tests] remove unnecessary call to urEventCreateWithNativeHandle This had no effect on the test as UR always allocates a new object for the signal event. --- test/adapters/level_zero/multi_device_event_cache_tests.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/adapters/level_zero/multi_device_event_cache_tests.cpp b/test/adapters/level_zero/multi_device_event_cache_tests.cpp index 6ad5b0a9ef..7848bb1003 100644 --- a/test/adapters/level_zero/multi_device_event_cache_tests.cpp +++ b/test/adapters/level_zero/multi_device_event_cache_tests.cpp @@ -69,8 +69,6 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, uur::raii::Event eventWait = nullptr; uur::raii::Event eventWaitDummy = nullptr; zeCommandListAppendWaitOnEventsCount = 0; - EXPECT_SUCCESS( - urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr())); EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr, eventWaitDummy.ptr())); EXPECT_SUCCESS( @@ -106,8 +104,6 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest, uur::raii::Event eventWait = nullptr; uur::raii::Event eventWaitDummy = nullptr; zeCommandListAppendWaitOnEventsCount = 0; - EXPECT_SUCCESS( - urEventCreateWithNativeHandle(0, context2, nullptr, eventWait.ptr())); EXPECT_SUCCESS(urEventCreateWithNativeHandle(0, context1, nullptr, eventWaitDummy.ptr())); EXPECT_SUCCESS(