[shortfin] Plumb through amdgpu_tracing_level config option. (#431)

* Corresponds to IREE runtime `--hip_tracing` flag.
* Adds `amdgpu_tracing_level=<int>` with a default of 2 to match IREE.
* Verified that the default produces traces with GPU zones and that both
the Python level setting and the env var
(`SHORTFIN_AMDGPU_TRACING_LEVEL`) works.
* Not a good way to automate this kind of test, so I just did manual
verification.

shortfin/python/lib_ext.cc

@@ -1267,6 +1267,27 @@ environment variable is searched as a fallback in all cases. Multiple paths
 can be separated by semicolons on all platforms.
 )";
 
+static const char DOCSTRING_AMDGPU_SYSTEM_BUILDER_TRACING_LEVEL[] =
+    R"(Tracing level for AMDGPU device behavior.
+
+Controls the verbosity of tracing when Tracy instrumentation is enabled.
+The impact to benchmark timing becomes more severe as the verbosity
+increases, and thus should be only enabled when needed.
+
+This is the equivalent of the `--hip_tracing` IREE tools flag.
+Permissible values are:
+  * 0 : stream tracing disabled.
+  * 1 : coarse command buffer level tracing enabled.
+  * 2 : (default) fine-grained kernel level tracing enabled.
+
+The setting only has an effect if using a tracing enabled runtime (i.e.
+by running with `SHORTFIN_PY_RUNTIME=tracy` or equiv).
+
+The default value for this setting is available as a
+`amdgpu.SystemBuilder(amdgpu_tracing_level=2)` or (by default) from an
+environment variable `SHORTFIN_AMDGPU_TRACING_LEVEL`.
+)";
+
 static const char DOCSTRING_AMDGPU_SYSTEM_BUILDER_AVAILABLE_DEVICES[] =
     R"(List of available device ids on the system.
 
@@ -1343,6 +1364,15 @@ void BindAMDGPUSystem(py::module_ &global_m) {
           [](local::systems::AMDGPUSystemBuilder &self,
              std::vector<std::string> vs) { self.hip_lib_search_paths() = vs; },
           DOCSTRING_AMDGPU_SYSTEM_BUILDER_HIP_LIB_SEARCH_PATHS)
+      .def_prop_rw(
+          "tracing_level",
+          [](local::systems::AMDGPUSystemBuilder &self) -> int {
+            return self.tracing_level();
+          },
+          [](local::systems::AMDGPUSystemBuilder &self, int tracing_level) {
+            self.tracing_level() = tracing_level;
+          },
+          DOCSTRING_AMDGPU_SYSTEM_BUILDER_TRACING_LEVEL)
       .def_prop_rw(
           "visible_devices",
           [](local::systems::AMDGPUSystemBuilder &self)

shortfin/src/shortfin/local/systems/amdgpu.cc

@@ -20,8 +20,8 @@ AMDGPUSystemBuilder::AMDGPUSystemBuilder(iree_allocator_t host_allocator,
                                          ConfigOptions options)
     : HostCPUSystemBuilder(host_allocator, std::move(options)),
       available_devices_(host_allocator) {
-  InitializeDefaultSettings();
   iree_hal_hip_device_params_initialize(&default_device_params_);
+  InitializeDefaultSettings();
 }
 
 AMDGPUSystemBuilder::~AMDGPUSystemBuilder() = default;
@@ -41,6 +41,16 @@ void AMDGPUSystemBuilder::InitializeDefaultSettings() {
     }
   }
 
+  // HIP options.
+  // "amdgpu_tracing_level": Matches IREE flag --hip_tracing:
+  // Permissible values are:
+  //   0 : stream tracing disabled.
+  //   1 : coarse command buffer level tracing enabled.
+  //   2 : fine-grained kernel level tracing enabled.
+  auto tracing_level =
+      config_options().GetInt("amdgpu_tracing_level", /*non_negative=*/true);
+  default_device_params_.stream_tracing = tracing_level ? *tracing_level : 2;
+
   // CPU devices.
   cpu_devices_enabled_ = config_options().GetBool("amdgpu_cpu_devices_enabled");
 

shortfin/src/shortfin/local/systems/amdgpu.h

@@ -57,6 +57,8 @@ class SHORTFIN_API AMDGPUSystemBuilder : public HostCPUSystemBuilder {
     return visible_devices_;
   };
 
+  int32_t &tracing_level() { return default_device_params_.stream_tracing; }
+
   // Gets all enumerated available device ids. This triggers enumeration, so
   // any settings required for that must already be set. This does no filtering
   // and will return all device ids.

shortfin/src/shortfin/support/config.cc

@@ -24,8 +24,8 @@ const std::optional<std::string_view> ConfigOptions::GetOption(
     std::string_view key) const {
   // Get explicit option.
   auto found_it = options_.find(key);
+  consumed_keys_.insert(key);
   if (found_it != options_.end()) {
-    consumed_keys_.insert(key);
     return found_it->second;
   }
 
@@ -138,8 +138,9 @@ void ConfigOptions::ValidateUndef() const {
     }
   }
   if (!unused_options.empty()) {
-    std::string message = fmt::format("Specified options were not used: {}",
-                                      fmt::join(unused_options, ", "));
+    std::string message = fmt::format(
+        "Specified options were not used: {} (available: {})",
+        fmt::join(unused_options, ", "), fmt::join(consumed_keys_, ", "));
     switch (validation_) {
       case ValidationLevel::UNDEF_DEBUG:
         logging::debug("{}", message);

shortfin/tests/amdgpu_system_test.py

@@ -20,6 +20,14 @@ def test_create_amd_gpu_system_defaults():
         assert "hostcpu:0:0@0" not in ls.device_names
 
 
+@pytest.mark.system("amdgpu")
+def test_create_amd_gpu_tracing_level():
+    sc = sf.amdgpu.SystemBuilder()
+    assert sc.tracing_level == 2  # Default
+    sc = sf.amdgpu.SystemBuilder(amdgpu_tracing_level=1)
+    assert sc.tracing_level == 1
+
+
 @pytest.mark.system("amdgpu")
 def test_create_amd_gpu_system_defaults():
     sc = sf.amdgpu.SystemBuilder(amdgpu_cpu_devices_enabled=True)

shortfin/tests/conftest.py

@@ -53,6 +53,7 @@ def pytest_runtest_setup(item):
 # Keys that will be cleaned project wide prior to and after each test run.
 # Test code can freely modify these.
 CLEAN_ENV_KEYS = [
+    "SHORTFIN_AMDGPU_TRACING_LEVEL",
     "SHORTFIN_HOSTCPU_TOPOLOGY_NODES",
     "SHORTFIN_HOSTCPU_TOPOLOGY_MAX_GROUP_COUNT",
     "SHORTFIN_SYSTEM_TYPE",