Tracing in ML-Commons

common/src/main/java/org/opensearch/ml/common/MLModel.java

@@ -965,7 +965,7 @@ Tags getRemoteModelTags(Connector connector) {
      * @param url The URL to analyze for service provider identification
      * @return The identified service provider name, or "unknown" if not found
      */
-    String identifyServiceProvider(String url) {
+    static String identifyServiceProvider(String url) {
         for (String provider : MODEL_SERVICE_PROVIDER_KEYWORDS) {
             if (url.contains(provider)) {
                 return provider;
@@ -975,6 +975,10 @@ String identifyServiceProvider(String url) {
         return TAG_VALUE_UNKNOWN;
     }
 
+    public static String identifyServiceProviderFromUrl(String url) {
+        return identifyServiceProvider(url);
+    }
+
     /**
      * Identifies the model name from the connector configuration using multiple strategies.
      * The method attempts to extract the model name in the following order:

common/src/main/java/org/opensearch/ml/common/settings/MLCommonsSettings.java

@@ -368,4 +368,32 @@ private MLCommonsSettings() {}
         .boolSetting("plugins.ml_commons.agentic_memory_enabled", false, Setting.Property.NodeScope, Setting.Property.Dynamic);
     public static final String ML_COMMONS_AGENTIC_MEMORY_DISABLED_MESSAGE =
         "The Agentic Memory APIs are not enabled. To enable, please update the setting " + ML_COMMONS_AGENTIC_MEMORY_ENABLED.getKey();
+
+    // Feature flag for enabling telemetry tracer
+    // This setting is Final because it controls the core tracing infrastructure initialization.
+    // Once the tracer is initialized, changing this setting would require a node restart
+    // to properly reinitialize the tracing components.
+    public static final Setting<Boolean> ML_COMMONS_TRACING_ENABLED = Setting
+        .boolSetting("plugins.ml_commons.tracing_enabled", false, Setting.Property.NodeScope, Setting.Property.Final);
+
+    // Feature flag for enabling telemetry agent tracing
+    // This setting is Dynamic because agent tracing can be enabled/disabled at runtime
+    // without requiring a node restart. The MLAgentTracer singleton can be updated
+    // to switch between real tracer and NoopTracer based on this setting.
+    public static final Setting<Boolean> ML_COMMONS_AGENT_TRACING_ENABLED = Setting
+        .boolSetting("plugins.ml_commons.agent_tracing_enabled", false, Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+    // Feature flag for enabling telemetry connector tracing
+    // This setting is Dynamic because connector tracing can be enabled/disabled at runtime
+    // without requiring a node restart. The MLConnectorTracer singleton can be updated
+    // to switch between real tracer and NoopTracer based on this setting.
+    public static final Setting<Boolean> ML_COMMONS_CONNECTOR_TRACING_ENABLED = Setting
+        .boolSetting("plugins.ml_commons.connector_tracing_enabled", false, Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+    // Feature flag for enabling telemetry model tracing
+    // This setting is Dynamic because model tracing can be enabled/disabled at runtime
+    // without requiring a node restart. The MLModelTracer singleton can be updated
+    // to switch between real tracer and NoopTracer based on this setting.
+    public static final Setting<Boolean> ML_COMMONS_MODEL_TRACING_ENABLED = Setting
+        .boolSetting("plugins.ml_commons.model_tracing_enabled", false, Setting.Property.NodeScope, Setting.Property.Dynamic);
 }

common/src/main/java/org/opensearch/ml/common/settings/MLFeatureEnabledSetting.java

@@ -8,19 +8,23 @@
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_AGENTIC_MEMORY_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_AGENTIC_SEARCH_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_AGENT_FRAMEWORK_ENABLED;
+import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_AGENT_TRACING_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_CONNECTOR_PRIVATE_IP_ENABLED;
+import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_CONNECTOR_TRACING_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_CONTROLLER_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_EXECUTE_TOOL_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_LOCAL_MODEL_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_MCP_CONNECTOR_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_MCP_SERVER_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_METRIC_COLLECTION_ENABLED;
+import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_MODEL_TRACING_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_MULTI_TENANCY_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_OFFLINE_BATCH_INFERENCE_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_OFFLINE_BATCH_INGESTION_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_RAG_PIPELINE_FEATURE_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_REMOTE_INFERENCE_ENABLED;
 import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_STATIC_METRIC_COLLECTION_ENABLED;
+import static org.opensearch.ml.common.settings.MLCommonsSettings.ML_COMMONS_TRACING_ENABLED;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -61,6 +65,11 @@ public class MLFeatureEnabledSetting {
 
     private volatile Boolean isAgenticMemoryEnabled;
 
+    private volatile Boolean isTracingEnabled;
+    private volatile Boolean isAgentTracingEnabled;
+    private volatile Boolean isConnectorTracingEnabled;
+    private volatile Boolean isModelTracingEnabled;
+
     private final List<SettingsChangeListener> listeners = new ArrayList<>();
 
     public MLFeatureEnabledSetting(ClusterService clusterService, Settings settings) {
@@ -80,6 +89,10 @@ public MLFeatureEnabledSetting(ClusterService clusterService, Settings settings)
         isAgenticSearchEnabled = ML_COMMONS_AGENTIC_SEARCH_ENABLED.get(settings);
         isMcpConnectorEnabled = ML_COMMONS_MCP_CONNECTOR_ENABLED.get(settings);
         isAgenticMemoryEnabled = ML_COMMONS_AGENTIC_MEMORY_ENABLED.get(settings);
+        isTracingEnabled = ML_COMMONS_TRACING_ENABLED.get(settings);
+        isAgentTracingEnabled = ML_COMMONS_AGENT_TRACING_ENABLED.get(settings);
+        isConnectorTracingEnabled = ML_COMMONS_CONNECTOR_TRACING_ENABLED.get(settings);
+        isModelTracingEnabled = ML_COMMONS_MODEL_TRACING_ENABLED.get(settings);
 
         clusterService
             .getClusterSettings()
@@ -106,6 +119,15 @@ public MLFeatureEnabledSetting(ClusterService clusterService, Settings settings)
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ML_COMMONS_AGENTIC_SEARCH_ENABLED, it -> isAgenticSearchEnabled = it);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ML_COMMONS_MCP_CONNECTOR_ENABLED, it -> isMcpConnectorEnabled = it);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ML_COMMONS_AGENTIC_MEMORY_ENABLED, it -> isAgenticMemoryEnabled = it);
+        clusterService
+            .getClusterSettings()
+            .addSettingsUpdateConsumer(MLCommonsSettings.ML_COMMONS_AGENT_TRACING_ENABLED, it -> isAgentTracingEnabled = it);
+        clusterService
+            .getClusterSettings()
+            .addSettingsUpdateConsumer(MLCommonsSettings.ML_COMMONS_CONNECTOR_TRACING_ENABLED, it -> isConnectorTracingEnabled = it);
+        clusterService
+            .getClusterSettings()
+            .addSettingsUpdateConsumer(MLCommonsSettings.ML_COMMONS_MODEL_TRACING_ENABLED, it -> isModelTracingEnabled = it);
     }
 
     /**
@@ -212,6 +234,22 @@ public boolean isAgenticMemoryEnabled() {
         return isAgenticMemoryEnabled;
     }
 
+    public boolean isTracingEnabled() {
+        return isTracingEnabled;
+    }
+
+    public boolean isAgentTracingEnabled() {
+        return isAgentTracingEnabled;
+    }
+
+    public boolean isConnectorTracingEnabled() {
+        return isConnectorTracingEnabled;
+    }
+
+    public boolean isModelTracingEnabled() {
+        return isModelTracingEnabled;
+    }
+
     @VisibleForTesting
     public void notifyMultiTenancyListeners(boolean isEnabled) {
         for (SettingsChangeListener listener : listeners) {

common/src/test/java/org/opensearch/ml/common/settings/MLFeatureEnabledSettingTests.java

@@ -47,7 +47,11 @@ public void setUp() {
                     MLCommonsSettings.ML_COMMONS_EXECUTE_TOOL_ENABLED,
                     MLCommonsSettings.ML_COMMONS_AGENTIC_SEARCH_ENABLED,
                     MLCommonsSettings.ML_COMMONS_MCP_CONNECTOR_ENABLED,
-                    MLCommonsSettings.ML_COMMONS_AGENTIC_MEMORY_ENABLED
+                    MLCommonsSettings.ML_COMMONS_AGENTIC_MEMORY_ENABLED,
+                    MLCommonsSettings.ML_COMMONS_TRACING_ENABLED,
+                    MLCommonsSettings.ML_COMMONS_AGENT_TRACING_ENABLED,
+                    MLCommonsSettings.ML_COMMONS_CONNECTOR_TRACING_ENABLED,
+                    MLCommonsSettings.ML_COMMONS_MODEL_TRACING_ENABLED
                 )
         );
         when(mockClusterService.getClusterSettings()).thenReturn(mockClusterSettings);
@@ -72,6 +76,10 @@ public void testDefaults_allFeaturesEnabled() {
             .put("plugins.ml_commons.mcp_connector_enabled", true)
             .put("plugins.ml_commons.agentic_search_enabled", true)
             .put("plugins.ml_commons.agentic_memory_enabled", true)
+            .put("plugins.ml_commons.tracing_enabled", true)
+            .put("plugins.ml_commons.agent_tracing_enabled", true)
+            .put("plugins.ml_commons.connector_tracing_enabled", true)
+            .put("plugins.ml_commons.model_tracing_enabled", true)
             .build();
 
         MLFeatureEnabledSetting setting = new MLFeatureEnabledSetting(mockClusterService, settings);
@@ -91,6 +99,10 @@ public void testDefaults_allFeaturesEnabled() {
         assertTrue(setting.isMcpConnectorEnabled());
         assertTrue(setting.isAgenticSearchEnabled());
         assertTrue(setting.isAgenticMemoryEnabled());
+        assertTrue(setting.isTracingEnabled());
+        assertTrue(setting.isAgentTracingEnabled());
+        assertTrue(setting.isConnectorTracingEnabled());
+        assertTrue(setting.isModelTracingEnabled());
     }
 
     @Test
@@ -112,6 +124,10 @@ public void testDefaults_someFeaturesDisabled() {
             .put("plugins.ml_commons.mcp_connector_enabled", false)
             .put("plugins.ml_commons.agentic_search_enabled", false)
             .put("plugins.ml_commons.agentic_memory_enabled", false)
+            .put("plugins.ml_commons.tracing_enabled", false)
+            .put("plugins.ml_commons.agent_tracing_enabled", false)
+            .put("plugins.ml_commons.connector_tracing_enabled", false)
+            .put("plugins.ml_commons.model_tracing_enabled", false)
             .build();
 
         MLFeatureEnabledSetting setting = new MLFeatureEnabledSetting(mockClusterService, settings);
@@ -131,6 +147,10 @@ public void testDefaults_someFeaturesDisabled() {
         assertFalse(setting.isMcpConnectorEnabled());
         assertFalse(setting.isAgenticSearchEnabled());
         assertFalse(setting.isAgenticMemoryEnabled());
+        assertFalse(setting.isTracingEnabled());
+        assertFalse(setting.isAgentTracingEnabled());
+        assertFalse(setting.isConnectorTracingEnabled());
+        assertFalse(setting.isModelTracingEnabled());
     }
 
     @Test

docs/tutorials/agent_tracing/data-prepper-config.yaml

@@ -0,0 +1,3 @@
+# Sample Data Prepper configuration for agent tracing
+# This is an example configuration for development purposes
+ssl: false

docs/tutorials/agent_tracing/docker-compose.yml

@@ -0,0 +1,97 @@
+# Sample Docker Compose configuration for agent tracing setup
+# This is an example configuration for development purposes
+version: '3'
+services:
+  opensearch-node1:
+    image: opensearchproject/opensearch:3.1.0 # NOTE: Ensure versions are consistent across all OpenSearch services (OpenSearch, Dashboards, plugins)
+    container_name: opensearch-node1
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch-node1
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dopensearch.experimental.feature.telemetry.enabled=true"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=<PASSWORD>
+      - DISABLE_SECURITY_PLUGIN=true
+      - opensearch.experimental.feature.telemetry.enabled=true
+      - telemetry.feature.tracer.enabled=true
+      - telemetry.tracer.enabled=true
+      - telemetry.tracer.sampler.probability=1.0
+      - telemetry.otel.tracer.span.exporter.class=io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter
+      - plugins.ml_commons.tracing_enabled=true
+      - plugins.ml_commons.agent_tracing_enabled=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+      # USE THE BELOW TO MOUNT YOUR LOCAL BUILD TO AN INSTANCE
+      - <YOUR_ML_COMMONS_DIRECTORY>/plugin/build/distributions:/usr/share/opensearch/<YOUR_NAME> 
+    command: >
+      bash -c "
+      bin/opensearch-plugin remove opensearch-skills; 
+      bin/opensearch-plugin remove opensearch-ml; 
+      bin/opensearch-plugin install --batch telemetry-otel;
+      bin/opensearch-plugin install --batch file:///usr/share/opensearch/<YOUR_NAME>/opensearch-ml-3.1.0.0-SNAPSHOT.zip;
+      OPENSEARCH_LOG4J_CONFIG_FILE=/usr/share/opensearch/config/telemetry-log4j2.xml ./opensearch-docker-entrypoint.sh opensearch"
+    ports:
+      - 9200:9200
+      - 9600:9600 # required for Performance Analyzer
+    networks:
+      - opensearch-net
+    depends_on:
+      - otel-collector
+    extra_hosts:
+      - "localhost:172.17.0.1"  # This maps localhost to the Docker host IP
+
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:3.1.0 # Must match OpenSearch version
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]'
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+    networks:
+      - opensearch-net
+    depends_on:
+      - opensearch-node1
+
+  data-prepper:
+    restart: unless-stopped
+    container_name: data-prepper
+    image: opensearchproject/data-prepper:2
+    volumes:
+      - ./data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+      - ./pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - <PATH_TO_INDEX_MAPPING>/ml_agent_trace.json:/usr/share/data-prepper/ml_agent_trace.json
+    ports:
+      - "21890:21890"
+    networks:
+      - opensearch-net
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317"  # OTLP gRPC port
+      - "4318:4318"  # OTLP HTTP port
+    networks:
+      - opensearch-net
+    depends_on:
+      - data-prepper
+
+volumes:
+  opensearch-data1:
+
+networks:
+  opensearch-net: