DEBUG-2334 dynamic instrumentation probe manager + instrumentation integration tests

.standard_todo.yml

@@ -38,6 +38,7 @@ ignore:
 - spec/datadog/appsec/**/**
 - spec/datadog/benchmark/**/**
 - spec/datadog/core/**/**
+- spec/datadog/di/integration/*_test_class*.rb
 - spec/datadog/kit/**/**
 - spec/datadog/tracing/**/**
 - spec/support/**/**

lib/datadog/di.rb

@@ -1,21 +1,44 @@
 # frozen_string_literal: true
 
 require_relative 'di/error'
-require_relative 'di/configuration'
 require_relative 'di/code_tracker'
+require_relative 'di/component'
+require_relative 'di/configuration'
 require_relative 'di/extensions'
 require_relative 'di/instrumenter'
 require_relative 'di/probe'
+require_relative 'di/probe_builder'
+require_relative 'di/probe_manager'
+require_relative 'di/probe_notification_builder'
+require_relative 'di/probe_notifier_worker'
 require_relative 'di/redactor'
 require_relative 'di/serializer'
 require_relative 'di/transport'
 require_relative 'di/utils'
 
+if defined?(ActiveRecord::Base)
+  # The third-party library integrations need to be loaded after the
+  # third-party libraries are loaded. Tracing and appsec use Railtie
+  # to delay integrations until all of the application's dependencies
+  # are loaded, when running under Rails. We should do the same here in
+  # principle, however DI currently only has an ActiveRecord integration
+  # and AR should be loaded before any application code is loaded, being
+  # part of Rails, therefore for now we should be OK to just require the
+  # AR integration from here.
+  require_relative 'di/contrib/active_record'
+end
+
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
   #
   # @api private
   module DI
+    class << self
+      def enabled?
+        Datadog.configuration.dynamic_instrumentation.enabled
+      end
+    end
+
     # Expose DI to global shared objects
     Extensions.activate!
 
@@ -52,6 +75,31 @@ def deactivate_tracking!
       def code_tracking_active?
         code_tracker&.active? || false
       end
+
+      def component
+        # TODO uncomment when remote is merged
+        #Datadog.send(:components).dynamic_instrumentation
+      end
+    end
+  end
+end
+
+=begin not yet enabled
+# :script_compiled trace point was added in Ruby 2.6.
+if RUBY_VERSION >= '2.6'
+  begin
+    # Activate code tracking by default because line trace points will not work
+    # without it.
+    Datadog::DI.activate_tracking!
+  rescue => exc
+    if defined?(Datadog.logger)
+      Datadog.logger.warn("Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+    else
+      # We do not have Datadog logger potentially because DI code tracker is
+      # being loaded early in application boot process and the rest of datadog
+      # wasn't loaded yet. Output to standard error.
+      warn("Failed to activate code tracking for DI: #{exc.class}: #{exc}")
     end
   end
 end
+=end

lib/datadog/di/code_tracker.rb

@@ -79,6 +79,9 @@ def start
                 registry[path] = tp.instruction_sequence
               end
             end
+
+            DI.component&.probe_manager&.install_pending_line_probes(path)
+
           # Since this method normally is called from customer applications,
           # rescue any exceptions that might not be handled to not break said
           # customer applications.

lib/datadog/di/component.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Component for dynamic instrumentation.
+    #
+    # Only one instance of the Component should ever be active;
+    # if configuration is changed, the old distance should be shut down
+    # prior to the new instance being created.
+    #
+    # The Component instance stores all state related to DI, for example
+    # which probes have been retrieved via remote config,
+    # intalled tracepoints and so on. Component will clean up all
+    # resources and installed tracepoints upon shutdown.
+    class Component
+      class << self
+        def build(settings, agent_settings, telemetry: nil)
+          return unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+
+          unless settings.respond_to?(:remote) && settings.remote.enabled
+            Datadog.logger.debug("Dynamic Instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
+            return
+          end
+
+          return unless environment_supported?(settings)
+
+          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+        end
+
+        def build!(settings, agent_settings, telemetry: nil)
+          unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+            raise "Requested DI component but DI is not enabled in settings"
+          end
+
+          unless settings.respond_to?(:remote) && settings.remote.enabled
+            raise "Requested DI component but remote config is not enabled in settings"
+          end
+
+          unless environment_supported?(settings)
+            raise "DI does not support the environment (development or Ruby version too low or not MRI)"
+          end
+
+          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+        end
+
+        # Checks whether the runtime environment is supported by
+        # dynamic instrumentation. Currently we only require that, if Rails
+        # is used, that Rails environment is not development because
+        # DI does not currently support code unloading and reloading.
+        def environment_supported?(settings)
+          # TODO add tests?
+          unless settings.dynamic_instrumentation.internal.development
+            if Datadog::Core::Environment::Execution.development?
+              Datadog.logger.debug("Not enabling dynamic instrumentation because we are in development environment")
+              return false
+            end
+          end
+          if RUBY_ENGINE != 'ruby' || RUBY_VERSION < '2.6'
+            Datadog.logger.debug("Not enabling dynamic instrumentation because of unsupported Ruby version")
+            return false
+          end
+          true
+        end
+      end
+
+      def initialize(settings, agent_settings, logger, code_tracker: nil, telemetry: nil)
+        @settings = settings
+        @agent_settings = agent_settings
+        @logger = logger
+        @telemetry = telemetry
+        @redactor = Redactor.new(settings)
+        @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
+        @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
+        @transport = Transport.new(agent_settings)
+        @probe_notifier_worker = ProbeNotifierWorker.new(settings, transport, logger, telemetry: telemetry)
+        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
+        @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
+        probe_notifier_worker.start
+      end
+
+      attr_reader :settings
+      attr_reader :agent_settings
+      attr_reader :logger
+      attr_reader :telemetry
+      attr_reader :instrumenter
+      attr_reader :transport
+      attr_reader :probe_notifier_worker
+      attr_reader :probe_notification_builder
+      attr_reader :probe_manager
+      attr_reader :redactor
+      attr_reader :serializer
+
+      # Shuts down dynamic instrumentation.
+      #
+      # Removes all code hooks and stops background threads.
+      #
+      # Does not clear out the code tracker, because it's only populated
+      # by code when code is compiled and therefore, if the code tracker
+      # was replaced by a new instance, the new instance of it wouldn't have
+      # any of the already loaded code tracked.
+      def shutdown!(replacement = nil)
+        probe_manager.clear_hooks
+        probe_manager.close
+        probe_notifier_worker.stop
+      end
+    end
+  end
+end

lib/datadog/di/probe.rb

@@ -165,6 +165,9 @@ def file_matches?(path)
       # For untargeted line trace points instrumented path will be nil.
       attr_accessor :instrumented_path
 
+      # TODO emitting_notified reads and writes should in theory be locked,
+      # however since DI is only implemented for MRI in practice the missing
+      # locking should not cause issues.
       attr_writer :emitting_notified
       def emitting_notified?
         !!@emitting_notified