Merge pull request #2474 from newrelic/sansgarçon

Introduce support for serverless operations (AWS Lambda for now)

.rubocop.yml

@@ -1249,10 +1249,12 @@ Style/MethodCallWithArgsParentheses:
     - add_dependency
     - add_development_dependency
     - catch
+    - debug
     - expect
     - fail
     - gem
     - include
+    - info
     - print
     - puts
     - pp

lib/new_relic/agent.rb

@@ -709,16 +709,19 @@ def tl_is_execution_traced?
     def add_custom_attributes(params) # THREAD_LOCAL_ACCESS
       record_api_supportability_metric(:add_custom_attributes)
 
-      if params.is_a?(Hash)
-        Transaction.tl_current&.add_custom_attributes(params)
-
-        segment = ::NewRelic::Agent::Tracer.current_segment
-        if segment
-          add_new_segment_attributes(params, segment)
-        end
-      else
+      unless params.is_a?(Hash)
         ::NewRelic::Agent.logger.warn("Bad argument passed to #add_custom_attributes. Expected Hash but got #{params.class}")
+        return
       end
+
+      if NewRelic::Agent.agent&.serverless?
+        ::NewRelic::Agent.logger.warn('Custom attributes are not supported in serverless mode')
+        return
+      end
+
+      Transaction.tl_current&.add_custom_attributes(params)
+      segment = ::NewRelic::Agent::Tracer.current_segment
+      add_new_segment_attributes(params, segment) if segment
     end
 
     def add_new_segment_attributes(params, segment)

lib/new_relic/agent/agent.rb

@@ -34,6 +34,7 @@
 require 'new_relic/environment_report'
 require 'new_relic/agent/attribute_filter'
 require 'new_relic/agent/adaptive_sampler'
+require 'new_relic/agent/serverless_handler'
 require 'new_relic/agent/connect/request_builder'
 require 'new_relic/agent/connect/response_handler'
 
@@ -96,6 +97,7 @@ def init_components
         @monotonic_gc_profiler = VM::MonotonicGCProfiler.new
         @adaptive_sampler = AdaptiveSampler.new(Agent.config[:sampling_target],
           Agent.config[:sampling_target_period_in_seconds])
+        @serverless_handler = ServerlessHandler.new
       end
 
       def init_event_handlers
@@ -172,6 +174,7 @@ module InstanceMethods
         attr_reader :transaction_event_recorder
         attr_reader :attribute_filter
         attr_reader :adaptive_sampler
+        attr_reader :serverless_handler
 
         def transaction_event_aggregator
           @transaction_event_recorder.transaction_event_aggregator
@@ -307,7 +310,7 @@ def reset_objects_with_locks
           @stats_engine = StatsEngine.new
         end
 
-        def flush_pipe_data
+        def flush_pipe_data # used only by resque
           if connected? && @service.is_a?(PipeService)
             transmit_data_types
           end

lib/new_relic/agent/agent_helpers/connect.rb

@@ -27,9 +27,13 @@ def disconnected?
           @connect_state == :disconnected
         end
 
-        # Don't connect if we're already connected, or if we tried to connect
-        # and were rejected with prejudice because of a license issue, unless
-        # we're forced to by force_reconnect.
+        def serverless?
+          Agent.config[:'serverless_mode.enabled']
+        end
+
+        # Don't connect if we're already connected, if we're in serverless mode,
+        # or if we tried to connect and were rejected with prejudice because of
+        # a license issue, unless we're forced to by force_reconnect.
         def should_connect?(force = false)
           force || (!connected? && !disconnected?)
         end
@@ -62,10 +66,8 @@ def log_error(error)
         # no longer try to connect to the server, saving the
         # application and the server load
         def handle_license_error(error)
-          ::NewRelic::Agent.logger.error( \
-            error.message, \
-            'Visit NewRelic.com to obtain a valid license key, or to upgrade your account.'
-          )
+          ::NewRelic::Agent.logger.error(error.message,
+            'Visit newrelic.com to obtain a valid license key, or to upgrade your account.')
           disconnect
         end
 
@@ -94,7 +96,7 @@ def event_harvest_config
         # connects, then configures the agent using the response from
         # the connect service
         def connect_to_server
-          request_builder = ::NewRelic::Agent::Connect::RequestBuilder.new( \
+          request_builder = ::NewRelic::Agent::Connect::RequestBuilder.new(
             @service,
             Agent.config,
             event_harvest_config,

lib/new_relic/agent/agent_helpers/start_worker_thread.rb

@@ -128,7 +128,7 @@ def deferred_work!(connection_options)
           catch_errors do
             NewRelic::Agent.disable_all_tracing do
               connect(connection_options)
-              if connected?
+              if NewRelic::Agent.instance.connected?
                 create_and_run_event_loop
                 # never reaches here unless there is a problem or
                 # the agent is exiting

lib/new_relic/agent/agent_helpers/startup.rb

@@ -124,6 +124,8 @@ def connect_in_foreground
         # Warn the user if they have configured their agent not to
         # send data, that way we can see this clearly in the log file
         def monitoring?
+          return false if Agent.config[:'serverless_mode.enabled']
+
           if Agent.config[:monitor_mode]
             true
           else
@@ -146,7 +148,6 @@ def has_license_key?
           end
         end
 
-        # A correct license key exists and is of the proper length
         def has_correct_license_key?
           has_license_key? && correct_license_length
         end

lib/new_relic/agent/agent_logger.rb

@@ -132,7 +132,8 @@ def create_null_logger
       end
 
       def wants_stdout?
-        ::NewRelic::Agent.config[:log_file_path].casecmp(NewRelic::STANDARD_OUT) == 0
+        ::NewRelic::Agent.config[:log_file_path].casecmp(NewRelic::STANDARD_OUT) == 0 ||
+          ::NewRelic::Agent.config[:'serverless_mode.enabled']
       end
 
       def find_or_create_file_path(path_setting, root)

lib/new_relic/agent/configuration/default_source.rb

@@ -1844,6 +1844,17 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :transform => DefaultSource.method(:convert_to_regexp_list),
           :description => 'Define transactions you want the agent to ignore, by specifying a list of patterns matching the URI you want to ignore. For more detail, see [the docs on ignoring specific transactions](/docs/agents/ruby-agent/api-guides/ignoring-specific-transactions/#config-ignoring).'
         },
+        # Serverless
+        :'serverless_mode.enabled' => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :transform => proc { |bool| NewRelic::Agent::ServerlessHandler.env_var_set? || bool },
+          :description => 'If `true`, the agent will operate in a streamlined mode suitable for use with short-lived ' \
+                          'serverless functions. NOTE: Only AWS Lambda functions are supported currently and this ' \
+                          "option is not intended for use without [New Relic's Ruby Lambda layer](https://docs.newrelic.com/docs/serverless-function-monitoring/aws-lambda-monitoring/get-started/monitoring-aws-lambda-serverless-monitoring/) offering."
+        },
         # Sidekiq
         :'sidekiq.args.include' => {
           default: NewRelic::EMPTY_ARRAY,

lib/new_relic/agent/configuration/environment_source.rb

@@ -99,7 +99,7 @@ def set_key_by_type(config_key, environment_key)
             elsif !value.nil?
               self[config_key] = true
             end
-          else
+          elsif !serverless?
             ::NewRelic::Agent.logger.info("#{environment_key} does not have a corresponding configuration setting (#{config_key} does not exist).")
             ::NewRelic::Agent.logger.info('Run `rake newrelic:config:docs` or visit https://docs.newrelic.com/docs/apm/agents/ruby-agent/configuration/ruby-agent-configuration to see a list of available configuration settings.')
             self[config_key] = value
@@ -114,6 +114,14 @@ def convert_environment_key_to_config_key(key)
         def collect_new_relic_environment_variable_keys
           ENV.keys.select { |key| key.match(SUPPORTED_PREFIXES) }
         end
+
+        # we can't rely on the :'serverless_mode.enabled' config parameter being
+        # set yet to signify serverless mode given that we're in the midst of
+        # building the config but we can always rely on the env var being set
+        # by the Lambda layer
+        def serverless?
+          NewRelic::Agent::ServerlessHandler.env_var_set?
+        end
       end
     end
   end

lib/new_relic/agent/configuration/yaml_source.rb

@@ -53,6 +53,8 @@ def failed?
         protected
 
         def validate_config_file_path(path)
+          return if NewRelic::Agent.config[:'serverless_mode.enabled']
+
           expanded_path = File.expand_path(path)
 
           if path.empty? || !File.exist?(expanded_path)

lib/new_relic/agent/connect/request_builder.rb

@@ -24,7 +24,7 @@ def connect_payload
             :host => local_host,
             :display_host => Agent.config[:'process_host.display_name'],
             :app_name => Agent.config[:app_name],
-            :language => 'ruby',
+            :language => LANGUAGE,
             :labels => Agent.config.parsed_labels,
             :agent_version => NewRelic::VERSION::STRING,
             :environment => @environment_report,

lib/new_relic/agent/distributed_tracing/distributed_trace_payload.rb

@@ -35,7 +35,7 @@ class DistributedTracePayload
 
       class << self
         def for_transaction(transaction)
-          return nil unless connected?
+          return nil unless Agent.instance.connected?
 
           payload = new
           payload.version = VERSION
@@ -101,10 +101,6 @@ def current_segment_id(transaction)
             transaction.current_segment.guid
           end
         end
-
-        def connected?
-          Agent.instance.connected?
-        end
       end
 
       attr_accessor :version,

lib/new_relic/agent/harvester.rb

@@ -38,7 +38,7 @@ def restart_in_children_enabled?
       end
 
       def harvest_thread_enabled?
-        !NewRelic::Agent.config[:disable_harvest_thread]
+        !NewRelic::Agent.config[:disable_harvest_thread] && !NewRelic::Agent.config[:'serverless_mode.enabled']
       end
 
       def restart_harvest_thread

lib/new_relic/agent/new_relic_service.rb

@@ -143,6 +143,9 @@ def build_metric_data_array(stats_hash)
       end
 
       def metric_data(stats_hash)
+        # let the serverless handler handle serialization
+        return NewRelic::Agent.agent.serverless_handler.metric_data(stats_hash) if NewRelic::Agent.agent.serverless?
+
         timeslice_start = stats_hash.started_at
         timeslice_end = stats_hash.harvested_at || Process.clock_gettime(Process::CLOCK_REALTIME)
         metric_data_array = build_metric_data_array(stats_hash)
@@ -154,6 +157,9 @@ def metric_data(stats_hash)
       end
 
       def error_data(unsent_errors)
+        # let the serverless handler handle serialization
+        return NewRelic::Agent.agent.serverless_handler.error_data(unsent_errors) if NewRelic::Agent.agent.serverless?
+
         invoke_remote(:error_data, [@agent_id, unsent_errors],
           :item_count => unsent_errors.size)
       end
@@ -554,15 +560,19 @@ def license_key
       # enough to be worth compressing, and handles any errors the
       # server may return
       def invoke_remote(method, payload = [], options = {})
+        return NewRelic::Agent.agent.serverless_handler.store_payload(method, payload) if NewRelic::Agent.agent.serverless?
+
         start_ts = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         request_send_ts, response_check_ts = nil
         data, encoding, size, serialize_finish_ts = marshal_payload(method, payload, options)
         prep_collector(method)
         response, request_send_ts, response_check_ts = invoke_remote_send_request(method, payload, data, encoding)
         @marshaller.load(decompress_response(response))
       ensure
-        record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
-        record_size_supportability_metrics(method, size, options[:item_count]) if size
+        unless NewRelic::Agent.agent.serverless?
+          record_timing_supportability_metrics(method, start_ts, serialize_finish_ts, request_send_ts, response_check_ts)
+          record_size_supportability_metrics(method, size, options[:item_count]) if size
+        end
       end
 
       def handle_serialization_error(method, e)