feat: Add mode to enable agent to not harvest until user consent (#656)

src/common/config/state/init.js

@@ -9,7 +9,7 @@ const model = () => {
   }
   return {
     privacy: { cookies_enabled: true }, // *cli - per discussion, default should be true
-    ajax: { deny_list: undefined, block_internal: true, enabled: true, harvestTimeSeconds: 10 },
+    ajax: { deny_list: undefined, block_internal: true, enabled: true, harvestTimeSeconds: 10, autoStart: true },
     distributed_tracing: {
       enabled: undefined,
       exclude_newrelic_header: undefined,
@@ -24,15 +24,16 @@ const model = () => {
     },
     ssl: undefined,
     obfuscate: undefined,
-    jserrors: { enabled: true, harvestTimeSeconds: 10 },
-    metrics: { enabled: true },
-    page_action: { enabled: true, harvestTimeSeconds: 30 },
-    page_view_event: { enabled: true },
-    page_view_timing: { enabled: true, harvestTimeSeconds: 30, long_task: false },
-    session_trace: { enabled: true, harvestTimeSeconds: 10 },
+    jserrors: { enabled: true, harvestTimeSeconds: 10, autoStart: true },
+    metrics: { enabled: true, autoStart: true },
+    page_action: { enabled: true, harvestTimeSeconds: 30, autoStart: true },
+    page_view_event: { enabled: true, autoStart: true },
+    page_view_timing: { enabled: true, harvestTimeSeconds: 30, long_task: false, autoStart: true },
+    session_trace: { enabled: true, harvestTimeSeconds: 10, autoStart: true },
     harvest: { tooManyRequestsDelay: 60 },
     session_replay: {
       // feature settings
+      autoStart: true,
       enabled: false,
       harvestTimeSeconds: 60,
       sampleRate: 0.1,
@@ -60,7 +61,7 @@ const model = () => {
         hiddenState.maskInputOptions = { ...val, password: true }
       }
     },
-    spa: { enabled: true, harvestTimeSeconds: 10 }
+    spa: { enabled: true, harvestTimeSeconds: 10, autoStart: true }
   }
 }
 

src/common/config/state/init.test.js

@@ -23,6 +23,6 @@ test('set/getConfiguration works correctly', () => {
 test('getConfigurationValue parses path correctly', () => {
   setConfiguration('ab', { page_action: { harvestTimeSeconds: 1000 } })
   expect(getConfigurationValue('ab', '')).toBeUndefined()
-  expect(getConfigurationValue('ab', 'page_action')).toEqual({ enabled: true, harvestTimeSeconds: 1000 })
+  expect(getConfigurationValue('ab', 'page_action')).toEqual({ enabled: true, harvestTimeSeconds: 1000, autoStart: true })
   expect(getConfigurationValue('ab', 'page_action.harvestTimeSeconds')).toEqual(1000)
 })

src/common/drain/drain.js

@@ -44,7 +44,6 @@ function curateRegistry (agentIdentifier) {
  */
 export function drain (agentIdentifier = '', featureName = 'feature') {
   curateRegistry(agentIdentifier)
-
   // If the feature for the specified agent is not in the registry, that means the instrument file was bypassed.
   // This could happen in tests, or loaders that directly import the aggregator. In these cases it is safe to
   // drain the feature group immediately rather than waiting to drain all at once.
@@ -59,6 +58,7 @@ export function drain (agentIdentifier = '', featureName = 'feature') {
   if (items.every(([key, values]) => values.staged)) {
     items.sort((a, b) => a[1].priority - b[1].priority)
     items.forEach(([group]) => {
+      registry[agentIdentifier].delete(group)
       drainGroup(group)
     })
   }

src/common/util/feature-flags.js

@@ -4,7 +4,6 @@
  */
 import { ee } from '../event-emitter/contextual-ee'
 import { handle } from '../event-emitter/handle'
-import { drain } from '../drain/drain'
 import { FEATURE_NAMES } from '../../loaders/features/features'
 
 const bucketMap = {
@@ -15,26 +14,25 @@ const bucketMap = {
   sr: [FEATURE_NAMES.sessionReplay, FEATURE_NAMES.sessionTrace]
 }
 
+const sentIds = new Set()
+
 /** Note that this function only processes each unique flag ONCE, with the first occurrence of each flag and numeric value determining its switch on/off setting. */
 export function activateFeatures (flags, agentIdentifier) {
   const sharedEE = ee.get(agentIdentifier)
   if (!(flags && typeof flags === 'object')) return
 
-  Object.entries(flags).forEach(([flag, num]) => {
-    if (activatedFeatures[flag] !== undefined) return
-
-    if (bucketMap[flag]) {
-      bucketMap[flag].forEach(feat => {
-        if (!num) handle('block-' + flag, [], undefined, feat, sharedEE)
-        else handle('feat-' + flag, [], undefined, feat, sharedEE)
-
-        handle('rumresp-' + flag, [Boolean(num)], undefined, feat, sharedEE) // this is a duplicate of feat-/block- but makes awaiting for 1 event easier than 2
-      })
-    } else if (num) handle('feat-' + flag, [], undefined, undefined, sharedEE) // not sure what other flags are overlooked, but there's a test for ones not in the map --
-    // ^^^ THIS DOESN'T ACTUALLY DO ANYTHHING AS UNDEFINED/FEATURE GROUP ISN'T DRAINED
-
-    activatedFeatures[flag] = Boolean(num)
-  })
+  if (!sentIds.has(agentIdentifier)) {
+    Object.entries(flags).forEach(([flag, num]) => {
+      if (bucketMap[flag]) {
+        bucketMap[flag].forEach(feat => {
+          if (!num) handle('block-' + flag, [], undefined, feat, sharedEE)
+          else handle('feat-' + flag, [], undefined, feat, sharedEE)
+          handle('rumresp-' + flag, [Boolean(num)], undefined, feat, sharedEE) // this is a duplicate of feat-/block- but makes awaiting for 1 event easier than 2
+        })
+      } else if (num) handle('feat-' + flag, [], undefined, undefined, sharedEE) // not sure what other flags are overlooked, but there's a test for ones not in the map --
+      activatedFeatures[flag] = Boolean(num)
+    })
+  }
 
   // Let the features waiting on their respective flags know that RUM response was received and that any missing flags are interpreted as bad entitlement / "off".
   // Hence, those features will not be hanging forever if their flags aren't included in the response.
@@ -44,7 +42,7 @@ export function activateFeatures (flags, agentIdentifier) {
       activatedFeatures[flag] = false
     }
   })
-  drain(agentIdentifier, FEATURE_NAMES.pageViewEvent)
+  sentIds.add(agentIdentifier)
 }
 
 export const activatedFeatures = {}

src/common/util/feature-flags.test.js

@@ -57,7 +57,6 @@ test('emits the right events when feature flag = 1', () => {
   expect(handleModule.handle).toHaveBeenCalledTimes(14)
   expect(handleModule.handle).toHaveBeenNthCalledWith(1, 'feat-stn', [], undefined, FEATURE_NAMES.sessionTrace, sharedEE)
   expect(handleModule.handle).toHaveBeenLastCalledWith('rumresp-sr', [true], undefined, FEATURE_NAMES.sessionTrace, sharedEE)
-  expect(drainModule.drain).toHaveBeenCalledWith(agentIdentifier, 'page_view_event')
 
   Object.keys(flags).forEach(flag => { flags[flag] = true })
   expect(activatedFeatures).toEqual(flags)
@@ -74,7 +73,6 @@ test('emits the right events when feature flag = 0', () => {
   expect(handleModule.handle).toHaveBeenCalledTimes(14)
   expect(handleModule.handle).toHaveBeenNthCalledWith(1, 'block-stn', [], undefined, FEATURE_NAMES.sessionTrace, sharedEE)
   expect(handleModule.handle).toHaveBeenLastCalledWith('rumresp-sr', [false], undefined, FEATURE_NAMES.sessionTrace, sharedEE)
-  expect(drainModule.drain).toHaveBeenCalledWith(agentIdentifier, 'page_view_event')
 
   Object.keys(flags).forEach(flag => { flags[flag] = false })
   expect(activatedFeatures).toEqual(flags)
@@ -92,7 +90,5 @@ test('only the first activate of the same feature is respected', () => {
   expect(handleModule.handle).toHaveBeenNthCalledWith(1, 'feat-stn', [], undefined, 'session_trace', sharedEE1)
   expect(handleModule.handle).toHaveBeenNthCalledWith(2, 'rumresp-stn', [true], undefined, 'session_trace', sharedEE1)
   expect(handleModule.handle).not.toHaveBeenNthCalledWith(1, 'feat-stn', [], undefined, 'session_trace', sharedEE2)
-  expect(drainModule.drain).toHaveBeenCalledWith(agentIdentifier, 'page_view_event')
-  expect(drainModule.drain).toHaveBeenCalledTimes(2)
   expect(activatedFeatures.stn).toBeTruthy()
 })

src/features/ajax/aggregate/index.js

@@ -10,7 +10,6 @@ import { getConfiguration, getInfo, getRuntime } from '../../../common/config/co
 import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { setDenyList, shouldCollectEvent } from '../../../common/deny-list/deny-list'
 import { FEATURE_NAME } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { FEATURE_NAMES } from '../../../loaders/features/features'
 import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants'
 import { AggregateBase } from '../../utils/aggregate-base'
@@ -24,7 +23,7 @@ export class Aggregate extends AggregateBase {
 
     register('xhr', storeXhr, this.featureName, this.ee)
     if (!allAjaxIsEnabled) {
-      drain(this.agentIdentifier, this.featureName)
+      this.drain()
       return // feature will only collect timeslice metrics & ajax trace nodes if it's not fully enabled
     }
 
@@ -66,7 +65,7 @@ export class Aggregate extends AggregateBase {
 
     ee.on(`drain-${this.featureName}`, () => { scheduler.startTimer(harvestTimeSeconds) })
 
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
 
     function storeXhr (params, metrics, startTime, endTime, type) {
       metrics.time = startTime

src/features/jserrors/aggregate/index.js

@@ -18,7 +18,6 @@ import { now } from '../../../common/timing/now'
 import { globalScope } from '../../../common/constants/runtime'
 
 import { FEATURE_NAME } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { FEATURE_NAMES } from '../../../loaders/features/features'
 import { AggregateBase } from '../../utils/aggregate-base'
 
@@ -63,7 +62,7 @@ export class Aggregate extends AggregateBase {
       scheduler.stopTimer(true)
     }, this.featureName, this.ee)
 
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
   }
 
   onHarvestStarted (options) {

src/features/metrics/aggregate/index.js

@@ -2,7 +2,6 @@ import { getRuntime } from '../../../common/config/config'
 import { registerHandler } from '../../../common/event-emitter/register-handler'
 import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { FEATURE_NAME, SUPPORTABILITY_METRIC, CUSTOM_METRIC, SUPPORTABILITY_METRIC_CHANNEL, CUSTOM_METRIC_CHANNEL } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { getFrameworks } from './framework-detection'
 import { isFileProtocol } from '../../../common/url/protocol'
 import { getRules, validateRules } from '../../../common/util/obfuscate'
@@ -34,7 +33,7 @@ export class Aggregate extends AggregateBase {
     scheduler = new HarvestScheduler('jserrors', { onUnload: () => this.unload() }, this)
     scheduler.harvest.on('jserrors', () => ({ body: this.aggregator.take(['cm', 'sm']) }))
 
-    drain(this.agentIdentifier, this.featureName) // regardless if this is blocked or not, drain is needed to unblock other features from harvesting (counteract registerDrain)
+    this.drain()
   }
 
   storeSupportabilityMetrics (name, value) {

src/features/page_action/aggregate/index.js

@@ -10,7 +10,6 @@ import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { cleanURL } from '../../../common/url/clean-url'
 import { getConfigurationValue, getInfo, getRuntime } from '../../../common/config/config'
 import { FEATURE_NAME } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { isBrowserScope } from '../../../common/constants/runtime'
 import { AggregateBase } from '../../utils/aggregate-base'
 
@@ -32,17 +31,17 @@ export class Aggregate extends AggregateBase {
 
     register('api-addPageAction', (...args) => this.addPageAction(...args), this.featureName, this.ee)
 
-    var scheduler = new HarvestScheduler('ins', { onFinished: (...args) => this.onHarvestFinished(...args) }, this)
-    scheduler.harvest.on('ins', (...args) => this.onHarvestStarted(...args))
-    this.ee.on(`drain-${this.featureName}`, () => { if (!this.blocked) scheduler.startTimer(this.harvestTimeSeconds, 0) })
-
-    // if rum response determines that customer lacks entitlements for ins endpoint, block it
-    register('block-ins', () => {
-      this.blocked = true
-      scheduler.stopTimer(true)
-    }, this.featureName, this.ee)
+    this.waitForFlags(['ins']).then(([enabled]) => {
+      if (enabled) {
+        const scheduler = new HarvestScheduler('ins', { onFinished: (...args) => this.onHarvestFinished(...args) }, this)
+        scheduler.harvest.on('ins', (...args) => this.onHarvestStarted(...args))
+        scheduler.startTimer(this.harvestTimeSeconds, 0)
+      } else {
+        this.blocked = true
+      }
+    })
 
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
   }
 
   onHarvestStarted (options) {

src/features/page_view_event/aggregate/index.js

@@ -9,7 +9,6 @@ import { getConfigurationValue, getInfo, getRuntime } from '../../../common/conf
 import { Harvest } from '../../../common/harvest/harvest'
 import * as CONSTANTS from '../constants'
 import { getActivatedFeaturesFlags } from './initialized-features'
-import { drain } from '../../../common/drain/drain'
 import { activateFeatures } from '../../../common/util/feature-flags'
 import { warn } from '../../../common/util/console'
 import { AggregateBase } from '../../utils/aggregate-base'
@@ -131,7 +130,7 @@ export class Aggregate extends AggregateBase {
 
         try {
           activateFeatures(JSON.parse(responseText), this.agentIdentifier)
-          drain(this.agentIdentifier, this.featureName)
+          this.drain()
         } catch (err) {
           this.ee.abort()
           warn('RUM call failed. Agent shutting down.')

src/features/page_view_timing/aggregate/index.js

@@ -15,7 +15,6 @@ import { cleanURL } from '../../../common/url/clean-url'
 import { handle } from '../../../common/event-emitter/handle'
 import { getInfo, getConfigurationValue, getRuntime } from '../../../common/config/config'
 import { FEATURE_NAME } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { FEATURE_NAMES } from '../../../loaders/features/features'
 import { AggregateBase } from '../../utils/aggregate-base'
 
@@ -128,7 +127,7 @@ export class Aggregate extends AggregateBase {
     // send initial data sooner, then start regular
     this.ee.on(`drain-${this.featureName}`, () => { this.scheduler.startTimer(harvestTimeSeconds, initialHarvestSeconds) })
 
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
   }
 
   // takes an attributes object and appends connection attributes if available

src/features/session_replay/aggregate/index.js

@@ -10,7 +10,6 @@
  * functionality is validated and a full user experience is curated.
  */
 
-import { drain } from '../../../common/drain/drain'
 import { registerHandler } from '../../../common/event-emitter/register-handler'
 import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { FEATURE_NAME } from '../constants'
@@ -128,7 +127,7 @@ export class Aggregate extends AggregateBase {
         Math.random() < getConfigurationValue(this.agentIdentifier, 'session_replay.sampleRate')
       )).then(() => sharedChannel.onReplayReady(this.mode)) // notify watchers that replay started with the mode
 
-      drain(this.agentIdentifier, this.featureName)
+      this.drain()
     }
   }
 

src/features/session_trace/aggregate/index.js

@@ -8,7 +8,6 @@ import { parseUrl } from '../../../common/url/parse-url'
 import { getConfigurationValue, getRuntime } from '../../../common/config/config'
 import { now } from '../../../common/timing/now'
 import { FEATURE_NAME } from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { HandlerCache } from '../../utils/handler-cache'
 import { MODE, SESSION_EVENTS } from '../../../common/session/session-entity'
 import { getSessionReplayMode } from '../../session_replay/replay-mode'
@@ -156,7 +155,7 @@ export class Aggregate extends AggregateBase {
     registerHandler('bstApi', (...args) => operationalGate.settle(() => this.storeSTN(...args)), this.featureName, this.ee)
     registerHandler('errorAgg', (...args) => operationalGate.settle(() => this.storeErrorAgg(...args)), this.featureName, this.ee)
     registerHandler('pvtAdded', (...args) => operationalGate.settle(() => this.processPVT(...args)), this.featureName, this.ee)
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
   }
 
   startTracing (startupBuffer, dontStartHarvestYet = false) {

src/features/spa/aggregate/index.js

@@ -17,7 +17,6 @@ import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler'
 import { Serializer } from './serializer'
 import { ee } from '../../../common/event-emitter/contextual-ee'
 import * as CONSTANTS from '../constants'
-import { drain } from '../../../common/drain/drain'
 import { FEATURE_NAMES } from '../../../loaders/features/features'
 import { AggregateBase } from '../../utils/aggregate-base'
 
@@ -728,6 +727,6 @@ export class Aggregate extends AggregateBase {
       return enabled !== false
     }
 
-    drain(this.agentIdentifier, this.featureName)
+    this.drain()
   }
 }

src/features/utils/aggregate-base.js

@@ -3,6 +3,7 @@ import { FeatureBase } from './feature-base'
 import { getInfo, isConfigured, getRuntime } from '../../common/config/config'
 import { configure } from '../../loaders/configure/configure'
 import { gosCDN } from '../../common/window/nreum'
+import { drain } from '../../common/drain/drain'
 
 export class AggregateBase extends FeatureBase {
   constructor (...args) {
@@ -25,6 +26,10 @@ export class AggregateBase extends FeatureBase {
     )
   }
 
+  drain () {
+    drain(this.agentIdentifier, this.featureName)
+  }
+
   /**
    * Checks for additional `jsAttributes` items to support backward compatibility with implementations of the agent where
    * loader configurations may appear after the loader code is executed.

src/features/utils/aggregate-base.test.js

@@ -31,7 +31,8 @@ jest.mock('../../loaders/configure/configure', () => ({
 }))
 jest.mock('../../common/window/nreum', () => ({
   __esModule: true,
-  gosCDN: jest.fn().mockReturnValue({})
+  gosCDN: jest.fn().mockReturnValue({}),
+  gosNREUM: jest.fn().mockReturnValue({})
 }))
 
 let agentIdentifier

src/features/utils/instrument-base.js

@@ -45,7 +45,10 @@ export class InstrumentBase extends FeatureBase {
     */
     this.onAggregateImported = undefined
 
-    if (auto) registerDrain(agentIdentifier, featureName)
+    /** used in conjunction with newrelic.start() to defer harvesting in features */
+    if (getConfigurationValue(this.agentIdentifier, `${this.featureName}.autoStart`) === false) this.auto = false
+    /** if the feature requires opt-in (!auto-start), it will get registered once the api has been called */
+    if (this.auto) registerDrain(agentIdentifier, featureName)
   }
 
   /**
@@ -55,7 +58,21 @@ export class InstrumentBase extends FeatureBase {
    * @returns void
    */
   importAggregator (argsObjFromInstrument = {}) {
-    if (this.featAggregate || !this.auto) return
+    if (this.featAggregate) return
+
+    if (!this.auto) {
+      // this feature requires an opt in...
+      // wait for API to be called
+      this.ee.on(`${this.featureName}-opt-in`, () => {
+        // register the feature to drain only once the API has been called, it will drain when importAggregator finishes for all the features
+        // called by the api in that cycle
+        registerDrain(this.agentIdentifier, this.featureName)
+        this.auto = true
+        this.importAggregator()
+      })
+      return
+    }
+
     const enableSessionTracking = isBrowserScope && getConfigurationValue(this.agentIdentifier, 'privacy.cookies_enabled') === true
     let loadedSuccessfully
     this.onAggregateImported = new Promise(resolve => {
@@ -74,7 +91,7 @@ export class InstrumentBase extends FeatureBase {
       }
 
       /**
-       * Note this try-catch differs from the one in Agent.start() in that it's placed later in a page's lifecycle and
+       * Note this try-catch differs from the one in Agent.run() in that it's placed later in a page's lifecycle and
        * it's only responsible for aborting its one specific feature, rather than all.
        */
       try {