diff --git a/packages/analytics-js-common/src/types/ApplicationState.ts b/packages/analytics-js-common/src/types/ApplicationState.ts index 606ea8f62..222194957 100644 --- a/packages/analytics-js-common/src/types/ApplicationState.ts +++ b/packages/analytics-js-common/src/types/ApplicationState.ts @@ -117,6 +117,7 @@ export type SessionState = { readonly initialReferrer: Signal; readonly initialReferringDomain: Signal; readonly sessionInfo: Signal; + readonly authToken: Signal>; }; export type SourceConfigState = Signal; diff --git a/packages/analytics-js-common/src/types/userSessionStorageKeys.ts b/packages/analytics-js-common/src/types/userSessionStorageKeys.ts index 8a38fb588..565e56975 100644 --- a/packages/analytics-js-common/src/types/userSessionStorageKeys.ts +++ b/packages/analytics-js-common/src/types/userSessionStorageKeys.ts @@ -6,4 +6,5 @@ export type UserSessionKeys = | 'groupTraits' | 'initialReferrer' | 'initialReferringDomain' - | 'sessionInfo'; + | 'sessionInfo' + | 'authToken'; diff --git a/packages/analytics-js-plugins/__fixtures__/fixtures.ts b/packages/analytics-js-plugins/__fixtures__/fixtures.ts new file mode 100644 index 000000000..cf149a7c1 --- /dev/null +++ b/packages/analytics-js-plugins/__fixtures__/fixtures.ts @@ -0,0 +1,174 @@ +const sdkName = 'RudderLabs JavaScript SDK'; +const rudderEventPage = { + properties: { + path: '', + referrer: '', + search: '', + title: '', + url: '', + name: 'Cart Viewed', + category: 'Home', + referring_domain: '', + tab_url: 'http://localhost:3001/index.html', + initial_referrer: '$direct', + initial_referring_domain: '', + token: 'sample token', + }, + name: 'Cart Viewed', + category: 'Home', + type: 'page', + channel: 'web', + context: { + traits: { + name: 'John Doe', + title: 'CEO', + email: 'name.surname@domain.com', + company: 'Company123', + phone: '123-456-7890', + city: 'Austin', + postalCode: '12345', + country: 'US', + street: 'Sample Address', + state: 'TX', + }, + sessionId: 1695960096754, + consentManagement: {}, + app: { + name: sdkName, + namespace: 'com.rudderlabs.javascript', + version: 'dev-snapshot', + }, + library: { + name: sdkName, + version: 'dev-snapshot', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + os: { + name: '', + version: '', + }, + locale: 'en-GB', + screen: { + width: 1728, + height: 1117, + density: 2, + innerWidth: 1659, + innerHeight: 379, + }, + campaign: {}, + page: { + path: '/index.html', + referrer: '$direct', + referring_domain: '', + search: '', + title: '', + url: 'http://localhost:3001/index.html', + tab_url: 'http://localhost:3001/index.html', + initial_referrer: '$direct', + initial_referring_domain: '', + }, + }, + originalTimestamp: '2023-09-29T04:01:42.622Z', + integrations: { + All: true, + }, + messageId: 'ff73a416-c3a4-4759-9982-73cd3pob4576', + userId: 'customUserID', + anonymousId: '1901c08d-d505-41cd-81a6-060457fad648', + event: null, +}; + +const errorMessage = 'Invalid request payload'; + +const dummyDataplaneHost = 'https://dummy.dataplane.host.com'; + +const dmtSuccessResponse = { + transformedBatch: [ + { + id: '2CO2YmLozA3SZe6JtmdmMKTrCOl', + payload: [ + { + orderNo: 1659505271417, + status: '200', + event: { + message: { + anonymousId: '7105960b-0174-4d31-a7a1-561925dedde3', + channel: 'web', + context: { + library: { + name: sdkName, + version: 'dev-snapshot', + }, + }, + integrations: { + All: true, + }, + messageId: '1659505271412300-2d882451-7f50-4f23-b5ac-919fa8a1957d', + name: 'page view 123', + originalTimestamp: '2022-08-03T05:41:11.412Z', + properties: {}, + type: 'page', + }, + }, + }, + ], + }, + ], +}; + +const dmtPartialSuccessResponse = { + transformedBatch: [ + { + id: '2CO2YmLozA3SZe6JtmdmMKTrCOl', + payload: [ + { + orderNo: 1659505271417, + status: '200', + event: { + message: { + anonymousId: '7105960b-0174-4d31-a7a1-561925dedde3', + channel: 'web', + context: { + library: { + name: sdkName, + version: 'dev-snapshot', + }, + }, + integrations: { + All: true, + }, + messageId: '1659505271412300-2d882451-7f50-4f23-b5ac-919fa8a1957d', + name: 'page view 123', + originalTimestamp: '2022-08-03T05:41:11.412Z', + properties: {}, + type: 'page', + }, + }, + }, + ], + }, + { + id: '2CO2YmLozA3SZe6JtmdmMKTrCKr', + payload: [ + { + orderNo: 1659505271418, + status: '410', + }, + ], + }, + ], +}; + +const dummyWriteKey = 'dummy-write-key'; +const authToken = 'sample-auth-token'; + +export { + rudderEventPage, + dummyDataplaneHost, + errorMessage, + dmtSuccessResponse, + dmtPartialSuccessResponse, + dummyWriteKey, + authToken, +}; diff --git a/packages/analytics-js-plugins/__fixtures__/msw.handlers.js b/packages/analytics-js-plugins/__fixtures__/msw.handlers.js new file mode 100644 index 000000000..15a5da633 --- /dev/null +++ b/packages/analytics-js-plugins/__fixtures__/msw.handlers.js @@ -0,0 +1,34 @@ +import { rest } from 'msw'; +import { + dummyDataplaneHost, + dmtSuccessResponse, + dmtPartialSuccessResponse, + errorMessage, +} from './fixtures'; + +const handlers = [ + rest.post(`${dummyDataplaneHost}/success/transform`, (req, res, ctx) => { + return res(ctx.status(200), ctx.json(dmtSuccessResponse)); + }), + rest.post(`${dummyDataplaneHost}/partialSuccess/transform`, (req, res, ctx) => { + return res(ctx.status(200), ctx.json(dmtPartialSuccessResponse)); + }), + rest.post(`${dummyDataplaneHost}/invalidResponse/transform`, (req, res, ctx) => { + return res( + ctx.status(200), + ctx.text(dmtSuccessResponse), + ctx.set('Content-Type', 'application/json; charset=utf-8'), + ); + }), + rest.post(`${dummyDataplaneHost}/badRequest/transform`, (req, res, ctx) => { + return res(ctx.status(400), ctx.text(errorMessage)); + }), + rest.post(`${dummyDataplaneHost}/accessDenied/transform`, (req, res, ctx) => { + return res(ctx.status(404)); + }), + rest.post(`${dummyDataplaneHost}/serverDown/transform`, (req, res, ctx) => { + return res(ctx.status(500)); + }), +]; + +export { handlers }; diff --git a/packages/analytics-js-plugins/__fixtures__/msw.server.js b/packages/analytics-js-plugins/__fixtures__/msw.server.js new file mode 100644 index 000000000..52da765b9 --- /dev/null +++ b/packages/analytics-js-plugins/__fixtures__/msw.server.js @@ -0,0 +1,6 @@ +import { setupServer } from 'msw/node'; +import { handlers } from './msw.handlers'; + +const server = setupServer(...handlers); + +export { server }; diff --git a/packages/analytics-js-plugins/__tests__/deviceModeTransformation/index.test.ts b/packages/analytics-js-plugins/__tests__/deviceModeTransformation/index.test.ts new file mode 100644 index 000000000..91601a230 --- /dev/null +++ b/packages/analytics-js-plugins/__tests__/deviceModeTransformation/index.test.ts @@ -0,0 +1,287 @@ +/* eslint-disable no-plusplus */ +import { batch } from '@preact/signals-core'; +import { HttpClient } from '@rudderstack/analytics-js/services/HttpClient'; +import { state } from '@rudderstack/analytics-js/state'; +import { PluginsManager } from '@rudderstack/analytics-js/components/pluginsManager'; +import { defaultPluginEngine } from '@rudderstack/analytics-js/services/PluginEngine'; +import { defaultErrorHandler } from '@rudderstack/analytics-js/services/ErrorHandler'; +import { defaultLogger } from '@rudderstack/analytics-js/services/Logger'; +import { StoreManager } from '@rudderstack/analytics-js/services/StoreManager'; +import { RudderEvent } from '@rudderstack/analytics-js-common/types/Event'; +import { IHttpClient } from '@rudderstack/analytics-js-common/types/HttpClient'; +import { + dummyDataplaneHost, + dummyWriteKey, + authToken, + dmtSuccessResponse, +} from '../../__fixtures__/fixtures'; +import { server } from '../../__fixtures__/msw.server'; +import * as utils from '../../src/deviceModeTransformation/utilities'; +import { DeviceModeTransformation } from '../../src/deviceModeTransformation'; + +jest.mock('@rudderstack/analytics-js-common/utilities/uuId', () => ({ + ...jest.requireActual('@rudderstack/analytics-js-common/utilities/uuId'), + generateUUID: jest.fn(() => 'sample_uuid'), +})); + +describe('Device mode transformation plugin', () => { + const defaultPluginsManager = new PluginsManager( + defaultPluginEngine, + defaultErrorHandler, + defaultLogger, + ); + + const defaultStoreManager = new StoreManager(defaultPluginsManager); + + beforeAll(() => { + server.listen(); + batch(() => { + state.lifecycle.writeKey.value = dummyWriteKey; + state.lifecycle.activeDataplaneUrl.value = dummyDataplaneHost; + state.session.authToken.value = authToken; + }); + }); + + const httpClient = new HttpClient(); + + afterAll(() => { + server.close(); + }); + + const destinations = [ + { + id: 'id1', + displayName: 'Destination 1', + userFriendlyId: 'Destination_568fhgvb7689', + shouldApplyDeviceModeTransformation: true, + propagateEventsUntransformedOnError: false, + config: {}, + }, + { + id: 'id2', + displayName: 'Destination 2', + userFriendlyId: 'Destination_0986fhgvb7689', + shouldApplyDeviceModeTransformation: true, + propagateEventsUntransformedOnError: true, + config: {}, + }, + { + id: 'id3', + displayName: 'Destination 3', + userFriendlyId: 'Destination_123fhgvb7689', + shouldApplyDeviceModeTransformation: true, + propagateEventsUntransformedOnError: false, + config: {}, + }, + ]; + const destinationIds = ['id1', 'id2', 'id3']; + + it('should add DeviceModeTransformation plugin in the loaded plugin list', () => { + DeviceModeTransformation().initialize(state); + expect(state.plugins.loadedPlugins.value.includes('DeviceModeTransformation')).toBe(true); + }); + + it('should return a queue object on init', () => { + const queue = DeviceModeTransformation().transformEvent?.init( + state, + defaultPluginsManager, + httpClient, + defaultStoreManager, + defaultErrorHandler, + defaultLogger, + ); + + expect(queue).toBeDefined(); + expect(queue.name).toBe('rudder_dummy-write-key'); + }); + + it('should add item in queue on enqueue', () => { + const queue = DeviceModeTransformation().transformEvent?.init( + state, + defaultPluginsManager, + httpClient, + defaultStoreManager, + defaultErrorHandler, + defaultLogger, + ); + + const addItemSpy = jest.spyOn(queue, 'addItem'); + + const event: RudderEvent = { + type: 'track', + event: 'test', + userId: 'test', + properties: { + test: 'test', + }, + anonymousId: 'sampleAnonId', + messageId: 'test', + originalTimestamp: 'test', + }; + + DeviceModeTransformation().transformEvent?.enqueue(state, queue, event, destinations); + + expect(addItemSpy).toBeCalledWith({ + token: authToken, + destinationIds, + event, + }); + + addItemSpy.mockRestore(); + }); + + it('should process queue item on start', () => { + const mockHttpClient = { + getAsyncData: ({ callback }) => { + callback(true); + }, + setAuthHeader: jest.fn(), + }; + const queue = DeviceModeTransformation().transformEvent?.init( + state, + defaultPluginsManager, + mockHttpClient, + defaultStoreManager, + ); + + const event: RudderEvent = { + type: 'track', + event: 'test', + userId: 'test', + properties: { + test: 'test', + }, + anonymousId: 'sampleAnonId', + messageId: 'test', + originalTimestamp: 'test', + }; + + const queueProcessCbSpy = jest.spyOn(queue, 'processQueueCb'); + + DeviceModeTransformation().transformEvent?.enqueue(state, queue, event, destinations); + + // Explicitly start the queue to process the item + // In actual implementation, this is done based on the state signals + queue.start(); + + expect(queueProcessCbSpy).toBeCalledWith( + { + token: authToken, + destinationIds, + event, + }, + expect.any(Function), + 0, + 3, + true, + ); + + // Item is successfully processed and removed from queue + expect(queue.getQueue('queue').length).toBe(0); + + queueProcessCbSpy.mockRestore(); + }); + + it('SendTransformedEventToDestinations function is called in case of successful transformation', () => { + const mockHttpClient = { + getAsyncData: ({ callback }) => { + callback(JSON.stringify(dmtSuccessResponse), { xhr: { status: 200 } }); + }, + setAuthHeader: jest.fn(), + } as unknown as IHttpClient; + const mockSendTransformedEventToDestinations = jest.spyOn( + utils, + 'sendTransformedEventToDestinations', + ); + + const queue = DeviceModeTransformation().transformEvent?.init( + state, + defaultPluginsManager, + mockHttpClient, + defaultStoreManager, + defaultErrorHandler, + defaultLogger, + ); + + const event: RudderEvent = { + type: 'track', + event: 'test', + userId: 'test', + properties: { + test: 'test', + }, + anonymousId: 'sampleAnonId', + messageId: 'test', + originalTimestamp: 'test', + }; + + queue.start(); + DeviceModeTransformation().transformEvent?.enqueue(state, queue, event, destinations); + + expect(mockSendTransformedEventToDestinations).toBeCalledTimes(1); + expect(mockSendTransformedEventToDestinations).toHaveBeenCalledWith( + state, + defaultPluginsManager, + destinationIds, + JSON.stringify(dmtSuccessResponse), + 200, + event, + defaultErrorHandler, + defaultLogger, + ); + mockSendTransformedEventToDestinations.mockRestore(); + }); + it('SendTransformedEventToDestinations function should not be called in case of unsuccessful transformation', () => { + const mockHttpClient = { + getAsyncData: ({ callback }) => { + callback(false, { error: 'some error', xhr: { status: 502 } }); + }, + setAuthHeader: jest.fn(), + } as unknown as IHttpClient; + const mockSendTransformedEventToDestinations = jest.spyOn( + utils, + 'sendTransformedEventToDestinations', + ); + + const queue = DeviceModeTransformation().transformEvent?.init( + state, + defaultPluginsManager, + mockHttpClient, + defaultStoreManager, + defaultErrorHandler, + defaultLogger, + ); + + const event: RudderEvent = { + type: 'track', + event: 'test', + userId: 'test', + properties: { + test: 'test', + }, + anonymousId: 'sampleAnonId', + messageId: 'test', + originalTimestamp: 'test', + }; + + queue.start(); + DeviceModeTransformation().transformEvent?.enqueue(state, queue, event, destinations); + + expect(mockSendTransformedEventToDestinations).not.toBeCalled(); + // The element is requeued + expect(queue.getQueue('queue')).toStrictEqual([ + { + item: { + token: authToken, + destinationIds, + event, + }, + attemptNumber: 1, + id: 'sample_uuid', + time: expect.any(Number), + // time: 1 + 500 * 2 ** 1, // this is the delay calculation in RetryQueue + }, + ]); + mockSendTransformedEventToDestinations.mockRestore(); + }); +}); diff --git a/packages/analytics-js-plugins/__tests__/deviceModeTransformation/utilities.test.ts b/packages/analytics-js-plugins/__tests__/deviceModeTransformation/utilities.test.ts new file mode 100644 index 000000000..96a133fe6 --- /dev/null +++ b/packages/analytics-js-plugins/__tests__/deviceModeTransformation/utilities.test.ts @@ -0,0 +1,22 @@ +import { rudderEventPage } from '../../__fixtures__/fixtures'; +import { createPayload } from '../../src/deviceModeTransformation/utilities'; + +describe('DMT Plugin Utilities', () => { + it('should create request payload in appropriate format', () => { + const destinationIds = ['destination id 1', 'destination id 2']; + const payload = createPayload(rudderEventPage, destinationIds, 'sample-auth-token'); + + expect(payload).toEqual({ + metadata: { + 'Custom-Authorization': 'sample-auth-token', + }, + batch: [ + { + orderNo: expect.any(Number), + destinationIds, + event: rudderEventPage, + }, + ], + }); + }); +}); diff --git a/packages/analytics-js-plugins/src/deviceModeTransformation/constants.ts b/packages/analytics-js-plugins/src/deviceModeTransformation/constants.ts new file mode 100644 index 000000000..22a68e9a6 --- /dev/null +++ b/packages/analytics-js-plugins/src/deviceModeTransformation/constants.ts @@ -0,0 +1,12 @@ +const DEFAULT_TRANSFORMATION_QUEUE_OPTIONS = { + minRetryDelay: 500, + backoffFactor: 2, + maxAttempts: 3, +}; + +const REQUEST_TIMEOUT_MS = 10 * 1000; // 10 seconds + +const QUEUE_NAME = 'rudder'; +const DMT_PLUGIN = 'DeviceModeTransformationPlugin'; + +export { DEFAULT_TRANSFORMATION_QUEUE_OPTIONS, REQUEST_TIMEOUT_MS, QUEUE_NAME, DMT_PLUGIN }; diff --git a/packages/analytics-js-plugins/src/deviceModeTransformation/index.ts b/packages/analytics-js-plugins/src/deviceModeTransformation/index.ts index b9e48090d..a61b24438 100644 --- a/packages/analytics-js-plugins/src/deviceModeTransformation/index.ts +++ b/packages/analytics-js-plugins/src/deviceModeTransformation/index.ts @@ -7,6 +7,16 @@ import { Destination } from '@rudderstack/analytics-js-common/types/Destination' import { IErrorHandler } from '@rudderstack/analytics-js-common/types/ErrorHandler'; import { ILogger } from '@rudderstack/analytics-js-common/types/Logger'; import { ExtensionPlugin } from '@rudderstack/analytics-js-common/types/PluginEngine'; +import { IHttpClient } from '@rudderstack/analytics-js-common/types/HttpClient'; +import { MEMORY_STORAGE } from '@rudderstack/analytics-js-common/constants/storages'; +import { IStoreManager } from '@rudderstack/analytics-js-common/types/Store'; +import { isErrRetryable } from '@rudderstack/analytics-js-common/utilities/http'; +import { createPayload, sendTransformedEventToDestinations } from './utilities'; +import { getDMTDeliveryPayload } from '../utilities/eventsDelivery'; +import { DEFAULT_TRANSFORMATION_QUEUE_OPTIONS, QUEUE_NAME, REQUEST_TIMEOUT_MS } from './constants'; +import { RetryQueue } from '../utilities/retryQueue/RetryQueue'; +import { DoneCallback, IQueue } from '../types/plugins'; +import { TransformationQueueItemData } from './types'; const pluginName: PluginName = 'DeviceModeTransformation'; @@ -17,25 +27,78 @@ const DeviceModeTransformation = (): ExtensionPlugin => ({ state.plugins.loadedPlugins.value = [...state.plugins.loadedPlugins.value, pluginName]; }, transformEvent: { - enqueue( + init( state: ApplicationState, pluginsManager: IPluginsManager, - event: RudderEvent, - destination: Destination, + httpClient: IHttpClient, + storeManager: IStoreManager, errorHandler?: IErrorHandler, logger?: ILogger, ) { - // TODO: Implement DMT logic here + const writeKey = state.lifecycle.writeKey.value as string; + httpClient.setAuthHeader(writeKey); - // TODO: for now this is a pass through - pluginsManager.invokeSingle( - 'destinationsEventsQueue.enqueueEventToDestination', - state, - event, - destination, - errorHandler, - logger, + const eventsQueue = new RetryQueue( + // adding write key to the queue name to avoid conflicts + `${QUEUE_NAME}_${writeKey}`, + DEFAULT_TRANSFORMATION_QUEUE_OPTIONS, + ( + item: TransformationQueueItemData, + done: DoneCallback, + attemptNumber?: number, + maxRetryAttempts?: number, + ) => { + const payload = createPayload(item.event, item.destinationIds, item.token); + + httpClient.getAsyncData({ + url: `${state.lifecycle.dataPlaneUrl.value}/transform`, + options: { + method: 'POST', + data: getDMTDeliveryPayload(payload) as string, + sendRawData: true, + }, + isRawResponse: true, + timeout: REQUEST_TIMEOUT_MS, + callback: (result, details) => { + // null means item will not be requeued + const queueErrResp = isErrRetryable(details) ? details : null; + + if (!queueErrResp || attemptNumber === maxRetryAttempts) { + sendTransformedEventToDestinations( + state, + pluginsManager, + item.destinationIds, + result, + details?.xhr?.status, + item.event, + errorHandler, + logger, + ); + } + + done(queueErrResp, result); + }, + }); + }, + storeManager, + MEMORY_STORAGE, ); + + return eventsQueue; + }, + + enqueue( + state: ApplicationState, + eventsQueue: IQueue, + event: RudderEvent, + destinations: Destination[], + ) { + const destinationIds = destinations.map(d => d.id); + eventsQueue.addItem({ + event, + destinationIds, + token: state.session.authToken.value, + } as TransformationQueueItemData); }, }, }); diff --git a/packages/analytics-js-plugins/src/deviceModeTransformation/logMessages.ts b/packages/analytics-js-plugins/src/deviceModeTransformation/logMessages.ts new file mode 100644 index 000000000..c508edf56 --- /dev/null +++ b/packages/analytics-js-plugins/src/deviceModeTransformation/logMessages.ts @@ -0,0 +1,28 @@ +import { LOG_CONTEXT_SEPARATOR } from '@rudderstack/analytics-js-common/constants/logMessages'; + +const DMT_TRANSFORMATION_UNSUCCESSFUL_ERROR = ( + context: string, + displayName: string, + reason: string, + action: string, +): string => + `${context}${LOG_CONTEXT_SEPARATOR}Event transformation unsuccessful for destination "${displayName}". Reason: ${reason}. ${action}.`; + +const DMT_REQUEST_FAILED_ERROR = ( + context: string, + displayName: string, + status: number | undefined, + action: string, +): string => + `${context}${LOG_CONTEXT_SEPARATOR}[Destination: ${displayName}].Transformation request failed with status: ${status}. Retries exhausted. ${action}.`; + +const DMT_EXCEPTION = (displayName: string): string => `[Destination:${displayName}].`; +const DMT_SERVER_ACCESS_DENIED_WARNING = (context: string): string => + `${context}${LOG_CONTEXT_SEPARATOR}Transformation server access is denied. The configuration data seems to be out of sync. Sending untransformed event to the destination.`; + +export { + DMT_TRANSFORMATION_UNSUCCESSFUL_ERROR, + DMT_REQUEST_FAILED_ERROR, + DMT_EXCEPTION, + DMT_SERVER_ACCESS_DENIED_WARNING, +}; diff --git a/packages/analytics-js-plugins/src/deviceModeTransformation/types.ts b/packages/analytics-js-plugins/src/deviceModeTransformation/types.ts new file mode 100644 index 000000000..a0a4382f0 --- /dev/null +++ b/packages/analytics-js-plugins/src/deviceModeTransformation/types.ts @@ -0,0 +1,38 @@ +import { RudderEvent } from '@rudderstack/analytics-js-common/types/Event'; +import { Nullable } from '@rudderstack/analytics-js-common/types/Nullable'; + +export type BatchPayloadData = { + orderNo: number; + destinationIds: string[]; + event: RudderEvent; +}; + +export type TransformationRequestPayload = { + metadata: { + 'Custom-Authorization': Nullable; + }; + batch: BatchPayloadData[]; +}; + +export type TransformationQueueItemData = { + event: RudderEvent; + destinationIds: string[]; + token: Nullable; +}; + +export type TransformedEvent = RudderEvent | null | Record | unknown; + +export type TransformedPayload = { + orderNo: number; + status: string; + event?: TransformedEvent; +}; + +export type TransformedBatch = { + id: string; + payload: TransformedPayload[] | []; +}; + +export type TransformationResponsePayload = { + transformedBatch: TransformedBatch[]; +}; diff --git a/packages/analytics-js-plugins/src/deviceModeTransformation/utilities.ts b/packages/analytics-js-plugins/src/deviceModeTransformation/utilities.ts new file mode 100644 index 000000000..d9b31d4a8 --- /dev/null +++ b/packages/analytics-js-plugins/src/deviceModeTransformation/utilities.ts @@ -0,0 +1,155 @@ +import { RudderEvent } from '@rudderstack/analytics-js-common/types/Event'; +import { Nullable } from '@rudderstack/analytics-js-common/types/Nullable'; +import { isNonEmptyObject } from '@rudderstack/analytics-js-common/utilities/object'; +import { Destination } from '@rudderstack/analytics-js-common/types/Destination'; +import { ApplicationState } from '@rudderstack/analytics-js-common/types/ApplicationState'; +import { IPluginsManager } from '@rudderstack/analytics-js-common/types/PluginsManager'; +import { IErrorHandler } from '@rudderstack/analytics-js-common/types/ErrorHandler'; +import { ILogger } from '@rudderstack/analytics-js-common/types/Logger'; +import { + TransformationRequestPayload, + TransformationResponsePayload, + TransformedBatch, + TransformedEvent, + TransformedPayload, +} from './types'; +import { DMT_PLUGIN } from './constants'; +import { + DMT_EXCEPTION, + DMT_REQUEST_FAILED_ERROR, + DMT_SERVER_ACCESS_DENIED_WARNING, + DMT_TRANSFORMATION_UNSUCCESSFUL_ERROR, +} from './logMessages'; + +/** + * A helper function that will take rudderEvent and generate + * a batch payload that will be sent to transformation server + * + */ +const createPayload = ( + event: RudderEvent, + destinationIds: string[], + token: Nullable, +): TransformationRequestPayload => { + const orderNo = Date.now(); + const payload = { + metadata: { + 'Custom-Authorization': token, + }, + batch: [ + { + orderNo, + destinationIds, + event, + }, + ], + }; + return payload; +}; + +const sendTransformedEventToDestinations = ( + state: ApplicationState, + pluginsManager: IPluginsManager, + destinationIds: string[], + result: any, + status: number | undefined, + event: RudderEvent, + errorHandler?: IErrorHandler, + logger?: ILogger, +) => { + const NATIVE_DEST_EXT_POINT = 'destinationsEventsQueue.enqueueEventToDestination'; + const ACTION_TO_SEND_UNTRANSFORMED_EVENT = 'Sending untransformed event'; + const ACTION_TO_DROP_EVENT = 'Dropping the event'; + const destinations: Destination[] = state.nativeDestinations.initializedDestinations.value.filter( + d => d && destinationIds.includes(d.id), + ); + + destinations.forEach(dest => { + try { + const eventsToSend: TransformedEvent[] = []; + switch (status) { + case 200: { + const response: TransformationResponsePayload = JSON.parse(result); + const destTransformedResult = response.transformedBatch.find( + (e: TransformedBatch) => e.id === dest.id, + ); + destTransformedResult?.payload.forEach((tEvent: TransformedPayload) => { + if (tEvent.status === '200') { + eventsToSend.push(tEvent.event); + } else { + let reason = 'Unknown'; + if (tEvent.status === '410') { + reason = 'Transformation is not available'; + } + + let action = ACTION_TO_DROP_EVENT; + if (dest.propagateEventsUntransformedOnError === true) { + action = ACTION_TO_SEND_UNTRANSFORMED_EVENT; + eventsToSend.push(event); + logger?.warn( + DMT_TRANSFORMATION_UNSUCCESSFUL_ERROR( + DMT_PLUGIN, + dest.displayName, + reason, + action, + ), + ); + } else { + logger?.error( + DMT_TRANSFORMATION_UNSUCCESSFUL_ERROR( + DMT_PLUGIN, + dest.displayName, + reason, + action, + ), + ); + } + } + }); + + break; + } + // Transformation server access denied + case 404: { + logger?.warn(DMT_SERVER_ACCESS_DENIED_WARNING(DMT_PLUGIN)); + eventsToSend.push(event); + break; + } + default: { + if (dest.propagateEventsUntransformedOnError === true) { + logger?.warn( + DMT_REQUEST_FAILED_ERROR( + DMT_PLUGIN, + dest.displayName, + status, + ACTION_TO_SEND_UNTRANSFORMED_EVENT, + ), + ); + eventsToSend.push(event); + } else { + logger?.error( + DMT_REQUEST_FAILED_ERROR(DMT_PLUGIN, dest.displayName, status, ACTION_TO_DROP_EVENT), + ); + } + break; + } + } + eventsToSend?.forEach((tEvent?: TransformedEvent) => { + if (isNonEmptyObject(tEvent)) { + pluginsManager.invokeSingle( + NATIVE_DEST_EXT_POINT, + state, + tEvent, + dest, + errorHandler, + logger, + ); + } + }); + } catch (e) { + errorHandler?.onError(e, DMT_PLUGIN, DMT_EXCEPTION(dest.displayName)); + } + }); +}; + +export { createPayload, sendTransformedEventToDestinations }; diff --git a/packages/analytics-js-plugins/src/nativeDestinationQueue/index.ts b/packages/analytics-js-plugins/src/nativeDestinationQueue/index.ts index 62011175b..587174182 100644 --- a/packages/analytics-js-plugins/src/nativeDestinationQueue/index.ts +++ b/packages/analytics-js-plugins/src/nativeDestinationQueue/index.ts @@ -40,6 +40,7 @@ const NativeDestinationQueue = (): ExtensionPlugin => ({ state: ApplicationState, pluginsManager: IPluginsManager, storeManager: IStoreManager, + dmtQueue: IQueue, errorHandler?: IErrorHandler, logger?: ILogger, ): IQueue { @@ -58,8 +59,11 @@ const NativeDestinationQueue = (): ExtensionPlugin => ({ state.nativeDestinations.initializedDestinations.value, ); + // list of destinations which are enable for DMT + const destWithTransformationEnabled: Destination[] = []; + const clonedRudderEvent = clone(rudderEvent); + destinationsToSend.forEach((dest: Destination) => { - const clonedRudderEvent = clone(rudderEvent); const sendEvent = !isEventDenyListed( clonedRudderEvent.type, clonedRudderEvent.event, @@ -77,17 +81,22 @@ const NativeDestinationQueue = (): ExtensionPlugin => ({ } if (dest.shouldApplyDeviceModeTransformation) { - pluginsManager.invokeSingle( - 'transformEvent.enqueue', - state, - clonedRudderEvent, - dest, - logger, - ); + destWithTransformationEnabled.push(dest); } else { sendEventToDestination(clonedRudderEvent, dest, errorHandler, logger); } }); + if (destWithTransformationEnabled.length > 0) { + pluginsManager.invokeSingle( + 'transformEvent.enqueue', + state, + dmtQueue, + clonedRudderEvent, + destWithTransformationEnabled, + errorHandler, + logger, + ); + } // Mark success always done(null); diff --git a/packages/analytics-js-plugins/src/utilities/eventsDelivery.ts b/packages/analytics-js-plugins/src/utilities/eventsDelivery.ts index 5d8c3d73e..3f6344c28 100644 --- a/packages/analytics-js-plugins/src/utilities/eventsDelivery.ts +++ b/packages/analytics-js-plugins/src/utilities/eventsDelivery.ts @@ -6,6 +6,7 @@ import { ILogger } from '@rudderstack/analytics-js-common/types/Logger'; import { getCurrentTimeFormatted } from '@rudderstack/analytics-js-common/utilities/timestamp'; import { stringifyWithoutCircular } from '@rudderstack/analytics-js-common/utilities/json'; import { EVENT_PAYLOAD_SIZE_BYTES_LIMIT } from './constants'; +import { TransformationRequestPayload } from '../deviceModeTransformation/types'; const EVENT_PAYLOAD_SIZE_CHECK_FAIL_WARNING = ( context: string, @@ -28,6 +29,17 @@ const QUEUE_UTILITIES = 'QueueUtilities'; const getDeliveryPayload = (event: RudderEvent, logger?: ILogger): Nullable => stringifyWithoutCircular(event, true, undefined, logger); +const getDMTDeliveryPayload = ( + dmtRequestPayload: TransformationRequestPayload, + logger?: ILogger, +): Nullable => + stringifyWithoutCircular( + dmtRequestPayload, + true, + undefined, + logger, + ); + /** * Utility to validate final payload size before sending to server * @param event RudderEvent object @@ -66,4 +78,9 @@ const getFinalEventForDeliveryMutator = (event: RudderEvent): RudderEvent => { return finalEvent; }; -export { validateEventPayloadSize, getFinalEventForDeliveryMutator, getDeliveryPayload }; +export { + getDeliveryPayload, + validateEventPayloadSize, + getFinalEventForDeliveryMutator, + getDMTDeliveryPayload, +}; diff --git a/packages/analytics-js/.size-limit.js b/packages/analytics-js/.size-limit.js index 5b0185194..1b46c15cb 100644 --- a/packages/analytics-js/.size-limit.js +++ b/packages/analytics-js/.size-limit.js @@ -25,7 +25,7 @@ module.exports = [ name: 'Core Legacy - CDN', path: 'dist/cdn/legacy/iife/rsa.min.js', gzip: true, - limit: '47.5 KiB', + limit: '48.5 KiB', }, { name: 'Core - CDN', diff --git a/packages/analytics-js/__mocks__/remotePlugins/DeviceModeTransformation.ts b/packages/analytics-js/__mocks__/remotePlugins/DeviceModeTransformation.ts index dd20ec14d..0ca63e2c2 100644 --- a/packages/analytics-js/__mocks__/remotePlugins/DeviceModeTransformation.ts +++ b/packages/analytics-js/__mocks__/remotePlugins/DeviceModeTransformation.ts @@ -1,6 +1,7 @@ const DeviceModeTransformation = () => ({ name: 'DeviceModeTransformation', transformEvent: { + init: jest.fn(() => {}), enqueue: jest.fn(() => {}), }, }); diff --git a/packages/analytics-js/__tests__/components/eventRepository/EventRepository.test.ts b/packages/analytics-js/__tests__/components/eventRepository/EventRepository.test.ts index d5632c39b..57f68fe47 100644 --- a/packages/analytics-js/__tests__/components/eventRepository/EventRepository.test.ts +++ b/packages/analytics-js/__tests__/components/eventRepository/EventRepository.test.ts @@ -28,11 +28,18 @@ describe('EventRepository', () => { start: jest.fn(), }; + const mockDMTEventsQueue = { + start: jest.fn(), + }; + const mockPluginsManager = { invokeSingle: (extPoint: string) => { if (extPoint === 'destinationsEventsQueue.init') { return mockDestinationsEventsQueue; } + if (extPoint === 'transformEvent.init') { + return mockDMTEventsQueue; + } return mockDataplaneEventsQueue; }, } as IPluginsManager; @@ -86,12 +93,23 @@ describe('EventRepository', () => { ); expect(spy).nthCalledWith( 2, + 'transformEvent.init', + state, + defaultPluginsManager, + expect.objectContaining({}), + defaultStoreManager, + undefined, + undefined, + ); + expect(spy).nthCalledWith( + 3, 'destinationsEventsQueue.init', state, defaultPluginsManager, defaultStoreManager, undefined, undefined, + undefined, ); spy.mockRestore(); }); diff --git a/packages/analytics-js/src/app/RudderAnalytics.ts b/packages/analytics-js/src/app/RudderAnalytics.ts index e80295303..2f57702cf 100644 --- a/packages/analytics-js/src/app/RudderAnalytics.ts +++ b/packages/analytics-js/src/app/RudderAnalytics.ts @@ -70,6 +70,7 @@ class RudderAnalytics implements IRudderAnalytics { this.startSession = this.startSession.bind(this); this.endSession = this.endSession.bind(this); this.getSessionId = this.getSessionId.bind(this); + this.setAuthToken = this.setAuthToken.bind(this); RudderAnalytics.globalSingleton = this; @@ -278,6 +279,10 @@ class RudderAnalytics implements IRudderAnalytics { getSessionId() { return this.getAnalyticsInstance().getSessionId(); } + + setAuthToken(token: string) { + return this.getAnalyticsInstance().setAuthToken(token); + } } export { RudderAnalytics }; diff --git a/packages/analytics-js/src/components/capabilitiesManager/detection/dom.ts b/packages/analytics-js/src/components/capabilitiesManager/detection/dom.ts index 2327623f5..f80d31bf0 100644 --- a/packages/analytics-js/src/components/capabilitiesManager/detection/dom.ts +++ b/packages/analytics-js/src/components/capabilitiesManager/detection/dom.ts @@ -21,7 +21,7 @@ const legacyJSEngineRequiredPolyfills: Record boolean> = { 'String.prototype.includes': () => !String.prototype.includes, 'Object.entries': () => !Object.entries, 'Object.values': () => !Object.values, - 'Object.assign': () => typeof Object.assign !== "function", + 'Object.assign': () => typeof Object.assign !== 'function', 'Element.prototype.dataset': () => !isDatasetAvailable(), 'String.prototype.replaceAll': () => !String.prototype.replaceAll, TextEncoder: () => isUndefined(TextEncoder), diff --git a/packages/analytics-js/src/components/core/Analytics.ts b/packages/analytics-js/src/components/core/Analytics.ts index b3e89e0c7..2e22b4e31 100644 --- a/packages/analytics-js/src/components/core/Analytics.ts +++ b/packages/analytics-js/src/components/core/Analytics.ts @@ -692,6 +692,10 @@ class Analytics implements IAnalytics { const sessionId = this.userSessionManager?.getSessionId(); return sessionId ?? null; } + + setAuthToken(token: string): void { + this.userSessionManager?.setAuthToken(token); + } // End consumer exposed methods } diff --git a/packages/analytics-js/src/components/core/IAnalytics.ts b/packages/analytics-js/src/components/core/IAnalytics.ts index 59c6ca871..379a4fb5f 100644 --- a/packages/analytics-js/src/components/core/IAnalytics.ts +++ b/packages/analytics-js/src/components/core/IAnalytics.ts @@ -196,4 +196,9 @@ export interface IAnalytics { * To fetch the current sessionId */ getSessionId(): Nullable; + + /** + * To set auth token + */ + setAuthToken(token: string): void; } diff --git a/packages/analytics-js/src/components/eventRepository/EventRepository.ts b/packages/analytics-js/src/components/eventRepository/EventRepository.ts index 67804bd7e..009ce8eee 100644 --- a/packages/analytics-js/src/components/eventRepository/EventRepository.ts +++ b/packages/analytics-js/src/components/eventRepository/EventRepository.ts @@ -17,6 +17,7 @@ import { IEventRepository } from './types'; import { DATA_PLANE_QUEUE_EXT_POINT_PREFIX, DESTINATIONS_QUEUE_EXT_POINT_PREFIX, + DMT_EXT_POINT_PREFIX, } from './constants'; import { getFinalEvent } from './utils'; @@ -31,6 +32,7 @@ class EventRepository implements IEventRepository { storeManager: IStoreManager; dataplaneEventsQueue: any; destinationsEventsQueue: any; + dmtEventsQueue: any; /** * @@ -66,11 +68,22 @@ class EventRepository implements IEventRepository { this.logger, ); + this.dmtEventsQueue = this.pluginsManager.invokeSingle( + `${DMT_EXT_POINT_PREFIX}.init`, + state, + this.pluginsManager, + this.httpClient, + this.storeManager, + this.errorHandler, + this.logger, + ); + this.destinationsEventsQueue = this.pluginsManager.invokeSingle( `${DESTINATIONS_QUEUE_EXT_POINT_PREFIX}.init`, state, this.pluginsManager, this.storeManager, + this.dmtEventsQueue, this.errorHandler, this.logger, ); @@ -79,6 +92,7 @@ class EventRepository implements IEventRepository { effect(() => { if (state.nativeDestinations.clientDestinationsReady.value === true) { this.destinationsEventsQueue?.start(); + this.dmtEventsQueue?.start(); } }); diff --git a/packages/analytics-js/src/components/eventRepository/constants.ts b/packages/analytics-js/src/components/eventRepository/constants.ts index 475ddd314..207b78fb1 100644 --- a/packages/analytics-js/src/components/eventRepository/constants.ts +++ b/packages/analytics-js/src/components/eventRepository/constants.ts @@ -1,4 +1,9 @@ const DATA_PLANE_QUEUE_EXT_POINT_PREFIX = 'dataplaneEventsQueue'; const DESTINATIONS_QUEUE_EXT_POINT_PREFIX = 'destinationsEventsQueue'; +const DMT_EXT_POINT_PREFIX = 'transformEvent'; -export { DATA_PLANE_QUEUE_EXT_POINT_PREFIX, DESTINATIONS_QUEUE_EXT_POINT_PREFIX }; +export { + DATA_PLANE_QUEUE_EXT_POINT_PREFIX, + DESTINATIONS_QUEUE_EXT_POINT_PREFIX, + DMT_EXT_POINT_PREFIX, +}; diff --git a/packages/analytics-js/src/components/pluginsManager/defaultPluginsList.ts b/packages/analytics-js/src/components/pluginsManager/defaultPluginsList.ts index e1ad02c40..14d3f2b70 100644 --- a/packages/analytics-js/src/components/pluginsManager/defaultPluginsList.ts +++ b/packages/analytics-js/src/components/pluginsManager/defaultPluginsList.ts @@ -7,6 +7,7 @@ const defaultOptionalPluginsList: PluginName[] = [ 'BeaconQueue', 'Bugsnag', 'DeviceModeDestinations', + 'DeviceModeTransformation', 'ErrorReporting', 'ExternalAnonymousId', 'GoogleLinker', diff --git a/packages/analytics-js/src/components/userSessionManager/UserSessionManager.ts b/packages/analytics-js/src/components/userSessionManager/UserSessionManager.ts index a65e3db3f..269f63429 100644 --- a/packages/analytics-js/src/components/userSessionManager/UserSessionManager.ts +++ b/packages/analytics-js/src/components/userSessionManager/UserSessionManager.ts @@ -92,6 +92,10 @@ class UserSessionManager implements IUserSessionManager { if (anonymousId) { this.setAnonymousId(anonymousId); } + const authToken = this.getAuthToken(); + if (authToken) { + this.setAuthToken(authToken); + } const initialReferrer = this.getInitialReferrer(); const initialReferringDomain = this.getInitialReferringDomain(); @@ -311,6 +315,12 @@ class UserSessionManager implements IUserSessionManager { effect(() => { this.syncValueToStorage('sessionInfo', state.session.sessionInfo.value); }); + /** + * Update session tracking info in storage automatically when it is updated in state + */ + effect(() => { + this.syncValueToStorage('authToken', state.session.authToken.value); + }); } /** @@ -438,6 +448,14 @@ class UserSessionManager implements IUserSessionManager { return this.getItem('sessionInfo'); } + /** + * Fetches auth token from storage + * @returns + */ + getAuthToken(): Nullable { + return this.getItem('authToken'); + } + /** * If session is active it returns the sessionId * @returns @@ -489,6 +507,7 @@ class UserSessionManager implements IUserSessionManager { state.session.userTraits.value = defaultUserSessionValues.userTraits; state.session.groupId.value = defaultUserSessionValues.groupId; state.session.groupTraits.value = defaultUserSessionValues.groupTraits; + state.session.authToken.value = defaultUserSessionValues.authToken; if (resetAnonymousId) { state.session.anonymousId.value = defaultUserSessionValues.anonymousId; @@ -609,6 +628,16 @@ class UserSessionManager implements IUserSessionManager { end() { state.session.sessionInfo.value = {}; } + + /** + * Set auth token + * @param userId + */ + setAuthToken(token: string) { + if (this.isPersistenceEnabledForStorageEntry('authToken')) { + state.session.authToken.value = token; + } + } } export { UserSessionManager }; diff --git a/packages/analytics-js/src/components/userSessionManager/types.ts b/packages/analytics-js/src/components/userSessionManager/types.ts index 93fe3e8a1..bd30de806 100644 --- a/packages/analytics-js/src/components/userSessionManager/types.ts +++ b/packages/analytics-js/src/components/userSessionManager/types.ts @@ -22,6 +22,7 @@ export interface IUserSessionManager { reset(resetAnonymousId?: boolean, noNewSessionStart?: boolean): void; start(sessionId?: number): void; end(): void; + setAuthToken(token: string): void; } export type UserSessionStorageKeysType = keyof typeof userSessionStorageKeys; diff --git a/packages/analytics-js/src/components/userSessionManager/userSessionStorageKeys.ts b/packages/analytics-js/src/components/userSessionManager/userSessionStorageKeys.ts index 97d875af5..a9dfc2272 100644 --- a/packages/analytics-js/src/components/userSessionManager/userSessionStorageKeys.ts +++ b/packages/analytics-js/src/components/userSessionManager/userSessionStorageKeys.ts @@ -7,6 +7,7 @@ const userSessionStorageKeys = { initialReferrer: 'rl_page_init_referrer', initialReferringDomain: 'rl_page_init_referring_domain', sessionInfo: 'rl_session', + authToken: 'rl_auth_token', }; const defaultUserSessionValues = { @@ -18,6 +19,7 @@ const defaultUserSessionValues = { initialReferrer: '', initialReferringDomain: '', sessionInfo: {}, + authToken: null, }; const inMemorySessionKeys = { diff --git a/packages/analytics-js/src/state/slices/session.ts b/packages/analytics-js/src/state/slices/session.ts index 469383a64..2c7f253fc 100644 --- a/packages/analytics-js/src/state/slices/session.ts +++ b/packages/analytics-js/src/state/slices/session.ts @@ -18,6 +18,7 @@ const sessionState: SessionState = { initialReferrer: signal(defaultUserSessionValues.initialReferrer), initialReferringDomain: signal(defaultUserSessionValues.initialReferringDomain), sessionInfo: signal(defaultUserSessionValues.sessionInfo), + authToken: signal(defaultUserSessionValues.authToken), }; export { sessionState, defaultSessionInfo };