From f4b38eba3ca8dff602915853fda5cd7ca284bba3 Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Thu, 7 Nov 2024 10:48:57 +0530 Subject: [PATCH] feat(GARL): support vdm next for GARL (#3835) * feat(GARL): support vdm next for GARL * feat(GARL): support vdm next for GARL * feat(GARL): support vdm next for GARL * chore: updated tests with new structure --- .../recordTransform.js | 71 +++++++++++++++++-- .../transform.js | 20 +++++- .../google_adwords_remarketing_lists/util.js | 47 +++++++----- .../util.test.js | 20 ++++-- .../processor/data.ts | 37 +++++----- .../router/audience.ts | 2 +- .../router/data.ts | 12 ++-- .../router/record.ts | 2 +- 8 files changed, 151 insertions(+), 60 deletions(-) diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index b05ddb07a2..f8a2b0e586 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -7,9 +7,11 @@ const { constructPayload, returnArrayOfSubarrays, getSuccessRespEvents, + isEventSentByVDMV1Flow, + isEventSentByVDMV2Flow, } = require('../../util'); const { populateConsentFromConfig } = require('../../util/googleUtils'); -const { populateIdentifiers, responseBuilder } = require('./util'); +const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); @@ -19,6 +21,9 @@ const processRecordEventArray = ( destination, accessToken, developerToken, + audienceId, + typeOfList, + isHashRequired, operationType, ) => { let outputPayloads = {}; @@ -31,7 +36,12 @@ const processRecordEventArray = ( metadata.push(record.metadata); }); - const userIdentifiersList = populateIdentifiers(fieldsArray, destination); + const userIdentifiersList = populateIdentifiers( + fieldsArray, + destination, + typeOfList, + isHashRequired, + ); const outputPayload = constructPayload(message, offlineDataJobsMapping); outputPayload.operations = []; @@ -68,7 +78,7 @@ const processRecordEventArray = ( Object.values(outputPayloads).forEach((data) => { const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); toSendEvents.push( - responseBuilder(accessToken, developerToken, data, destination, message, consentObj), + responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), ); }); @@ -77,12 +87,13 @@ const processRecordEventArray = ( return successResponse; }; -async function processRecordInputs(groupedRecordInputs) { - const { destination, message, metadata } = groupedRecordInputs[0]; +function preparepayload(events, config) { + const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); + const { audienceId, typeOfList, isHashRequired } = config; - const groupedRecordsByAction = lodash.groupBy(groupedRecordInputs, (record) => + const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), ); @@ -97,6 +108,9 @@ async function processRecordInputs(groupedRecordInputs) { destination, accessToken, developerToken, + audienceId, + typeOfList, + isHashRequired, 'remove', ); } @@ -108,6 +122,9 @@ async function processRecordInputs(groupedRecordInputs) { destination, accessToken, developerToken, + audienceId, + typeOfList, + isHashRequired, 'add', ); } @@ -119,6 +136,9 @@ async function processRecordInputs(groupedRecordInputs) { destination, accessToken, developerToken, + audienceId, + typeOfList, + isHashRequired, 'add', ); } @@ -139,6 +159,45 @@ async function processRecordInputs(groupedRecordInputs) { return finalResponse; } +function processRecordInputsV0(groupedRecordInputs) { + const { destination, message } = groupedRecordInputs[0]; + const { audienceId, typeOfList, isHashRequired } = destination.Config; + + return preparepayload(groupedRecordInputs, { + audienceId: getOperationAudienceId(audienceId, message), + typeOfList, + isHashRequired, + }); +} + +function processRecordInputsV1(groupedRecordInputs) { + const { connection } = groupedRecordInputs[0]; + const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + + const events = groupedRecordInputs.map((record) => ({ + ...record, + message: { + ...record.message, + fields: record.message.identifiers, + }, + })); + + return preparepayload(events, { + audienceId, + typeOfList, + isHashRequired, + }); +} + +function processRecordInputs(groupedRecordInputs) { + const event = groupedRecordInputs[0]; + // First check for rETL flow and second check for ES flow + if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) { + return processRecordInputsV0(groupedRecordInputs); + } + return processRecordInputsV1(groupedRecordInputs); +} + module.exports = { processRecordInputs, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/transform.js b/src/v0/destinations/google_adwords_remarketing_lists/transform.js index 3deb9be775..299ab94846 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/transform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/transform.js @@ -12,7 +12,7 @@ const { const { populateConsentFromConfig } = require('../../util/googleUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); const { processRecordInputs } = require('./recordTransform'); -const { populateIdentifiers, responseBuilder } = require('./util'); +const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); function extraKeysPresent(dictionary, keyList) { // eslint-disable-next-line no-restricted-syntax @@ -37,12 +37,18 @@ function extraKeysPresent(dictionary, keyList) { const createPayload = (message, destination) => { const { listData } = message.properties; const properties = ['add', 'remove']; + const { typeOfList, isHashRequired } = destination.Config; let outputPayloads = {}; const typeOfOperation = Object.keys(listData); typeOfOperation.forEach((key) => { if (properties.includes(key)) { - const userIdentifiersList = populateIdentifiers(listData[key], destination); + const userIdentifiersList = populateIdentifiers( + listData[key], + destination, + typeOfList, + isHashRequired, + ); if (userIdentifiersList.length === 0) { logger.info( `Google_adwords_remarketing_list]:: No attributes are present in the '${key}' property.`, @@ -113,8 +119,16 @@ const processEvent = async (metadata, message, destination) => { Object.values(createdPayload).forEach((data) => { const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); + const { audienceId } = destination.Config; response.push( - responseBuilder(accessToken, developerToken, data, destination, message, consentObj), + responseBuilder( + accessToken, + developerToken, + data, + destination, + getOperationAudienceId(audienceId, message), + consentObj, + ), ); }); return response; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.js b/src/v0/destinations/google_adwords_remarketing_lists/util.js index 3e04dd8f6f..f4c33a9a6f 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.js @@ -29,26 +29,24 @@ const hashEncrypt = (object) => { }); }; -const responseBuilder = (accessToken, developerToken, body, { Config }, message, consentBlock) => { +const responseBuilder = ( + accessToken, + developerToken, + body, + { Config }, + audienceId, + consentBlock, +) => { const payload = body; const response = defaultRequestConfig(); const filteredCustomerId = removeHyphens(Config.customerId); response.endpoint = `${BASE_ENDPOINT}/${filteredCustomerId}/offlineUserDataJobs`; response.body.JSON = removeUndefinedAndNullValues(payload); - let operationAudienceId = Config.audienceId || Config.listId; - const mappedToDestination = get(message, MappedToDestinationKey); - if (!operationAudienceId && mappedToDestination) { - const { objectType } = getDestinationExternalIDInfoForRetl( - message, - 'GOOGLE_ADWORDS_REMARKETING_LISTS', - ); - operationAudienceId = objectType; - } - if (!isDefinedAndNotNullAndNotEmpty(operationAudienceId)) { + if (!isDefinedAndNotNullAndNotEmpty(audienceId)) { throw new ConfigurationError('List ID is a mandatory field'); } response.params = { - listId: operationAudienceId, + listId: audienceId, customerId: filteredCustomerId, consent: consentBlock, }; @@ -69,14 +67,15 @@ const responseBuilder = (accessToken, developerToken, body, { Config }, message, * This function helps creates an array with proper mapping for userIdentiFier. * Logics: Here we are creating an array with all the attributes provided in the add/remove array * inside listData. - * @param {rudder event message properties listData add} attributeArray - * @param {rudder event destination} Config + * @param {Array} attributeArray rudder event message properties listData add + * @param {object} Config rudder event destination + * @param {string} typeOfList + * @param {boolean} isHashRequired * @returns */ -const populateIdentifiers = (attributeArray, { Config }) => { +const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequired) => { const userIdentifier = []; - const { typeOfList } = Config; - const { isHashRequired, userSchema } = Config; + const { userSchema } = Config; let attribute; if (TYPEOFLIST[typeOfList]) { attribute = TYPEOFLIST[typeOfList]; @@ -116,7 +115,21 @@ const populateIdentifiers = (attributeArray, { Config }) => { return userIdentifier; }; +const getOperationAudienceId = (audienceId, message) => { + let operationAudienceId = audienceId; + const mappedToDestination = get(message, MappedToDestinationKey); + if (!operationAudienceId && mappedToDestination) { + const { objectType } = getDestinationExternalIDInfoForRetl( + message, + 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ); + operationAudienceId = objectType; + } + return operationAudienceId; +}; + module.exports = { populateIdentifiers, responseBuilder, + getOperationAudienceId, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js index a5897776c0..0b74b07b8e 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js @@ -1,4 +1,4 @@ -const { populateIdentifiers, responseBuilder } = require('./util'); +const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); const { API_VERSION } = require('./config'); const accessToken = 'abcd1234'; const developerToken = 'ijkl9101'; @@ -29,7 +29,7 @@ const body = { const baseDestination = { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -150,7 +150,7 @@ describe('GARL utils test', () => { developerToken, body, baseDestination, - message, + getOperationAudienceId(baseDestination.Config.audienceId, message), consentBlock, ); expect(response).toEqual(expectedResponse); @@ -166,7 +166,7 @@ describe('GARL utils test', () => { developerToken, body, destination2, - message, + getOperationAudienceId(baseDestination.Config.audienceId, message), consentBlock, ); expect(response).toEqual(); @@ -178,13 +178,13 @@ describe('GARL utils test', () => { it('Should throw error if operationAudienceId is not defined', () => { try { const destination1 = Object.create(baseDestination); - destination1.Config.listId = ''; + destination1.Config.audienceId = ''; const response = responseBuilder( accessToken, developerToken, body, destination1, - message, + getOperationAudienceId(baseDestination.Config.audienceId, message), consentBlock, ); expect(response).toEqual(); @@ -196,7 +196,13 @@ describe('GARL utils test', () => { describe('populateIdentifiers function tests', () => { it('Should hash and return identifiers for a given list of attributes', () => { - const identifier = populateIdentifiers(attributeArray, baseDestination); + const { typeOfList, isHashRequired } = baseDestination.Config; + const identifier = populateIdentifiers( + attributeArray, + baseDestination, + typeOfList, + isHashRequired, + ); expect(identifier).toEqual(hashedArray); }); }); diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/processor/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/processor/data.ts index e20ed89545..4398bc14e1 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/processor/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/processor/data.ts @@ -20,7 +20,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -151,7 +151,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '769-372-9833', loginCustomerId: '870-483-0944', subAccount: true, @@ -269,7 +269,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -387,7 +387,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -495,7 +495,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -585,7 +585,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -673,7 +673,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -749,7 +749,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: true, @@ -845,7 +845,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -2740,7 +2740,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -2993,7 +2993,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -6724,7 +6724,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -6880,7 +6880,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -6977,7 +6977,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -10708,7 +10708,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -10961,7 +10961,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -11207,7 +11207,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -11328,7 +11328,7 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', + audienceId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -11412,7 +11412,6 @@ export const data = [ destination: { Config: { rudderAccountId: 'rudder-acc-id', - listId: 'list111', customerId: '7693729833', loginCustomerId: '', subAccount: false, diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/audience.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/audience.ts index e0b534bb15..2158a57a5d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/audience.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/audience.ts @@ -4,7 +4,7 @@ import { generateGoogleOAuthMetadata } from '../../../testUtils'; const destination: Destination = { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index f5789bf7ef..a5e28996b1 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -92,7 +92,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -187,7 +187,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -332,7 +332,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -451,7 +451,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -578,7 +578,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, @@ -675,7 +675,7 @@ export const data = [ destination: { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false, diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index bedf112866..de76aae17c 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -4,7 +4,7 @@ import { generateGoogleOAuthMetadata } from '../../../testUtils'; const destination: Destination = { Config: { rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', - listId: '7090784486', + audienceId: '7090784486', customerId: '7693729833', loginCustomerId: '', subAccount: false,