diff --git a/.github/workflows/dt-test-and-report-code-coverage.yml b/.github/workflows/dt-test-and-report-code-coverage.yml index 33b9d881d9..2c76898882 100644 --- a/.github/workflows/dt-test-and-report-code-coverage.yml +++ b/.github/workflows/dt-test-and-report-code-coverage.yml @@ -80,7 +80,7 @@ jobs: - name: SonarCloud Scan if: always() - uses: SonarSource/sonarcloud-github-action@v3.0.0 + uses: SonarSource/sonarcloud-github-action@v3.1.0 env: GITHUB_TOKEN: ${{ secrets.PAT }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfd6f2835..8c7591edec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.83.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.1...v1.83.2) (2024-11-05) + + +### Bug Fixes + +* update gaec destination with config validation ([#3847](https://github.com/rudderlabs/rudder-transformer/issues/3847)) ([e5c5b0a](https://github.com/rudderlabs/rudder-transformer/commit/e5c5b0a28070ff5ca89a274c3998b96780139149)) + +### [1.83.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.0...v1.83.1) (2024-11-01) + ## [1.83.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.82.2...v1.83.0) (2024-10-25) diff --git a/package-lock.json b/package-lock.json index ccf7ed2c2a..0965688626 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.83.0", + "version": "1.83.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.83.0", + "version": "1.83.2", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -19,7 +19,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.10", + "@rudderstack/integrations-lib": "^0.2.12", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -50,7 +50,7 @@ "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", - "libphonenumber-js": "^1.11.1", + "libphonenumber-js": "^1.11.12", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -107,7 +107,7 @@ "eslint-plugin-unicorn": "^46.0.1", "glob": "^10.3.3", "http-terminator": "^3.2.0", - "husky": "^9.1.1", + "husky": "^9.1.6", "jest": "^29.5.0", "jest-sonar": "^0.2.16", "jest-when": "^3.5.2", @@ -6602,8 +6602,9 @@ } }, "node_modules/@rudderstack/integrations-lib": { - "version": "0.2.10", - "license": "MIT", + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.12.tgz", + "integrity": "sha512-xy+T9SHFkSeVDd4svGOyrTtIGljZ/l4qUh5o5EQWk3dTStzaV9mKnbXLsG62kEO3aTmCVg+VYr4OPwZY2+6rxQ==", "dependencies": { "axios": "^1.4.0", "axios-mock-adapter": "^1.22.0", @@ -13683,9 +13684,10 @@ } }, "node_modules/husky": { - "version": "9.1.1", + "version": "9.1.6", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.6.tgz", + "integrity": "sha512-sqbjZKK7kf44hfdE94EoX8MZNk0n7HeW37O4YrVGCF4wzgQjp+akPAkfUK5LZ6KuR/6sqeAVuXHji+RzQgOn5A==", "dev": true, - "license": "MIT", "bin": { "husky": "bin.js" }, @@ -16545,8 +16547,9 @@ } }, "node_modules/libphonenumber-js": { - "version": "1.11.1", - "license": "MIT" + "version": "1.11.12", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.12.tgz", + "integrity": "sha512-QkJn9/D7zZ1ucvT++TQSvZuSA2xAWeUytU+DiEQwbPKLyrDpvbul2AFs1CGbRAPpSCCk47aRAb5DX5mmcayp4g==" }, "node_modules/lilconfig": { "version": "2.1.0", diff --git a/package.json b/package.json index 79571462d3..65b7313e88 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.83.0", + "version": "1.83.2", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -64,7 +64,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.10", + "@rudderstack/integrations-lib": "^0.2.12", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -95,7 +95,7 @@ "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", - "libphonenumber-js": "^1.11.1", + "libphonenumber-js": "^1.11.12", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -152,7 +152,7 @@ "eslint-plugin-unicorn": "^46.0.1", "glob": "^10.3.3", "http-terminator": "^3.2.0", - "husky": "^9.1.1", + "husky": "^9.1.6", "jest": "^29.5.0", "jest-sonar": "^0.2.16", "jest-when": "^3.5.2", diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js index 69fea4299d..93a46fe68b 100644 --- a/src/cdk/v2/destinations/linkedin_ads/utils.js +++ b/src/cdk/v2/destinations/linkedin_ads/utils.js @@ -89,7 +89,7 @@ function checkIfPricePresent(properties) { } const calculateConversionObject = (message) => { - const { properties, event } = message; + const { properties } = message; const calculateAmount = () => { if (properties?.products && properties.products.length > 0) { @@ -107,9 +107,7 @@ const calculateConversionObject = (message) => { }; return conversionObject; } - throw new InstrumentationError( - `[LinkedIn Conversion API]: Cannot map price for event ${event}. Aborting`, - ); + return null; }; const deduceConversionRules = (trackEventName, destConfig) => { diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.test.js b/src/cdk/v2/destinations/linkedin_ads/utils.test.js index ee52928198..d66bda47dc 100644 --- a/src/cdk/v2/destinations/linkedin_ads/utils.test.js +++ b/src/cdk/v2/destinations/linkedin_ads/utils.test.js @@ -29,12 +29,13 @@ describe('formatEmail', () => { }); describe('calculateConversionObject', () => { - // Returns a conversion object with currency code 'USD' and amount 0 when message properties are empty - it('should throw instrumentation error when message properties are empty', () => { + // Returns empty object when message properties are empty + it('should return empty object when message properties are empty', () => { const message = { properties: {} }; expect(() => { - fetchUserIds(calculateConversionObject(message)); - }).toThrow(InstrumentationError); + const conversionObject = calculateConversionObject(message); + expect(conversionObject).toEqual({}); + }); }); // Returns a conversion object with currency code 'USD' and amount 0 when message properties price is defined but quantity is 0 diff --git a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml index 5d97da5a9e..64d391c888 100644 --- a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml +++ b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml @@ -167,11 +167,7 @@ steps: "content_ids": (props.product_id ?? props.sku ?? props.id)[], "contents": { "quantity": Number(props.quantity) || 1, - "item_price": String(props.price), - "item_name": String(props.name), - "id": props.product_id ?? props.sku, - "item_category": props.category, - "item_brand": props.brand + "item_price": String(props.price) }[] }; - name: combineAllEcomFields diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 13a294ea95..497d4f294f 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -56,13 +56,11 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { if (isNumber(customerId)) { customerId = customerId.toString(); } - if (isNumber(loginCustomerId)) { - loginCustomerId = loginCustomerId.toString(); - } - if (!isString(customerId) || !isString(loginCustomerId)) { - throw new InstrumentationError('customerId and loginCustomerId should be a string or number'); + if (!isString(customerId)) { + throw new InstrumentationError('customerId should be a string or number'); } const filteredCustomerId = removeHyphens(customerId); + response.endpoint = `${BASE_ENDPOINT}/${filteredCustomerId}:uploadConversionAdjustments`; response.body.JSON = payload; const accessToken = getAccessToken(metadata, 'access_token'); @@ -72,11 +70,19 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { 'developer-token': getValueFromMessage(metadata, 'secret.developer_token'), }; response.params = { event, customerId: filteredCustomerId }; - if (subAccount) - if (loginCustomerId) { - const filteredLoginCustomerId = removeHyphens(loginCustomerId); - response.headers['login-customer-id'] = filteredLoginCustomerId; - } else throw new ConfigurationError(`LoginCustomerId is required as subAccount is true.`); + if (subAccount) { + if (!loginCustomerId) { + throw new ConfigurationError(`loginCustomerId is required as subAccount is true.`); + } + if (isNumber(loginCustomerId)) { + loginCustomerId = loginCustomerId.toString(); + } + if (loginCustomerId && !isString(loginCustomerId)) { + throw new InstrumentationError('loginCustomerId should be a string or number'); + } + const filteredLoginCustomerId = removeHyphens(loginCustomerId); + response.headers['login-customer-id'] = filteredLoginCustomerId; + } if (loginCustomerId) { const filteredLoginCustomerId = removeHyphens(loginCustomerId); diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js index 0478a1b11b..1e943aee56 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/config.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/config.js @@ -7,6 +7,7 @@ const CONFIG_CATEGORIES = { AUDIENCE_LIST: { type: 'audienceList', name: 'offlineDataJobs' }, ADDRESSINFO: { type: 'addressInfo', name: 'addressInfo' }, }; +const ADDRESS_INFO_ATTRIBUTES = ['firstName', 'lastName', 'country', 'postalCode']; const attributeMapping = { email: 'hashedEmail', phone: 'hashedPhoneNumber', @@ -31,6 +32,7 @@ module.exports = { hashAttributes, offlineDataJobsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.AUDIENCE_LIST.name], addressInfoMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.ADDRESSINFO.name], + ADDRESS_INFO_ATTRIBUTES, consentConfigMap, destType: 'google_adwords_remarketing_lists', }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index f8a2b0e586..1c6284cd09 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -11,7 +11,11 @@ const { isEventSentByVDMV2Flow, } = require('../../util'); const { populateConsentFromConfig } = require('../../util/googleUtils'); -const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); +const { + populateIdentifiersForRecordEvent, + responseBuilder, + getOperationAudienceId, +} = require('./util'); const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); @@ -23,6 +27,7 @@ const processRecordEventArray = ( developerToken, audienceId, typeOfList, + userSchema, isHashRequired, operationType, ) => { @@ -36,10 +41,10 @@ const processRecordEventArray = ( metadata.push(record.metadata); }); - const userIdentifiersList = populateIdentifiers( + const userIdentifiersList = populateIdentifiersForRecordEvent( fieldsArray, - destination, typeOfList, + userSchema, isHashRequired, ); @@ -91,7 +96,7 @@ function preparepayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { audienceId, typeOfList, isHashRequired } = config; + const { audienceId, typeOfList, isHashRequired, userSchema } = config; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), @@ -110,6 +115,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'remove', ); @@ -124,6 +130,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'add', ); @@ -138,6 +145,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'add', ); @@ -161,19 +169,26 @@ function preparepayload(events, config) { function processRecordInputsV0(groupedRecordInputs) { const { destination, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = destination.Config; + const { audienceId, typeOfList, isHashRequired, userSchema } = destination.Config; return preparepayload(groupedRecordInputs, { audienceId: getOperationAudienceId(audienceId, message), typeOfList, + userSchema, isHashRequired, }); } function processRecordInputsV1(groupedRecordInputs) { - const { connection } = groupedRecordInputs[0]; + const { connection, message } = groupedRecordInputs[0]; const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + const identifiers = message?.identifiers; + let userSchema; + if (identifiers) { + userSchema = Object.keys(identifiers); + } + const events = groupedRecordInputs.map((record) => ({ ...record, message: { @@ -185,6 +200,7 @@ function processRecordInputsV1(groupedRecordInputs) { return preparepayload(events, { audienceId, typeOfList, + userSchema, isHashRequired, }); } diff --git a/src/v0/destinations/google_adwords_remarketing_lists/transform.js b/src/v0/destinations/google_adwords_remarketing_lists/transform.js index 299ab94846..4d173589e8 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/transform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/transform.js @@ -37,7 +37,7 @@ function extraKeysPresent(dictionary, keyList) { const createPayload = (message, destination) => { const { listData } = message.properties; const properties = ['add', 'remove']; - const { typeOfList, isHashRequired } = destination.Config; + const { typeOfList, userSchema, isHashRequired } = destination.Config; let outputPayloads = {}; const typeOfOperation = Object.keys(listData); @@ -45,8 +45,8 @@ const createPayload = (message, destination) => { if (properties.includes(key)) { const userIdentifiersList = populateIdentifiers( listData[key], - destination, typeOfList, + userSchema, isHashRequired, ); if (userIdentifiersList.length === 0) { diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.js b/src/v0/destinations/google_adwords_remarketing_lists/util.js index f4c33a9a6f..8e0aed0365 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.js @@ -18,6 +18,7 @@ const { TYPEOFLIST, BASE_ENDPOINT, hashAttributes, + ADDRESS_INFO_ATTRIBUTES, } = require('./config'); const hashEncrypt = (object) => { @@ -68,14 +69,13 @@ const responseBuilder = ( * Logics: Here we are creating an array with all the attributes provided in the add/remove array * inside listData. * @param {Array} attributeArray rudder event message properties listData add - * @param {object} Config rudder event destination * @param {string} typeOfList + * @param {Array} userSchema * @param {boolean} isHashRequired * @returns */ -const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequired) => { +const populateIdentifiers = (attributeArray, typeOfList, userSchema, isHashRequired) => { const userIdentifier = []; - const { userSchema } = Config; let attribute; if (TYPEOFLIST[typeOfList]) { attribute = TYPEOFLIST[typeOfList]; @@ -115,6 +115,40 @@ const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequi return userIdentifier; }; +const populateIdentifiersForRecordEvent = ( + identifiersArray, + typeOfList, + userSchema, + isHashRequired, +) => { + const userIdentifiers = []; + + if (isDefinedAndNotNullAndNotEmpty(identifiersArray)) { + // traversing through every element in the add array + identifiersArray.forEach((identifiers) => { + if (isHashRequired) { + hashEncrypt(identifiers); + } + if (TYPEOFLIST[typeOfList] && identifiers[TYPEOFLIST[typeOfList]]) { + userIdentifiers.push({ [TYPEOFLIST[typeOfList]]: identifiers[TYPEOFLIST[typeOfList]] }); + } else { + Object.entries(attributeMapping).forEach(([key, mappedKey]) => { + if (identifiers[key] && userSchema.includes(key)) + userIdentifiers.push({ [mappedKey]: identifiers[key] }); + }); + const addressInfo = constructPayload(identifiers, addressInfoMapping); + if ( + isDefinedAndNotNullAndNotEmpty(addressInfo) && + (userSchema.includes('addressInfo') || + userSchema.some((schema) => ADDRESS_INFO_ATTRIBUTES.includes(schema))) + ) + userIdentifiers.push({ addressInfo }); + } + }); + } + return userIdentifiers; +}; + const getOperationAudienceId = (audienceId, message) => { let operationAudienceId = audienceId; const mappedToDestination = get(message, MappedToDestinationKey); @@ -132,4 +166,5 @@ module.exports = { populateIdentifiers, responseBuilder, getOperationAudienceId, + populateIdentifiersForRecordEvent, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js index 0b74b07b8e..a41c00f12f 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js @@ -199,8 +199,8 @@ describe('GARL utils test', () => { const { typeOfList, isHashRequired } = baseDestination.Config; const identifier = populateIdentifiers( attributeArray, - baseDestination, typeOfList, + baseDestination.Config.userSchema, isHashRequired, ); expect(identifier).toEqual(hashedArray); diff --git a/src/v0/destinations/marketo_bulk_upload/transform.js b/src/v0/destinations/marketo_bulk_upload/transform.js index 1943d817cd..ada1d8c66a 100644 --- a/src/v0/destinations/marketo_bulk_upload/transform.js +++ b/src/v0/destinations/marketo_bulk_upload/transform.js @@ -35,7 +35,16 @@ function responseBuilderSimple(message, destination) { Object.keys(fieldHashmap).forEach((key) => { const val = traits[fieldHashmap[key]]; if (isDefined(val)) { - payload[key] = val; + let newVal = val; + // If value contains comma or newline then we need to escape it + if (typeof val === 'string') { + newVal = val + .toString() + .replaceAll(/\\/g, '\\\\') + .replaceAll(/,/g, '\\,') + .replaceAll(/\n/g, '\\n'); + } + payload[key] = newVal; } }); const response = defaultRequestConfig(); diff --git a/src/v0/destinations/salesforce/utils.js b/src/v0/destinations/salesforce/utils.js index a7731f07de..bbd5216c5b 100644 --- a/src/v0/destinations/salesforce/utils.js +++ b/src/v0/destinations/salesforce/utils.js @@ -23,6 +23,29 @@ const { const ACCESS_TOKEN_CACHE = new Cache(ACCESS_TOKEN_CACHE_TTL); +/** + * Extracts and returns the error message from a response object. + * If the response is an array and contains a message in the first element, + * it returns that message. Otherwise, it returns the stringified response. + * Error Message Format Example: ref: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/errorcodes.htm#:~:text=Incorrect%20ID%20example + [ + { + "fields" : [ "Id" ], + "message" : "Account ID: id value of incorrect type: 001900K0001pPuOAAU", + "errorCode" : "MALFORMED_ID" + } + ] + * @param {Object|Array} response - The response object or array to extract the message from. + * @returns {string} The extracted error message or the stringified response. + */ + +const getErrorMessage = (response) => { + if (Array.isArray(response) && response?.[0]?.message && response?.[0]?.message?.length > 0) { + return response[0].message; + } + return JSON.stringify(response); +}; + /** * ref: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/errorcodes.htm * handles Salesforce application level failures @@ -77,15 +100,19 @@ const salesforceResponseHandler = (destResponse, sourceMessage, authKey, authori } else if (status === 503 || status === 500) { // The salesforce server is unavailable to handle the request. Typically this occurs if the server is down // for maintenance or is currently overloaded. - throw new RetryableError( - `${DESTINATION} Request Failed - due to "${ - response && Array.isArray(response) && response[0]?.message?.length > 0 - ? response[0].message - : JSON.stringify(response) - }", (Retryable) ${sourceMessage}`, - 500, - destResponse, - ); + // ref : https://help.salesforce.com/s/articleView?id=000387190&type=1 + if (matchErrorCode('SERVER_UNAVAILABLE')) { + throw new ThrottledError( + `${DESTINATION} Request Failed: ${status} - due to ${getErrorMessage(response)}, ${sourceMessage}`, + destResponse, + ); + } else { + throw new RetryableError( + `${DESTINATION} Request Failed: ${status} - due to "${getErrorMessage(response)}", (Retryable) ${sourceMessage}`, + 500, + destResponse, + ); + } } // check the error message let errorMessage = ''; diff --git a/src/v0/destinations/twitter_ads/data/TwitterAdsTrackConfig.json b/src/v0/destinations/twitter_ads/data/TwitterAdsTrackConfig.json index 9394e25c0a..1eaf40db49 100644 --- a/src/v0/destinations/twitter_ads/data/TwitterAdsTrackConfig.json +++ b/src/v0/destinations/twitter_ads/data/TwitterAdsTrackConfig.json @@ -44,5 +44,13 @@ "destKey": "contents", "sourceKeys": "properties.contents", "required": false + }, + { + "destKey": "ip_address", + "sourceKeys": ["context.ip", "request_ip"] + }, + { + "destKey": "user_agent", + "sourceKeys": "context.userAgent" } ] diff --git a/src/v0/destinations/twitter_ads/transform.js b/src/v0/destinations/twitter_ads/transform.js index 268dca3636..71536be2d9 100644 --- a/src/v0/destinations/twitter_ads/transform.js +++ b/src/v0/destinations/twitter_ads/transform.js @@ -131,6 +131,20 @@ function processTrack(message, metadata, destination) { identifiers.push({ twclid: message.properties.twclid }); } + if (message.properties.ip_address) { + const ipAddress = message.properties.ip_address.trim(); + if (ipAddress) { + identifiers.push({ ip_address: ipAddress }); + } + } + + if (message.properties.user_agent) { + const userAgent = message.properties.user_agent.trim(); + if (userAgent) { + identifiers.push({ user_agent: userAgent }); + } + } + requestJson = populateContents(requestJson); requestJson.identifiers = identifiers; @@ -149,9 +163,15 @@ function validateRequest(message) { ); } - if (!properties.email && !properties.phone && !properties.twclid) { + if ( + !properties.email && + !properties.phone && + !properties.twclid && + !properties.ip_address && + !properties.user_agent + ) { throw new InstrumentationError( - '[TWITTER ADS]: one of twclid, phone or email must be present in properties.', + '[TWITTER ADS]: one of twclid, phone, email, ip_address or user_agent must be present in properties.', ); } } diff --git a/src/v0/util/index.js b/src/v0/util/index.js index f034ab802b..1676498fdb 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -10,7 +10,7 @@ const Handlebars = require('handlebars'); const fs = require('fs'); const path = require('path'); const lodash = require('lodash'); -const set = require('set-value'); +const { setValue: set } = require('@rudderstack/integrations-lib'); const get = require('get-value'); const uaParser = require('ua-parser-js'); const moment = require('moment-timezone'); diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index eaf8b79d54..0b05b6f2d6 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -977,6 +977,7 @@ describe('removeHyphens', () => { { input: null, expected: null }, { input: undefined, expected: undefined }, { input: 12345, expected: 12345 }, + { input: '123-12-241', expected: '12312241' }, ]; it('should remove hyphens from string else return the input as it is', () => { data.forEach(({ input, expected }) => { diff --git a/test/apitests/data_scenarios/cdk_v2/failure.json b/test/apitests/data_scenarios/cdk_v2/failure.json index 154d24481d..1635a3f0db 100644 --- a/test/apitests/data_scenarios/cdk_v2/failure.json +++ b/test/apitests/data_scenarios/cdk_v2/failure.json @@ -556,10 +556,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -683,10 +679,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/cdk_v2/success.json b/test/apitests/data_scenarios/cdk_v2/success.json index 88f430dd7c..ced7433a28 100644 --- a/test/apitests/data_scenarios/cdk_v2/success.json +++ b/test/apitests/data_scenarios/cdk_v2/success.json @@ -556,10 +556,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -638,10 +634,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -720,10 +712,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json index 3deb7d4b8b..3ce7c15091 100644 --- a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json +++ b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json @@ -388,10 +388,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -470,10 +466,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -552,10 +544,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json index a2652855d5..0e467c26d0 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json @@ -388,10 +388,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -470,10 +466,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_success.json b/test/apitests/data_scenarios/destination/proc/multiplex_success.json index ba4d5266f3..66b6c870a9 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_success.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_success.json @@ -207,10 +207,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -289,10 +285,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/router/failure_test.json b/test/apitests/data_scenarios/destination/router/failure_test.json index 197456f66a..9e36da50cb 100644 --- a/test/apitests/data_scenarios/destination/router/failure_test.json +++ b/test/apitests/data_scenarios/destination/router/failure_test.json @@ -754,10 +754,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -785,10 +781,6 @@ "content_ids": ["123"], "contents": [ { - "id": "123", - "item_brand": "Gamepro", - "item_category": "Games", - "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/integrations/destinations/ga4/processor/pageTestData.ts b/test/integrations/destinations/ga4/processor/pageTestData.ts index fa0b187aea..672f7e8f63 100644 --- a/test/integrations/destinations/ga4/processor/pageTestData.ts +++ b/test/integrations/destinations/ga4/processor/pageTestData.ts @@ -301,4 +301,216 @@ export const pageTestData: ProcessorTestData[] = [ }, mockFns: defaultMockFns, }, + { + id: 'ga4-page-test-4', + name: 'ga4', + description: + 'Scenario to test setting of reserved properties like constructor, __proto__, prototype in page call', + scenario: 'Business', + successCriteria: + 'Response status code should be 200 and event payload should not fail due to reserved properties', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: { + Config: { + apiSecret: 'api_secr', + debugMode: false, + typesOfClient: 'gtag', + measurementId: 'meas_id', + firebaseAppId: '', + whitelistedEvents: [ + { + eventName: '', + }, + ], + blacklistedEvents: [ + { + eventName: '', + }, + ], + eventFilteringOption: 'disable', + piiPropertiesToIgnore: [ + { + piiProperty: '', + }, + ], + sdkBaseUrl: 'https://www.googletagmanager.com', + serverContainerUrl: '', + debugView: true, + useNativeSDK: false, + connectionMode: 'cloud', + capturePageView: 'rs', + useNativeSDKToSend: false, + extendPageViewParams: false, + overrideClientAndSessionId: false, + eventDelivery: false, + }, + ID: '2ncdvkljndsvkuiurf', + WorkspaceID: 'wspId', + DestinationDefinition: { + ...destination.DestinationDefinition, + }, + Transformations: [], + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'my ga4', + Enabled: true, + }, + message: { + name: '', + type: 'page', + sentAt: '2022-04-29T05:17:09Z', + userId: '', + channel: 'web', + context: { + os: { + name: '', + version: '', + }, + app: { + name: 'RudderLabs JavaScript SDK', + version: '3.7.6', + namespace: 'com.rudderlabs.javascript', + installType: 'npm', + }, + page: { + url: 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + path: '/', + title: 'Mercedes-Benz Tire Center', + search: '?constructor.prototype.tenable_propexxx=tenable_something', + tab_url: + 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + referrer: '$direct', + initial_referrer: '$direct', + referring_domain: '', + initial_referring_domain: '', + }, + locale: 'en-US', + screen: { + width: 800, + height: 600, + density: 1, + innerWidth: 1600, + innerHeight: 1200, + }, + traits: {}, + library: { + name: 'RudderLabs JavaScript SDK', + version: '3.7.6', + }, + campaign: {}, + timezone: 'GMT+0000', + sessionId: 123465, + userAgent: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.207 Safari/537.36', + }, + rudderId: '7d02bb53-ff1a-46a2-9cb1-1ea78dcd4ca8', + timestamp: '2022-04-29T05:17:09Z', + properties: { + url: 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + path: '/', + title: 'Mercedes-Benz Tire Center', + search: '?constructor.prototype.tenable_propexxx=tenable_something', + tab_url: + 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + vehicle: { + make: '', + trim: '', + year: '', + model: '', + ratio: '', + width: '', + option: '', + diameter: '', + }, + national: true, + referrer: '$direct', + search_type: 'Vehicle', + initial_referrer: '$direct', + oem_program_code: 'CODE', + referring_domain: '', + initial_referring_domain: '', + 'constructor.prototype.tenable_propexxx': 'tenable_something', + }, + receivedAt: '2022-04-29T05:17:09Z', + request_ip: '34.201.223.160', + anonymousId: 'f577a7e1-6c76-49c3-8312-12846471e025', + integrations: { + All: true, + }, + originalTimestamp: '2022-04-29T05:17:09Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + XML: {}, + FORM: {}, + JSON: { + events: [ + { + name: 'page_view', + params: { + url: 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + path: '/', + title: 'Mercedes-Benz Tire Center', + search: '?constructor.prototype.tenable_propexxx=tenable_something', + tab_url: + 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + national: true, + referrer: '$direct', + page_title: 'Mercedes-Benz Tire Center', + session_id: 123465, + search_type: 'Vehicle', + page_location: + 'https://somewebsite.com/?constructor.prototype.tenable_propexxx=tenable_something', + page_referrer: '$direct', + initial_referrer: '$direct', + oem_program_code: 'CODE', + engagement_time_msec: 1, + 'constructor.prototype.tenable_propexxx': 'tenable_something', + }, + }, + ], + client_id: 'f577a7e1-6c76-49c3-8312-12846471e025', + timestamp_micros: 1651209429000000, + }, + JSON_ARRAY: {}, + }, + type: 'REST', + files: {}, + method: 'POST', + params: { + api_secret: 'api_secr', + measurement_id: 'meas_id', + }, + userId: '', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + version: '1', + endpoint: 'https://www.google-analytics.com/mp/collect', + }, + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + mockFns: defaultMockFns, + }, ]; diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts index 87fad8b9a5..fcdb6f15ca 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts @@ -814,7 +814,7 @@ export const data = [ }, }, statusCode: 400, - error: 'LoginCustomerId is required as subAccount is true.', + error: 'loginCustomerId is required as subAccount is true.', statTags: { errorCategory: 'dataValidation', errorType: 'configuration', diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts index 33cb4a832f..fe0acf7964 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts @@ -427,6 +427,8 @@ const events = [ Config: { rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', + customerId: '1234567890', + subAccount: true, listOfConversions: [{ conversions: 'Page View' }, { conversions: 'Product Added' }], authStatus: 'active', }, @@ -452,11 +454,17 @@ const events = [ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + customerID: {}, + subaccountID: 11, }, event: 'Page View', type: 'track', messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', originalTimestamp: '2019-10-14T11:15:18.299Z', + anonymousId: '00000000000000000000000000', userId: '12345', properties: { gclid: 'gclid1234', @@ -496,6 +504,78 @@ const events = [ }, integrations: { All: true }, name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + }, + { + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 7, + userId: 'u1', + }, + destination: { + Config: { + customerId: '1234567890', + subAccount: true, + loginCustomerId: { id: '1234567890' }, + listOfConversions: [{ conversions: 'Page View' }, { conversions: 'Product Added' }], + authStatus: 'active', + }, + }, + message: { + event: 'Page View', + type: 'track', + userId: '12345', + context: { + traits: { + email: 'user@testmail.com', + }, + }, + properties: { + gclid: 'gclid1234', + conversionDateTime: '2022-01-01 12:32:45-08:00', + order_id: 10000, + total: 1000, + }, + }, + }, + { + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 8, + userId: 'u1', + }, + destination: { + Config: { + customerId: '1234567890', + subAccount: true, + listOfConversions: [{ conversions: 'Page View' }, { conversions: 'Product Added' }], + authStatus: 'active', + }, + }, + message: { + event: 'Page View', + type: 'track', + userId: '12345', + context: { + traits: { + email: 'user@testmail.com', + }, + }, + properties: { + gclid: 'gclid1234', + conversionDateTime: '2022-01-01 12:32:45-08:00', + order_id: 10000, + total: 1000, + }, }, }, ]; @@ -628,30 +708,24 @@ export const data = [ body: { output: [ { + batched: false, batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - 'login-customer-id': '11', - }, - params: { event: 'Page View', customerId: '1234567890' }, body: { + FORM: {}, JSON: { - partialFailure: true, conversionAdjustments: [ { + adjustmentDateTime: '2022-01-01 12:32:45-08:00', + adjustmentType: 'ENHANCEMENT', gclidDateTimePair: { - gclid: 'gclid1234', conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid1234', }, - restatementValue: { adjustedValue: 10, currencyCode: 'INR' }, orderId: '10000', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', + restatementValue: { + adjustedValue: 10, + currencyCode: 'INR', + }, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', userIdentifiers: [ @@ -661,82 +735,102 @@ export const data = [ }, { addressInfo: { + city: 'London', hashedFirstName: 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', hashedLastName: '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - state: 'UK', - city: 'London', hashedStreetAddress: '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', + state: 'UK', }, }, ], - adjustmentType: 'ENHANCEMENT', }, ], + partialFailure: true, }, JSON_ARRAY: {}, XML: {}, - FORM: {}, }, + endpoint: + 'https://googleads.googleapis.com/v17/customers/1234567890:uploadConversionAdjustments', files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '11', + }, + method: 'POST', + params: { + customerId: '1234567890', + event: 'Page View', + }, + type: 'REST', + version: '1', + }, + destination: { + Config: { + authStatus: 'active', + customerId: '1234567890', + listOfConversions: [ + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, + ], + loginCustomerId: '11', + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, + }, }, metadata: [ { + jobId: 1, secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, - jobId: 1, userId: 'u1', }, ], - batched: false, statusCode: 200, + }, + { + batched: false, destination: { Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + authStatus: 'active', customerId: '1234567890', - subAccount: true, - loginCustomerId: '11', listOfConversions: [ - { conversions: 'Page View' }, - { conversions: 'Product Added' }, + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, ], - authStatus: 'active', + loginCustomerId: '', + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, }, }, - }, - { + error: 'Message Type identify is not supported. Aborting message.', metadata: [ { + jobId: 2, secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, - jobId: 2, userId: 'u1', }, ], - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - customerId: '1234567890', - subAccount: true, - loginCustomerId: '', - listOfConversions: [ - { conversions: 'Page View' }, - { conversions: 'Product Added' }, - ], - authStatus: 'active', - }, - }, - batched: false, - statusCode: 400, - error: 'Message Type identify is not supported. Aborting message.', statTags: { destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', errorCategory: 'dataValidation', @@ -745,25 +839,35 @@ export const data = [ implementation: 'native', module: 'destination', }, + statusCode: 400, }, { - metadata: [{ secret: {}, jobId: 3, userId: 'u1' }], + batched: false, destination: { Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + authStatus: 'active', customerId: '1234567890', - subAccount: true, - loginCustomerId: '11', listOfConversions: [ - { conversions: 'Page View' }, - { conversions: 'Product Added' }, + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, ], - authStatus: 'active', + loginCustomerId: '11', + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, }, }, - batched: false, - statusCode: 500, error: 'OAuth - access token not found', + metadata: [ + { + jobId: 3, + secret: {}, + userId: 'u1', + }, + ], statTags: { destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', errorCategory: 'platform', @@ -772,32 +876,27 @@ export const data = [ implementation: 'native', module: 'destination', }, + statusCode: 500, }, { + batched: false, batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - 'login-customer-id': '11', - }, - params: { event: 'Page View', customerId: '1234567890' }, body: { + FORM: {}, JSON: { - partialFailure: true, conversionAdjustments: [ { + adjustmentDateTime: '2022-01-01 12:32:45-08:00', + adjustmentType: 'ENHANCEMENT', gclidDateTimePair: { - gclid: 'gclid1234', conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid1234', }, - restatementValue: { adjustedValue: 10, currencyCode: 'INR' }, orderId: '10000', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', + restatementValue: { + adjustedValue: 10, + currencyCode: 'INR', + }, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', userIdentifiers: [ @@ -807,116 +906,131 @@ export const data = [ }, { addressInfo: { + city: 'London', hashedFirstName: 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', hashedLastName: '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - state: 'UK', - city: 'London', hashedStreetAddress: '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', + state: 'UK', }, }, ], - adjustmentType: 'ENHANCEMENT', }, ], + partialFailure: true, }, JSON_ARRAY: {}, XML: {}, - FORM: {}, }, + endpoint: + 'https://googleads.googleapis.com/v17/customers/1234567890:uploadConversionAdjustments', files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '11', + }, + method: 'POST', + params: { + customerId: '1234567890', + event: 'Page View', + }, + type: 'REST', + version: '1', + }, + destination: { + Config: { + authStatus: 'active', + customerId: 1234567890, + listOfConversions: [ + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, + ], + loginCustomerId: 11, + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, + }, }, metadata: [ { + jobId: 4, secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, - jobId: 4, userId: 'u1', }, ], - batched: false, statusCode: 200, + }, + { + batched: false, destination: { Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - customerId: 1234567890, - subAccount: true, - loginCustomerId: 11, + authStatus: 'active', + customerId: {}, listOfConversions: [ - { conversions: 'Page View' }, - { conversions: 'Product Added' }, + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, ], - authStatus: 'active', + loginCustomerId: 11, + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, }, }, - }, - { - batched: false, - statusCode: 400, - error: 'customerId and loginCustomerId should be a string or number', - statTags: { - destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', - errorCategory: 'dataValidation', - errorType: 'instrumentation', - feature: 'router', - implementation: 'native', - module: 'destination', - }, + error: 'customerId should be a string or number', metadata: [ { + jobId: 5, secret: { access_token: 'abcd1234', - refresh_token: 'efgh5678', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, - jobId: 5, userId: 'u1', }, ], - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - customerId: {}, - subAccount: true, - loginCustomerId: 11, - listOfConversions: [ - { conversions: 'Page View' }, - { conversions: 'Product Added' }, - ], - authStatus: 'active', - }, + statusCode: 400, + statTags: { + destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', }, }, { + batched: false, batchedRequest: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - 'login-customer-id': '65656565', - }, - params: { event: 'Page View', customerId: '1234567890' }, body: { + FORM: {}, JSON: { - partialFailure: true, conversionAdjustments: [ { + adjustmentDateTime: '2022-01-01 12:32:45-08:00', + adjustmentType: 'ENHANCEMENT', gclidDateTimePair: { - gclid: 'gclid1234', conversionDateTime: '2022-01-01 12:32:45-08:00', + gclid: 'gclid1234', }, - restatementValue: { adjustedValue: 10, currencyCode: 'INR' }, orderId: '10000', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', + restatementValue: { + adjustedValue: 10, + currencyCode: 'INR', + }, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', userIdentifiers: [ @@ -926,44 +1040,140 @@ export const data = [ }, { addressInfo: { + city: 'London', hashedFirstName: 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', hashedLastName: '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - state: 'UK', - city: 'London', hashedStreetAddress: '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', + state: 'UK', }, }, ], - adjustmentType: 'ENHANCEMENT', }, ], + partialFailure: true, }, JSON_ARRAY: {}, XML: {}, - FORM: {}, }, + endpoint: + 'https://googleads.googleapis.com/v17/customers/1234567890:uploadConversionAdjustments', files: {}, + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': '65656565', + }, + method: 'POST', + params: { + customerId: '1234567890', + event: 'Page View', + }, + type: 'REST', + version: '1', + }, + destination: { + Config: { + authStatus: 'active', + configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', + customerId: '1234567890', + listOfConversions: [ + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, + ], + rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + subAccount: true, + }, }, metadata: [ { + jobId: 6, secret: { access_token: 'abcd1234', + developer_token: 'ijkl91011', refresh_token: 'efgh5678', + }, + userId: 'u1', + }, + ], + statusCode: 200, + }, + { + batched: false, + destination: { + Config: { + authStatus: 'active', + customerId: '1234567890', + listOfConversions: [ + { + conversions: 'Page View', + }, + { + conversions: 'Product Added', + }, + ], + loginCustomerId: { + id: '1234567890', + }, + subAccount: true, + }, + }, + error: 'loginCustomerId should be a string or number', + metadata: [ + { + jobId: 7, + secret: { + access_token: 'abcd1234', developer_token: 'ijkl91011', + refresh_token: 'efgh5678', }, - jobId: 6, userId: 'u1', }, ], + statTags: { + destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + { batched: false, - statusCode: 200, + statusCode: 400, + error: 'loginCustomerId is required as subAccount is true.', + statTags: { + destType: 'GOOGLE_ADWORDS_ENHANCED_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'router', + implementation: 'native', + module: 'destination', + }, + metadata: [ + { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + jobId: 8, + userId: 'u1', + }, + ], destination: { Config: { - configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', + customerId: '1234567890', + subAccount: true, listOfConversions: [ { conversions: 'Page View' }, { conversions: 'Product Added' }, diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index a5e28996b1..6878e81f0d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -1,5 +1,9 @@ import { rETLAudienceRouterRequest } from './audience'; -import { rETLRecordRouterRequest } from './record'; +import { + rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, +} from './record'; import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_remarketing_lists/config'; export const data = [ @@ -732,4 +736,221 @@ export const data = [ }, }, }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 General typeOfList', + description: 'Test 2', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2General, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + hashedEmail: + 'd3142c8f9c9129484daf28df80cc5c955791efed5e69afabb603bc8cb9ffd419', + }, + { + hashedPhoneNumber: + '8846dcb6ab2d73a0e67dbd569fa17cec2d9d391e5b05d1dd42919bc21ae82c45', + }, + { + addressInfo: { + hashedFirstName: + '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + hashedLastName: + 'dcf000c2386fb76d22cefc0d118a8511bb75999019cd373df52044bccd1bd251', + countryCode: 'US', + postalCode: '1245', + }, + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 1, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 UserId typeOfList', + description: 'Test 3', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2UserId, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + thirdPartyUserId: 'useri1234', + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index de76aae17c..2661500b4d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -1,4 +1,5 @@ -import { Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { Connection, Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { VDM_V2_SCHEMA_VERSION } from '../../../../../src/v0/util/constant'; import { generateGoogleOAuthMetadata } from '../../../testUtils'; const destination: Destination = { @@ -27,6 +28,57 @@ const destination: Destination = { IsProcessorEnabled: true, }; +const destination2: Destination = { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Enabled: true, + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + DestinationDefinition: { + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Config: {}, + }, + Transformations: [], + IsConnectionEnabled: true, + IsProcessorEnabled: true, +}; + +const connection1: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'General', + audienceId: '7090784486', + }, + }, +}; + +const connection2: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'userID', + audienceId: '7090784486', + }, + }, +}; + export const rETLRecordRouterRequest: RouterTransformationRequest = { input: [ { @@ -153,6 +205,71 @@ export const rETLRecordRouterRequest: RouterTransformationRequest = { destType: 'google_adwords_remarketing_lists', }; +export const rETLRecordRouterRequestVDMv2General: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection1, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(1), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + +export const rETLRecordRouterRequestVDMv2UserId: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection2, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + thirdPartyUserId: 'useri1234', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(2), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + module.exports = { rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, }; diff --git a/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts index 4579cf68ee..5cb6ff8cf2 100644 --- a/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts +++ b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts @@ -1,4 +1,9 @@ -import { generateMetadata, generateTrackPayload, overrideDestination } from '../../../testUtils'; +import { + generateMetadata, + generateTrackPayload, + overrideDestination, + transformResultBuilder, +} from '../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -75,6 +80,14 @@ const commonStats = { workspaceId: 'default-workspaceId', }; +const commonHeader = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; + const commonTimestamp = new Date('2023-10-14'); const olderTimestamp = new Date('2023-07-13'); @@ -234,7 +247,7 @@ export const validationTestData: ProcessorTestData[] = [ description: 'Track call : properties without product array and no price', scenario: 'Business', successCriteria: - 'should throw error with status code 400 and error message regarding price is a mandatory field for linkedin conversions', + 'should not have conversionValue object as price is a mandatory field for linkedin conversions', feature: 'processor', module: 'destination', version: 'v0', @@ -243,7 +256,7 @@ export const validationTestData: ProcessorTestData[] = [ body: [ { message: generateTrackPayload({ - event: 'random event', + event: 'ABC Searched', properties: commonUserProperties, context: { traits: commonUserTraits, @@ -264,11 +277,51 @@ export const validationTestData: ProcessorTestData[] = [ status: 200, body: [ { - error: - '[LinkedIn Conversion API]: Cannot map price for event random event. Aborting: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: [LinkedIn Conversion API]: Cannot map price for event random event. Aborting', + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:1234567', + conversionHappenedAt: 1697241600000, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + { + conversion: 'urn:lla:llaPartnerConversion:34567', + conversionHappenedAt: 1697241600000, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, metadata: generateMetadata(1), - statTags: commonStats, - statusCode: 400, }, ], }, @@ -289,7 +342,7 @@ export const validationTestData: ProcessorTestData[] = [ body: [ { message: generateTrackPayload({ - event: 'random event', + event: 'ABC Searched', properties: commonUserPropertiesWithProductWithoutPrice, context: { traits: commonUserTraits, @@ -310,11 +363,51 @@ export const validationTestData: ProcessorTestData[] = [ status: 200, body: [ { - error: - '[LinkedIn Conversion API]: Cannot map price for event random event. Aborting: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: [LinkedIn Conversion API]: Cannot map price for event random event. Aborting', + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:1234567', + conversionHappenedAt: 1697241600000, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + { + conversion: 'urn:lla:llaPartnerConversion:34567', + conversionHappenedAt: 1697241600000, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, metadata: generateMetadata(1), - statTags: commonStats, - statusCode: 400, }, ], }, diff --git a/test/integrations/destinations/marketo_bulk_upload/processor/data.ts b/test/integrations/destinations/marketo_bulk_upload/processor/data.ts index 90a3ca8584..3a2b700534 100644 --- a/test/integrations/destinations/marketo_bulk_upload/processor/data.ts +++ b/test/integrations/destinations/marketo_bulk_upload/processor/data.ts @@ -595,4 +595,90 @@ export const data = [ }, }, }, + { + name: 'marketo_bulk_upload', + description: 'Test 6: Any comma or new line should be escaped through transform payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'identify', + userId: 'nicholas003', + anonymousId: 'anonId_003', + context: { + traits: { + firstName: 'Test', + lastName: 'hello\\world,new\nline', + email: 'badRecord.email.com', + city: '776 Elm St.\nRt. ,101', + }, + }, + request_ip: '192.168.10.106', + }, + destination: { + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + Config: { + munchkinId: 'XXXX', + clientId: 'YYYY', + clientSecret: 'ZZZZ', + columnFieldsMapping: [ + { + to: 'firstName', + from: 'firstName', + }, + { + to: 'lastName', + from: 'lastName', + }, + { + to: 'email', + from: 'email', + }, + { + to: 'city', + from: 'city', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: '/fileUpload', + headers: {}, + params: {}, + body: { + JSON: { + firstName: 'Test', + lastName: 'hello\\\\world\\,new\\nline', + email: 'badRecord.email.com', + city: '776 Elm St.\\nRt. \\,101', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/pinterest_tag/processor/data.ts b/test/integrations/destinations/pinterest_tag/processor/data.ts index 1788d13d56..b856d247d7 100644 --- a/test/integrations/destinations/pinterest_tag/processor/data.ts +++ b/test/integrations/destinations/pinterest_tag/processor/data.ts @@ -482,9 +482,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [ - { id: '123', item_name: 'undefined', quantity: 2, item_price: '25' }, - ], + contents: [{ quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2407,9 +2405,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [ - { id: '1234', item_name: 'undefined', quantity: 1, item_price: 'undefined' }, - ], + contents: [{ quantity: 1, item_price: 'undefined' }], }, }, JSON_ARRAY: {}, @@ -2670,7 +2666,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_price: 'undefined', quantity: 1, item_name: 'undefined' }], + contents: [{ item_price: 'undefined', quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3501,7 +3497,6 @@ export const data = [ subtotal: 22.5, affiliation: 'Google Store', checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', - category: 'Apparel', }, anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', integrations: { All: true }, @@ -3568,8 +3563,6 @@ export const data = [ { quantity: 1, item_price: 'undefined', - item_name: 'undefined', - item_category: 'Apparel', }, ], currency: 'USD', diff --git a/test/integrations/destinations/pinterest_tag/router/data.ts b/test/integrations/destinations/pinterest_tag/router/data.ts index 28c82c4679..c9ab29a45a 100644 --- a/test/integrations/destinations/pinterest_tag/router/data.ts +++ b/test/integrations/destinations/pinterest_tag/router/data.ts @@ -815,9 +815,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [ - { id: '123', item_name: 'undefined', quantity: 2, item_price: '25' }, - ], + contents: [{ quantity: 2, item_price: '25' }], }, }, { diff --git a/test/integrations/destinations/pinterest_tag/step/data.ts b/test/integrations/destinations/pinterest_tag/step/data.ts index 8f0680a77c..b607e3c9fa 100644 --- a/test/integrations/destinations/pinterest_tag/step/data.ts +++ b/test/integrations/destinations/pinterest_tag/step/data.ts @@ -468,9 +468,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [ - { id: '123', item_name: 'undefined', quantity: 2, item_price: '25' }, - ], + contents: [{ quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2422,9 +2420,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [ - { id: '1234', item_name: 'undefined', quantity: 1, item_price: 'undefined' }, - ], + contents: [{ quantity: 1, item_price: 'undefined' }], }, }, JSON_ARRAY: {}, @@ -2689,7 +2685,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_name: 'undefined', item_price: 'undefined', quantity: 1 }], + contents: [{ item_price: 'undefined', quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3609,7 +3605,6 @@ export const data = [ num_items: 0, contents: [ { - item_name: 'undefined', quantity: 1, item_price: 'undefined', }, diff --git a/test/integrations/destinations/salesforce/dataDelivery/business.ts b/test/integrations/destinations/salesforce/dataDelivery/business.ts index 4e98a3fc1a..5374e3fae2 100644 --- a/test/integrations/destinations/salesforce/dataDelivery/business.ts +++ b/test/integrations/destinations/salesforce/dataDelivery/business.ts @@ -150,7 +150,7 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ output: { status: 500, message: - 'Salesforce Request Failed - due to "Session expired or invalid", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 500 - due to "Session expired or invalid", (Retryable) during Salesforce Response Handling', response: [ { error: @@ -277,16 +277,16 @@ export const testScenariosForV1API: ProxyV1TestData[] = [ body: { output: { message: - 'Salesforce Request Failed - due to "Server Unavailable", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 503 - due to Server Unavailable, during Salesforce Response Handling', response: [ { error: '[{"message":"Server Unavailable","errorCode":"SERVER_UNAVAILABLE"}]', metadata: proxyMetdata, - statusCode: 500, + statusCode: 429, }, ], - statTags: statTags.retryable, - status: 500, + statTags: statTags.throttled, + status: 429, }, }, }, diff --git a/test/integrations/destinations/salesforce/dataDelivery/data.ts b/test/integrations/destinations/salesforce/dataDelivery/data.ts index d376289d97..f157161751 100644 --- a/test/integrations/destinations/salesforce/dataDelivery/data.ts +++ b/test/integrations/destinations/salesforce/dataDelivery/data.ts @@ -314,12 +314,12 @@ const legacyTests = [ }, output: { response: { - status: 500, + status: 429, body: { output: { - status: 500, + status: 429, message: - 'Salesforce Request Failed - due to "Server Unavailable", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 503 - due to Server Unavailable, during Salesforce Response Handling', destinationResponse: { response: [ { @@ -334,7 +334,7 @@ const legacyTests = [ errorCategory: 'network', destinationId: 'Non-determininable', workspaceId: 'Non-determininable', - errorType: 'retryable', + errorType: 'throttled', feature: 'dataDelivery', implementation: 'native', module: 'destination', @@ -461,7 +461,7 @@ const legacyTests = [ status: 503, }, message: - 'Salesforce Request Failed - due to "{"message":"Server Unavailable","errorCode":"SERVER_UNAVAILABLE"}", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 503 - due to "{"message":"Server Unavailable","errorCode":"SERVER_UNAVAILABLE"}", (Retryable) during Salesforce Response Handling', statTags: { destType: 'SALESFORCE', errorCategory: 'network', @@ -603,7 +603,7 @@ const legacyTests = [ output: { status: 500, message: - 'Salesforce Request Failed - due to ""[ECONNABORTED] :: Connection aborted"", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 500 - due to ""[ECONNABORTED] :: Connection aborted"", (Retryable) during Salesforce Response Handling', destinationResponse: { response: '[ECONNABORTED] :: Connection aborted', status: 500, @@ -696,7 +696,7 @@ const legacyTests = [ output: { status: 500, message: - 'Salesforce Request Failed - due to ""[EAI_AGAIN] :: Temporary failure in name resolution"", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 500 - due to ""[EAI_AGAIN] :: Temporary failure in name resolution"", (Retryable) during Salesforce Response Handling', destinationResponse: { response: '[EAI_AGAIN] :: Temporary failure in name resolution', status: 500, diff --git a/test/integrations/destinations/salesforce/dataDelivery/other.ts b/test/integrations/destinations/salesforce/dataDelivery/other.ts index b3361caba7..8bf154de9b 100644 --- a/test/integrations/destinations/salesforce/dataDelivery/other.ts +++ b/test/integrations/destinations/salesforce/dataDelivery/other.ts @@ -58,7 +58,7 @@ export const otherSalesforceScenariosV1: ProxyV1TestData[] = [ ], statTags, message: - 'Salesforce Request Failed - due to "{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 503 - due to "{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}", (Retryable) during Salesforce Response Handling', status: 500, }, }, @@ -96,7 +96,7 @@ export const otherSalesforceScenariosV1: ProxyV1TestData[] = [ ], statTags, message: - 'Salesforce Request Failed - due to ""Internal Server Error"", (Retryable) during Salesforce Response Handling', + 'Salesforce Request Failed: 500 - due to ""Internal Server Error"", (Retryable) during Salesforce Response Handling', status: 500, }, }, diff --git a/test/integrations/destinations/twitter_ads/processor/data.ts b/test/integrations/destinations/twitter_ads/processor/data.ts index b6a7512880..0ae21ffc98 100644 --- a/test/integrations/destinations/twitter_ads/processor/data.ts +++ b/test/integrations/destinations/twitter_ads/processor/data.ts @@ -131,6 +131,7 @@ export const data = [ conversion_time: '2023-06-01T06:03:08.739Z', number_items: 2, price_currency: 'USD', + user_agent: 'chrome', value: '25', conversion_id: '213123', contents: [ @@ -290,7 +291,8 @@ export const data = [ }, }, statusCode: 400, - error: '[TWITTER ADS]: one of twclid, phone or email must be present in properties.', + error: + '[TWITTER ADS]: one of twclid, phone, email, ip_address or user_agent must be present in properties.', statTags: { errorCategory: 'dataValidation', errorType: 'instrumentation', @@ -424,6 +426,7 @@ export const data = [ conversion_time: '2023-06-01T06:03:08.739Z', number_items: 2, price_currency: 'USD', + user_agent: 'chrome', value: '25.55', conversion_id: '213123', identifiers: [ @@ -842,6 +845,7 @@ export const data = [ conversion_time: '2023-06-01T06:03:08.739Z', number_items: 2, price_currency: 'USD', + user_agent: 'chrome', value: '25', conversion_id: '213123', contents: [ @@ -894,6 +898,233 @@ export const data = [ }, }, }, + { + name: 'twitter_ads', + description: 'Test case for track event with ip_address as an identifier', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Home Page Viewed', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + ge: 'male', + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2020-08-14T05:30:30.118Z', + properties: { + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + ip_address: '8.25.197.25', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + secret: { + consumerKey: 'qwe', + consumerSecret: 'fdghv', + accessToken: 'dummyAccessToken', + accessTokenSecret: 'testAccessTokenSecret', + }, + }, + destination: { + Config: { + pixelId: 'dummyPixelId', + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + twitterAdsEventNames: [ + { + rudderEventName: 'ABC Searched', + twitterEventId: 'tw-234234324234', + }, + { + rudderEventName: 'Home Page Viewed', + twitterEventId: 'tw-odt2o-odt2q', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://ads-api.twitter.com/12/measurement/conversions/dummyPixelId', + headers: { + Authorization: authHeaderConstant, + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + conversions: [ + { + conversion_time: '2020-08-14T05:30:30.118Z', + user_agent: 'chrome', + event_id: 'tw-odt2o-odt2q', + identifiers: [ + { + ip_address: '8.25.197.25', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + consumerKey: 'qwe', + consumerSecret: 'fdghv', + accessToken: 'dummyAccessToken', + accessTokenSecret: 'testAccessTokenSecret', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'twitter_ads', + description: 'Test case for track event with user_agent as an identifier', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + type: 'track', + event: 'Home Page Viewed', + channel: 'web', + context: { + source: 'test', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + ge: 'male', + }, + }, + properties: { + affiliation: 'Google Store', + user_agent: + ' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36.', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + secret: { + accessTokenSecret: 'testAccessTokenSecret', + }, + }, + destination: { + Config: { + pixelId: 'dummyPixelId', + rudderAccountId: '2EOknn1JNH7WK1MfNku4fGYKkRK', + twitterAdsEventNames: [ + { + rudderEventName: 'ABC Searched', + twitterEventId: 'tw-234234324234', + }, + { + rudderEventName: 'Home Page Viewed', + twitterEventId: 'tw-odt2o-odt2q', + }, + ], + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://ads-api.twitter.com/12/measurement/conversions/dummyPixelId', + headers: { + Authorization: authHeaderConstant, + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + conversions: [ + { + event_id: 'tw-odt2o-odt2q', + identifiers: [ + { + user_agent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36.', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + accessTokenSecret: 'testAccessTokenSecret', + }, + }, + statusCode: 200, + }, + ], + }, + }, + }, ].map((tc) => ({ ...tc, mockFns: (_) => { diff --git a/test/integrations/destinations/twitter_ads/router/data.ts b/test/integrations/destinations/twitter_ads/router/data.ts index ce9aea6595..7e8061dd7e 100644 --- a/test/integrations/destinations/twitter_ads/router/data.ts +++ b/test/integrations/destinations/twitter_ads/router/data.ts @@ -146,6 +146,7 @@ export const data = [ ], number_items: 2, price_currency: 'USD', + user_agent: 'chrome', value: '25', }, ],