diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b18fd29357..040e07f81d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: '/' schedule: interval: 'weekly' + - package-ecosystem: 'docker' + directory: '/' + schedule: + interval: 'daily' + - package-ecosystem: 'npm' + directory: '/' + schedule: + interval: 'daily' diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d3a37e122..b7a83098ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.61.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.60.0...v1.61.0) (2024-04-02) + + +### Features + +* consent field support for ga4 ([#3213](https://github.com/rudderlabs/rudder-transformer/issues/3213)) ([92515a5](https://github.com/rudderlabs/rudder-transformer/commit/92515a5fd8a2798c48010078f62b360ec6a49979)) +* consent field support for gaoc and upgrade the api version from v14 to v16 ([#3121](https://github.com/rudderlabs/rudder-transformer/issues/3121)) ([2aac2a6](https://github.com/rudderlabs/rudder-transformer/commit/2aac2a62547b7a7c617735fc3d6e88e0a1bed76e)), closes [#3190](https://github.com/rudderlabs/rudder-transformer/issues/3190) +* onboard new destination bloomreach ([#3185](https://github.com/rudderlabs/rudder-transformer/issues/3185)) ([d9b7e1f](https://github.com/rudderlabs/rudder-transformer/commit/d9b7e1f70565d59979aee3e62f60e39edb9a23c7)) +* onboarding linkedin conversion api ([#3194](https://github.com/rudderlabs/rudder-transformer/issues/3194)) ([eb7b197](https://github.com/rudderlabs/rudder-transformer/commit/eb7b197322c617b14c2579de8cb4d4dacf8e1df3)) +* update movable ink batch size ([#3223](https://github.com/rudderlabs/rudder-transformer/issues/3223)) ([667095f](https://github.com/rudderlabs/rudder-transformer/commit/667095fa8316cd95a066f15b848ad503c6b4af80)) + + +### Bug Fixes + +* fixed userId mapping, now mapping to uid instead of id ([#3192](https://github.com/rudderlabs/rudder-transformer/issues/3192)) ([70a468b](https://github.com/rudderlabs/rudder-transformer/commit/70a468bf16ecd5ee0b6fecee4b837895d19c525f)) +* ninetailed: remove page support ([#3218](https://github.com/rudderlabs/rudder-transformer/issues/3218)) ([2f30c56](https://github.com/rudderlabs/rudder-transformer/commit/2f30c56af62e983d09b5d4f2da9a0ba22f5c1612)) +* shopify invalid_event metric prometheus label ([#3200](https://github.com/rudderlabs/rudder-transformer/issues/3200)) ([345c87d](https://github.com/rudderlabs/rudder-transformer/commit/345c87d7c530c621ae3fd6c504d64e5a14e31f22)) +* fix: snapchat conversion: add event level_complete ([#3231](https://github.com/rudderlabs/rudder-transformer/issues/3231)) ([39368a0](https://github.com/rudderlabs/rudder-transformer/commit/39368a09e48acc324faa855186bc623e5c347881)) + ## [1.60.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.57.1...v1.60.0) (2024-03-20) diff --git a/package-lock.json b/package-lock.json index 5701e64a62..6d708b6f51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.60.0", + "version": "1.61.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.60.0", + "version": "1.61.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -19,8 +19,8 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.6", - "@rudderstack/integrations-lib": "^0.2.4", - "@rudderstack/workflow-engine": "^0.7.2", + "@rudderstack/integrations-lib": "^0.2.7", + "@rudderstack/workflow-engine": "^0.7.5", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", @@ -4446,33 +4446,35 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "node_modules/@pyroscope/nodejs": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/@pyroscope/nodejs/-/nodejs-0.2.6.tgz", - "integrity": "sha512-F37ROH//HzO7zKm2S7CtNG8OAp+i4ADg4erQR9D57BrSgi8+3Jjp5s5PWqyJABC6IzsABgGrentPobBDr8QdsA==", + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@pyroscope/nodejs/-/nodejs-0.2.9.tgz", + "integrity": "sha512-pIw4pIqcNZTZxTUuV0OUI18UZEmx9lT2GaT75ny6FKVe2L1gxAwTCf5TKk8VsnUGY66buUkyaTHcTm7fy0BP/Q==", "dependencies": { - "axios": "^0.26.1", + "axios": "^0.28.0", "debug": "^4.3.3", "form-data": "^4.0.0", - "pprof": "^3.2.0", + "pprof": "^4.0.0", "regenerator-runtime": "^0.13.11", "source-map": "^0.7.3" }, "engines": { - "node": "^12.20.0 || >=14.13.1" + "node": ">=v18" } }, "node_modules/@pyroscope/nodejs/node_modules/axios": { - "version": "0.26.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", - "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.28.0.tgz", + "integrity": "sha512-Tu7NYoGY4Yoc7I+Npf9HhUMtEEpV7ZiLH9yndTCoNhcpBH0kwcvFbzYN9/u5QKI5A6uefjsNNWaz5olJVYS62Q==", "dependencies": { - "follow-redirects": "^1.14.8" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/@rudderstack/integrations-lib": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.4.tgz", - "integrity": "sha512-32Zose9aOPNWd4EyUNuS5YY+Vq4LYMuDcabJ+s3t1ZfHHMfISlDNF02b60MWgOrU8PARYC+siDs5wgA6xfZpzQ==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.7.tgz", + "integrity": "sha512-F0QVIT2vpSeI+GcUk7AwxMJrmM5SsRk8AS6oH4nHkjjfDoKjdh9rrDVzhXKUYF//FAi32ecmSsW+/6ioB8louw==", "dependencies": { "axios": "^1.4.0", "axios-mock-adapter": "^1.22.0", @@ -4496,13 +4498,13 @@ "integrity": "sha512-+iH40g+ZA2ANgwjOITdEdZJLZV+ljR28Akn/dRoDia591tMu7PptyvDaAvl+m1DijWXddpLQ8SX9xaEcIdmqlw==" }, "node_modules/@rudderstack/workflow-engine": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.7.2.tgz", - "integrity": "sha512-aXQvoXMekvXxxDG6Yc5P5l3PJIwqVA+EmJ2w4SnQ94BUHhbsybPjgGvyzD17MUTAdWEOtqS38SuzLflBs/5T4g==", + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/@rudderstack/workflow-engine/-/workflow-engine-0.7.5.tgz", + "integrity": "sha512-HmhxiF/gZorrEEmVvQYopIN6xicQ7kr0mHtw2fPqXmHIFLr9MnEyefo4+MPw/Re9iNFbXNQC9uKkYd7lLHbAyw==", "dependencies": { "@aws-crypto/sha256-js": "^5.0.0", "@rudderstack/json-template-engine": "^0.8.4", - "jsonata": "^2.0.3", + "jsonata": "^2.0.4", "lodash": "^4.17.21", "object-sizeof": "^2.6.3", "yaml": "^2.3.2" @@ -8907,9 +8909,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", - "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", "engines": { "node": ">=8" } @@ -10498,9 +10500,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -14513,9 +14515,9 @@ } }, "node_modules/jsonata": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/jsonata/-/jsonata-2.0.3.tgz", - "integrity": "sha512-Up2H81MUtjqI/dWwWX7p4+bUMfMrQJVMN/jW6clFMTiYP528fBOBNtRu944QhKTs3+IsVWbgMeUTny5fw2VMUA==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/jsonata/-/jsonata-2.0.4.tgz", + "integrity": "sha512-vfavX4/G/yrYxE+UrmT/oUJ3ph7KqUrb0R7b0LVRcntQwxw+Z5kA1pNUIQzX5hF04Oe1eKxyoIPsmXtc2LgJTQ==", "engines": { "node": ">= 8" } @@ -15274,6 +15276,11 @@ "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==", "dev": true }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==" + }, "node_modules/lodash.startcase": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.startcase/-/lodash.startcase-4.4.0.tgz", @@ -16522,9 +16529,9 @@ "dev": true }, "node_modules/nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", + "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==" }, "node_modules/nanoid": { "version": "3.3.7", @@ -17523,24 +17530,23 @@ "dev": true }, "node_modules/pprof": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/pprof/-/pprof-3.2.1.tgz", - "integrity": "sha512-KnextTM3EHQ2zqN8fUjB0VpE+njcVR7cOfo7DjJSLKzIbKTPelDtokI04ScR/Vd8CLDj+M99tsaKV+K6FHzpzA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pprof/-/pprof-4.0.0.tgz", + "integrity": "sha512-Yhfk7Y0G1MYsy97oXxmSG5nvbM1sCz9EALiNhW/isAv5Xf7svzP+1RfGeBlS6mLSgRJvgSLh6Mi5DaisQuPttw==", "hasInstallScript": true, "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", + "@mapbox/node-pre-gyp": "^1.0.9", "bindings": "^1.2.1", "delay": "^5.0.0", "findit2": "^2.2.3", - "nan": "^2.14.0", + "nan": "^2.17.0", "p-limit": "^3.0.0", - "pify": "^5.0.0", "protobufjs": "~7.2.4", - "source-map": "^0.7.3", + "source-map": "~0.8.0-beta.0", "split": "^1.0.1" }, "engines": { - "node": ">=10.4.1" + "node": ">=14.0.0" } }, "node_modules/pprof-format": { @@ -17548,15 +17554,38 @@ "resolved": "https://registry.npmjs.org/pprof-format/-/pprof-format-2.0.7.tgz", "integrity": "sha512-1qWaGAzwMpaXJP9opRa23nPnt2Egi7RMNoNBptEE/XwHbcn4fC2b/4U4bKc5arkGkIh2ZabpF2bEb+c5GNHEKA==" }, - "node_modules/pprof/node_modules/pify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", - "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", - "engines": { - "node": ">=10" + "node_modules/pprof/node_modules/source-map": { + "version": "0.8.0-beta.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", + "integrity": "sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==", + "dependencies": { + "whatwg-url": "^7.0.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "engines": { + "node": ">= 8" + } + }, + "node_modules/pprof/node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/pprof/node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "node_modules/pprof/node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" } }, "node_modules/precinct": { @@ -17966,9 +17995,9 @@ } }, "node_modules/protobufjs": { - "version": "7.2.5", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", - "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", + "version": "7.2.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz", + "integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==", "hasInstallScript": true, "dependencies": { "@protobufjs/aspromise": "^1.1.2", @@ -19889,9 +19918,9 @@ } }, "node_modules/tar": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", - "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", diff --git a/package.json b/package.json index 5c1d9c1848..1f933cd1fa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.60.0", + "version": "1.61.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -64,8 +64,8 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.6", - "@rudderstack/integrations-lib": "^0.2.4", - "@rudderstack/workflow-engine": "^0.7.2", + "@rudderstack/integrations-lib": "^0.2.7", + "@rudderstack/workflow-engine": "^0.7.5", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", "ajv-draft-04": "^1.0.0", diff --git a/src/cdk/v2/destinations/bloomreach/config.ts b/src/cdk/v2/destinations/bloomreach/config.ts new file mode 100644 index 0000000000..90fbcc63c6 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/config.ts @@ -0,0 +1,30 @@ +import { getMappingConfig } from '../../../../v0/util'; + +export const CUSTOMER_COMMAND = 'customers'; +export const CUSTOMER_EVENT_COMMAND = 'customers/events'; +export const MAX_BATCH_SIZE = 50; + +// ref:- https://documentation.bloomreach.com/engagement/reference/batch-commands-2 +export const getBatchEndpoint = (apiBaseUrl: string, projectToken: string): string => + `${apiBaseUrl}/track/v2/projects/${projectToken}/batch`; + +const CONFIG_CATEGORIES = { + CUSTOMER_PROPERTIES_CONFIG: { name: 'BloomreachCustomerPropertiesConfig' }, +}; +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); +export const EXCLUSION_FIELDS: string[] = [ + 'email', + 'firstName', + 'firstname', + 'first_name', + 'lastName', + 'lastname', + 'last_name', + 'name', + 'phone', + 'city', + 'birthday', + 'country', +]; +export const CUSTOMER_PROPERTIES_CONFIG = + MAPPING_CONFIG[CONFIG_CATEGORIES.CUSTOMER_PROPERTIES_CONFIG.name]; diff --git a/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json b/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json new file mode 100644 index 0000000000..cb4c2f7201 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/data/BloomreachCustomerPropertiesConfig.json @@ -0,0 +1,36 @@ +[ + { + "destKey": "first_name", + "sourceKeys": "firstName", + "sourceFromGenericMap": true + }, + { + "destKey": "last_name", + "sourceKeys": "lastName", + "sourceFromGenericMap": true + }, + { + "destKey": "email", + "sourceKeys": "emailOnly", + "sourceFromGenericMap": true + }, + { + "destKey": "phone", + "sourceKeys": "phone", + "sourceFromGenericMap": true + }, + { + "destKey": "city", + "sourceKeys": "city", + "sourceFromGenericMap": true + }, + { + "destKey": "country", + "sourceKeys": ["traits.address.country", "context.traits.address.country"] + }, + { + "destKey": "birthday", + "sourceKeys": "birthday", + "sourceFromGenericMap": true + } +] diff --git a/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml b/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml new file mode 100644 index 0000000000..f092d90382 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/procWorkflow.yaml @@ -0,0 +1,119 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: toUnixTimestamp + path: ../../../../v0/util + - name: base64Convertor + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - name: generateExclusionList + path: ../../../../v0/util + - name: extractCustomFields + path: ../../../../v0/util + - name: constructPayload + path: ../../../../v0/util + - path: ./utils + - path: ./config + +steps: + - name: messageType + template: | + $.context.messageType = .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.context.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK,.PAGE,.SCREEN])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(.destination.Config.apiKey, "API Key is not present . Aborting"); + $.assertConfig(.destination.Config.apiSecret, "API Secret is not present. Aborting"); + $.assertConfig(.destination.Config.projectToken, "Project Token is not present. Aborting"); + $.assertConfig(.destination.Config.hardID, "Hard ID is not present. Aborting"); + $.assertConfig(.destination.Config.softID, "Soft ID is not present. Aborting"); + $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + const userId = .message.().( + {{{{$.getGenericPaths("userIdOnly")}}}}; + ); + $.assert(userId ?? .message.anonymousId, "Either one of userId or anonymousId is required. Aborting"); + + - name: prepareIdentifyPayload + condition: $.context.messageType === {{$.EventType.IDENTIFY}} + template: | + const customerIDs = $.prepareCustomerIDs(.message, .destination); + const customerProperties = $.constructPayload(.message, $.CUSTOMER_PROPERTIES_CONFIG); + const extraCustomerProperties = $.extractCustomFields(.message, {}, ['traits', 'context.traits'], $.EXCLUSION_FIELDS); + const properties = { + ...customerProperties, + ...extraCustomerProperties + } + const data = .message.().({ + "customer_ids": customerIDs, + "update_timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}), + properties + }); + + $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_COMMAND, data}) + + - name: prepareEventName + steps: + - name: pageEventName + condition: $.context.messageType === {{$.EventType.PAGE}} + template: | + const category = .message.category ?? .message.properties.category; + const name = .message.name || .message.properties.name; + const eventNameArray = ["Viewed"]; + category ? eventNameArray.push(category); + name ? eventNameArray.push(name); + eventNameArray.push("Page"); + $.context.event = eventNameArray.join(" "); + - name: screenEventName + condition: $.context.messageType === {{$.EventType.SCREEN}} + template: | + const category = .message.category ?? .message.properties.category; + const name = .message.name || .message.properties.name; + const eventNameArray = ["Viewed"]; + category ? eventNameArray.push(category); + name ? eventNameArray.push(name); + eventNameArray.push("Screen"); + $.context.event = eventNameArray.join(" "); + - name: trackEventName + condition: $.context.messageType === {{$.EventType.TRACK}} + template: | + $.assert(.message.event, "Event name is required. Aborting"); + $.context.event = .message.event + + - name: prepareTrackPageScreenPayload + condition: $.context.messageType !== {{$.EventType.IDENTIFY}} + template: | + const customerIDs = $.prepareCustomerIDs(.message, .destination); + const data = .message.().({ + "customer_ids": customerIDs, + "timestamp": $.toUnixTimestamp({{{{$.getGenericPaths("timestamp")}}}}), + "properties": .properties, + "event_type": $.context.event, + }); + + $.context.payload = $.removeUndefinedAndNullValues({name: $.CUSTOMER_EVENT_COMMAND, data}) + + - name: buildResponse + description: In batchMode we return payload directly + condition: $.batchMode + template: | + $.context.payload + else: + name: buildResponseForProcessTransformation + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = $.getBatchEndpoint(.destination.Config.apiBaseUrl, .destination.Config.projectToken); + response.method = "POST"; + response.headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.destination.Config.apiKey + ":" + .destination.Config.apiSecret) + } + response; diff --git a/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml b/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml new file mode 100644 index 0000000000..b8b27ca02e --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/rtWorkflow.yaml @@ -0,0 +1,76 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + - path: ./utils + exportAll: true + - name: base64Convertor + path: ../../../../v0/util + - name: toUnixTimestamp + path: ../../../../v0/util + - name: BatchUtils + path: '@rudderstack/workflow-engine' + - path: ./config + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata, + "statusCode": 200 + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + let batches = $.BatchUtils.chunkArrayBySizeAndLength( + $.outputs.successfulEvents, {maxItems: $.MAX_BATCH_SIZE}).items; + + batches@batch.({ + "batchedRequest": { + "body": { + "JSON": {"commands": ~r batch.batchedRequest[]}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": batch[0].destination.Config.().($.getBatchEndpoint(.apiBaseUrl, .projectToken)), + "headers": batch[0].destination.Config.().({ + "Content-Type": "application/json", + "Authorization": "Basic " + $.base64Convertor(.apiKey + ":" + .apiSecret) + }), + "params": {}, + "files": {} + }, + "metadata": ~r batch.metadata[], + "batched": true, + "statusCode": 200, + "destination": batch[0].destination + })[]; + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/bloomreach/utils.ts b/src/cdk/v2/destinations/bloomreach/utils.ts new file mode 100644 index 0000000000..f834fa74e7 --- /dev/null +++ b/src/cdk/v2/destinations/bloomreach/utils.ts @@ -0,0 +1,31 @@ +import { isObject, isEmptyObject, getIntegrationsObj } from '../../../../v0/util'; +import { RudderMessage, Destination } from '../../../../types'; + +const getCustomerIDsFromIntegrationObject = (message: RudderMessage): any => { + const integrationObj = getIntegrationsObj(message, 'bloomreach' as any) || {}; + const { hardID, softID } = integrationObj; + const customerIDs = {}; + + if (isObject(hardID) && !isEmptyObject(hardID)) { + Object.keys(hardID).forEach((id) => { + customerIDs[id] = hardID[id]; + }); + } + + if (isObject(softID) && !isEmptyObject(softID)) { + Object.keys(softID).forEach((id) => { + customerIDs[id] = softID[id]; + }); + } + + return customerIDs; +}; + +export const prepareCustomerIDs = (message: RudderMessage, destination: Destination): any => { + const customerIDs = { + [destination.Config.hardID]: message.userId, + [destination.Config.softID]: message.anonymousId, + ...getCustomerIDsFromIntegrationObject(message), + }; + return customerIDs; +}; diff --git a/src/cdk/v2/destinations/fullstory/procWorkflow.yaml b/src/cdk/v2/destinations/fullstory/procWorkflow.yaml index 1a54e8688c..26da955623 100644 --- a/src/cdk/v2/destinations/fullstory/procWorkflow.yaml +++ b/src/cdk/v2/destinations/fullstory/procWorkflow.yaml @@ -76,7 +76,7 @@ steps: "use_most_recent": .message.properties.useMostRecent, }; $.context.payload.user = { - "id": .message.properties.userId ?? .message.userId, + "uid": .message.properties.userId ?? .message.userId, } - name: cleanPayload diff --git a/src/cdk/v2/destinations/linkedin_ads/config.js b/src/cdk/v2/destinations/linkedin_ads/config.js new file mode 100644 index 0000000000..344980e7d0 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/config.js @@ -0,0 +1,26 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +// ref : https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=http#adding-multiple-conversion-events-in-a-batch +const BATCH_ENDPOINT = 'https://api.linkedin.com/rest/conversionEvents'; +const API_HEADER_METHOD = 'BATCH_CREATE'; +const API_VERSION = '202402'; // yyyymm format +const API_PROTOCOL_VERSION = '2.0.0'; + +const CONFIG_CATEGORIES = { + USER_INFO: { + name: 'linkedinUserInfoConfig', + type: 'user', + }, +}; + +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); + +module.exports = { + MAX_BATCH_SIZE: 5000, + BATCH_ENDPOINT, + API_HEADER_METHOD, + API_VERSION, + API_PROTOCOL_VERSION, + CONFIG_CATEGORIES, + MAPPING_CONFIG, +}; diff --git a/src/cdk/v2/destinations/linkedin_ads/data/linkedinUserInfoConfig.json b/src/cdk/v2/destinations/linkedin_ads/data/linkedinUserInfoConfig.json new file mode 100644 index 0000000000..760510b5b3 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/data/linkedinUserInfoConfig.json @@ -0,0 +1,31 @@ +[ + { + "destKey": "firstName", + "sourceKeys": "firstName", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "lastName", + "sourceKeys": "lastName", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "title", + "sourceKeys": "title", + "required": false, + "sourceFromGenericMap": true + }, + { + "destKey": "companyName", + "sourceKeys": "context.traits.companyName", + "required": false + }, + { + "destKey": "countryCode", + "sourceKeys": "countryCode", + "sourceFromGenericMap": true, + "required": false + } +] diff --git a/src/cdk/v2/destinations/linkedin_ads/procWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/procWorkflow.yaml new file mode 100644 index 0000000000..4b17afc368 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/procWorkflow.yaml @@ -0,0 +1,80 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + exportAll: true + - name: removeUndefinedValues + path: ../../../../v0/util + - name: removeUndefinedNullValuesAndEmptyObjectArray + path: ../../../../v0/util + - name: defaultRequestConfig + path: ../../../../v0/util + - name: OAuthSecretError + path: '@rudderstack/integrations-lib' + - path: ./utils + - path: ./config + - path: lodash + name: cloneDeep + +steps: + - name: checkIfProcessed + condition: .message.statusCode + template: | + $.batchMode ? .message.body.JSON : .message + onComplete: return + - name: messageType + template: | + .message.type.toLowerCase() + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "Message type is not present. Aborting message."); + $.assert(messageType in {{$.EventType.([.TRACK])}}, + "message type " + messageType + " is not supported") + + - name: validateInputForTrack + description: Additional validation for Track events + condition: $.outputs.messageType === {{$.EventType.TRACK}} + template: | + $.assert(.message.event, "event could not be mapped to conversion rule. Aborting.") + - name: commonFields + description: | + Builds common fields in destination payload. + ref: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=curl#adding-multiple-conversion-events-in-a-batch + template: | + let commonFields = .message.().({ + "conversionHappenedAt": $.fetchAndVerifyConversionHappenedAt(^.message), + "eventId": $.getOneByPaths(., ^.destination.Config.deduplicationKey) ?? .messageId, + "conversionValue":$.calculateConversionObject(^.message), + "user":{ + "userIds":$.fetchUserIds(^.message,^.destination.Config), + "userInfo":$.curateUserInfoObject(^.message) + } + }); + $.removeUndefinedValues(commonFields) + - name: basePayload + template: | + const payload = $.outputs.commonFields; + payload + + - name: deduceConversionEventRules + template: | + $.context.deduceConversionRulesArray = $.deduceConversionRules(.message.event,.destination.Config) + + - name: preparePayload + template: | + $.context.payloads = $.context.deduceConversionRulesArray@conversionRuleId.( + const newPayload = $.cloneDeep($.outputs.basePayload); + newPayload.conversion = $.createConversionString(conversionRuleId); + $.removeUndefinedNullValuesAndEmptyObjectArray(newPayload) + )[]; + - name: buildResponse + template: | + $.assertThrow((.metadata.secret && .metadata.secret.accessToken), new $.OAuthSecretError("Secret or accessToken is not present in the metadata")) + const accessToken = .metadata.secret.accessToken + const response = $.defaultRequestConfig(); + response.body.JSON = {elements: $.context.payloads}; + response.endpoint = $.BATCH_ENDPOINT; + response.method = "POST"; + response.headers = $.generateHeader(accessToken) + response diff --git a/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml new file mode 100644 index 0000000000..dda322e45e --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/rtWorkflow.yaml @@ -0,0 +1,39 @@ +bindings: + - path: ./utils + - path: ./config + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "message": .[], + "destination": ^ [idx].destination, + "metadata": ^ [idx].metadata + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.failedEvents, ...$.outputs.batchSuccessfulEvents] diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.js b/src/cdk/v2/destinations/linkedin_ads/utils.js new file mode 100644 index 0000000000..69fea4299d --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/utils.js @@ -0,0 +1,285 @@ +const lodash = require('lodash'); +const crypto = require('crypto'); +const moment = require('moment'); + +const { + InstrumentationError, + getHashFromArrayWithDuplicate, + isDefinedAndNotNullAndNotEmpty, + ConfigurationError, +} = require('@rudderstack/integrations-lib'); +const { + getFieldValueFromMessage, + constructPayload, + getDestinationExternalID, +} = require('../../../../v0/util'); +const { + MAPPING_CONFIG, + CONFIG_CATEGORIES, + MAX_BATCH_SIZE, + API_HEADER_METHOD, + API_PROTOCOL_VERSION, + API_VERSION, +} = require('./config'); +const { + AUTH_STATUS_INACTIVE, + REFRESH_TOKEN, +} = require('../../../../adapters/networkhandler/authConstants'); + +const formatEmail = (email, destConfig) => { + if (email) { + if (destConfig.hashData === true) { + return crypto.createHash('sha256').update(email).digest('hex'); + } + return email; + } + return null; +}; + +const fetchUserIds = (message, destConfig) => { + const userIds = []; + const email = formatEmail(getFieldValueFromMessage(message, 'email'), destConfig); + const linkedinFirstPartyAdsTrackingUUID = getDestinationExternalID( + message, + 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + ); + const acxiomId = getDestinationExternalID(message, 'ACXIOM_ID'); + const oracleMoatId = getDestinationExternalID(message, 'ORACLE_MOAT_ID'); + if (!email && !linkedinFirstPartyAdsTrackingUUID && !acxiomId && !oracleMoatId) { + throw new InstrumentationError( + '[LinkedIn Conversion API] no matching user id found. Please provide at least one of the following: email, linkedinFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + ); + } + + if (email) { + userIds.push({ idType: 'SHA256_EMAIL', idValue: email }); + } + if (linkedinFirstPartyAdsTrackingUUID) { + userIds.push({ + idType: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + idValue: linkedinFirstPartyAdsTrackingUUID, + }); + } + if (acxiomId) { + userIds.push({ idType: 'ACXIOM_ID', idValue: acxiomId }); + } + if (oracleMoatId) { + userIds.push({ idType: 'ORACLE_MOAT_ID', idValue: oracleMoatId }); + } + return userIds; +}; + +const curateUserInfoObject = (message) => { + const commonCategory = CONFIG_CATEGORIES.USER_INFO; + const commonPayload = constructPayload(message, MAPPING_CONFIG[commonCategory.name]); + if (commonPayload.firstName && commonPayload.lastName) { + return commonPayload; + } + return null; +}; + +function checkIfPricePresent(properties) { + // Check if 'products' exists and is an array + if (Array.isArray(properties?.products)) { + // Use 'some' to check if at least one object has a 'price' field + const hasPrice = properties.products.some((product) => product.hasOwnProperty('price')); + return hasPrice; + } + return !!properties.price; +} + +const calculateConversionObject = (message) => { + const { properties, event } = message; + + const calculateAmount = () => { + if (properties?.products && properties.products.length > 0) { + return properties.products.reduce( + (acc, product) => acc + (product.price || 0) * (product.quantity || 1), + 0, + ); + } + return properties.price * (properties.quantity ?? 1); + }; + if (checkIfPricePresent(properties)) { + const conversionObject = { + currencyCode: properties.currency || 'USD', + amount: `${calculateAmount()}`, + }; + return conversionObject; + } + throw new InstrumentationError( + `[LinkedIn Conversion API]: Cannot map price for event ${event}. Aborting`, + ); +}; + +const deduceConversionRules = (trackEventName, destConfig) => { + let conversionRule; + const { conversionMapping } = destConfig; + if (conversionMapping.length > 0) { + const keyMap = getHashFromArrayWithDuplicate(conversionMapping, 'from', 'to', false); + conversionRule = keyMap[trackEventName]; + } + if (isDefinedAndNotNullAndNotEmpty(conversionRule)) { + const finalEvent = typeof conversionRule === 'string' ? [conversionRule] : [...conversionRule]; + return finalEvent; + } + throw new ConfigurationError( + `[LinkedIn Conversion API] no matching conversion rule found for ${trackEventName}. Please provide a conversion rule. Aborting`, + ); +}; + +const createConversionString = (ruleId) => `urn:lla:llaPartnerConversion:${ruleId}`; + +const generateHeader = (accessToken) => { + const headers = { + 'Content-Type': 'application/json', + 'X-RestLi-Method': API_HEADER_METHOD, + 'X-Restli-Protocol-Version': API_PROTOCOL_VERSION, + 'LinkedIn-Version': API_VERSION, + Authorization: `Bearer ${accessToken}`, + }; + return headers; +}; + +const fetchAndVerifyConversionHappenedAt = (message) => { + const timeStamp = message.timestamp || message.originalTimestamp; + if (timeStamp) { + const start = moment(timeStamp); + if (!start.isValid()) { + throw new InstrumentationError('Invalid timestamp format.'); + } + const current = moment(); + // calculates past event in days + const deltaDay = current.diff(start, 'days', true); + + if (Math.ceil(deltaDay) > 90) { + throw new InstrumentationError('Events must be sent within ninety days of their occurrence.'); + } + } + + const timeInMilliseconds = moment(timeStamp).valueOf(); // `valueOf` returns the time in milliseconds + return timeInMilliseconds; +}; + +function batchResponseBuilder(successfulEvents) { + if (successfulEvents.length === 0) { + return []; + } + const constants = { + version: successfulEvents[0].message[0].version, + type: successfulEvents[0].message[0].type, + method: successfulEvents[0].message[0].method, + endpoint: successfulEvents[0].message[0].endpoint, + headers: successfulEvents[0].message[0].headers, + destination: successfulEvents[0].destination, + }; + + const allElements = successfulEvents.flatMap((event) => event.message[0].body.JSON.elements); + const allMetadata = successfulEvents.map((event) => event.metadata); + + // Using lodash to chunk the elements into groups of up to 3 + const chunkedElements = lodash.chunk(allElements, MAX_BATCH_SIZE); + const chunkedMetadata = lodash.chunk(allMetadata, MAX_BATCH_SIZE); + + return chunkedElements.map((elementsBatch, index) => ({ + batchedRequest: { + body: { + JSON: { elements: elementsBatch }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: constants.version, + type: constants.type, + method: constants.method, + endpoint: constants.endpoint, + headers: constants.headers, + params: {}, + files: {}, + }, + metadata: chunkedMetadata[index], + batched: true, + statusCode: 200, + destination: constants.destination, + })); +} + +function constructPartialStatus(errorMessage) { + const errorPattern = /Index: (\d+), ERROR :: (.*?)\n/g; + let match; + const errorMap = {}; + + try { + // eslint-disable-next-line no-cond-assign + while ((match = errorPattern.exec(errorMessage)) !== null) { + const [, index, message] = match; + errorMap[index] = message; + } + + return errorMap; + } catch (e) { + return null; + } +} + +function createResponseArray(metadata, partialStatus) { + const partialStatusArray = Object.entries(partialStatus).map(([index, message]) => [ + Number(index), + message, + ]); + // Convert destPartialStatus to an object for easier lookup + const errorMap = partialStatusArray.reduce((acc, [index, message]) => { + const jobId = metadata[index]?.jobId; // Get the jobId from the metadata array based on the index + if (jobId !== undefined) { + acc[jobId] = message; + } + return acc; + }, {}); + + return metadata.map((item) => { + const error = errorMap[item.jobId]; + return { + statusCode: error ? 400 : 500, + metadata: item, + error: error || 'success', + }; + }); +} + +/** + * + * @param {*} destinationResponse example: {status : 401, response {"status":401,"serviceErrorCode":65601,"code":"REVOKED_ACCESS_TOKEN","message":"The token used in the request has been revoked by the user"}} + * @returns proper auth error category + */ +const getAuthErrCategoryFromStCode = (destinationResponse) => { + const { status, response } = destinationResponse; + if (status === 401) { + if (response.code === 'REVOKED_ACCESS_TOKEN') { + // ACCESS_DENIED + return AUTH_STATUS_INACTIVE; + } + // UNAUTHORIZED + return REFRESH_TOKEN; + } + if (status === 403) { + // ACCESS_DENIED + return AUTH_STATUS_INACTIVE; + } + return ''; +}; + +module.exports = { + formatEmail, + calculateConversionObject, + curateUserInfoObject, + fetchUserIds, + deduceConversionRules, + createConversionString, + generateHeader, + fetchAndVerifyConversionHappenedAt, + batchResponseBuilder, + constructPartialStatus, + createResponseArray, + checkIfPricePresent, + getAuthErrCategoryFromStCode, +}; diff --git a/src/cdk/v2/destinations/linkedin_ads/utils.test.js b/src/cdk/v2/destinations/linkedin_ads/utils.test.js new file mode 100644 index 0000000000..ee52928198 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_ads/utils.test.js @@ -0,0 +1,293 @@ +const crypto = require('crypto'); +const { + formatEmail, + calculateConversionObject, + fetchUserIds, + curateUserInfoObject, + deduceConversionRules, + generateHeader, + constructPartialStatus, + createResponseArray, + checkIfPricePresent, +} = require('./utils'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); +const { API_HEADER_METHOD, API_PROTOCOL_VERSION, API_VERSION } = require('./config'); + +describe('formatEmail', () => { + // Returns a hashed email when a valid email is passed as argument. + it('should return a hashed email when a valid email is passed as argument', () => { + const email = 'test@example.com'; + const hashedEmail = crypto.createHash('sha256').update(email).digest('hex'); + expect(formatEmail(email, { hashData: true })).toEqual(hashedEmail); + }); + + // Returns null when an empty string is passed as argument. + it('should return null when an empty string is passed as argument', () => { + const email = ''; + expect(formatEmail(email)).toBeNull(); + }); +}); + +describe('calculateConversionObject', () => { + // Returns a conversion object with currency code 'USD' and amount 0 when message properties are empty + it('should throw instrumentation error when message properties are empty', () => { + const message = { properties: {} }; + expect(() => { + fetchUserIds(calculateConversionObject(message)); + }).toThrow(InstrumentationError); + }); + + // Returns a conversion object with currency code 'USD' and amount 0 when message properties price is defined but quantity is 0 + it('should return a conversion object with currency code "USD" and amount 0 when message properties price is defined but quantity is 0', () => { + const message = { properties: { price: 10, quantity: 0 } }; + const conversionObject = calculateConversionObject(message); + expect(conversionObject).toEqual({ currencyCode: 'USD', amount: '0' }); + }); +}); + +describe('fetchUserIds', () => { + // Throws an InstrumentationError when no user id is found in the message and no exception is caught + it('should throw an InstrumentationError when no user id is found in the message and no exception is caught', () => { + const message = {}; + const destConfig = { + hashData: true, + }; + expect(() => { + fetchUserIds(message, destConfig); + }).toThrow(InstrumentationError); + }); + it('should create user Ids array of objects with all allowed values', () => { + const message = { + context: { + traits: { + email: 'abc@gmail.com', + }, + externalId: [ + { + type: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + id: 'abcdefg', + }, + { + type: 'ACXIOM_ID', + id: '123456', + }, + { + type: 'ORACLE_MOAT_ID', + id: '789012', + }, + ], + }, + }; + const destConfig = { + hashData: true, + }; + const userIdArray = fetchUserIds(message, destConfig); + expect(userIdArray).toEqual([ + { + idType: 'SHA256_EMAIL', + idValue: '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + { + idType: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + idValue: 'abcdefg', + }, + { + idType: 'ACXIOM_ID', + idValue: '123456', + }, + { + idType: 'ORACLE_MOAT_ID', + idValue: '789012', + }, + ]); + }); +}); + +describe('curateUserInfoObject', () => { + // Returns a non-null object when given a message with both first and last name + it('should return a non-null object when given a message with both first and last name and other properties', () => { + const message = { + context: { + traits: { + firstName: 'John', + lastName: 'Doe', + title: 'Mr.', + companyName: 'RudderTest', + countryCode: 'USA', + }, + }, + }; + const result = curateUserInfoObject(message); + expect(result).toEqual({ + firstName: 'John', + lastName: 'Doe', + title: 'Mr.', + companyName: 'RudderTest', + countryCode: 'USA', + }); + }); + // Returns a null object when given a message with an empty first name + it('should return a null object when given a message without both first and last name', () => { + const message = { + context: { + traits: { + title: 'Mr.', + companyName: 'RudderTest', + countryCode: 'USA', + }, + }, + }; + const result = curateUserInfoObject(message); + expect(result).toEqual(null); + }); +}); + +describe('deduceConversionRules', () => { + // When conversionMapping is empty, return ConfigurationError + it('should return ConfigurationError when conversionMapping is empty', () => { + const trackEventName = 'eventName'; + const destConfig = { + conversionMapping: [], + }; + expect(() => deduceConversionRules(trackEventName, destConfig)).toThrow(ConfigurationError); + }); + + // When conversionMapping is not empty, return the conversion rule + it('should return the conversion rule when conversionMapping is not empty', () => { + const trackEventName = 'eventName'; + const destConfig = { + conversionMapping: [{ from: 'eventName', to: 'conversionEvent' }], + }; + const result = deduceConversionRules(trackEventName, destConfig); + expect(result).toEqual(['conversionEvent']); + }); + + it('should return the conversion rule when conversionMapping is not empty', () => { + const trackEventName = 'eventName'; + const destConfig = { + conversionMapping: [ + { from: 'eventName', to: 'conversionEvent' }, + { from: 'eventName', to: 'conversionEvent2' }, + ], + }; + const result = deduceConversionRules(trackEventName, destConfig); + expect(result).toEqual(['conversionEvent', 'conversionEvent2']); + }); +}); + +describe('generateHeader', () => { + // Returns a headers object with Content-Type, X-RestLi-Method, X-Restli-Protocol-Version, LinkedIn-Version, and Authorization keys when passed a valid access token. + it('should return a headers object with all keys when passed a valid access token', () => { + // Arrange + const accessToken = 'validAccessToken'; + + // Act + const result = generateHeader(accessToken); + + // Assert + expect(result).toEqual({ + 'Content-Type': 'application/json', + 'X-RestLi-Method': API_HEADER_METHOD, + 'X-Restli-Protocol-Version': API_PROTOCOL_VERSION, + 'LinkedIn-Version': API_VERSION, + Authorization: `Bearer ${accessToken}`, + }); + }); + + // Returns a headers object with default values for all keys when passed an invalid access token. + it('should return a headers object with default values for all keys when passed an invalid access token', () => { + // Arrange + const accessToken = 'invalidAccessToken'; + + // Act + const result = generateHeader(accessToken); + + // Assert + expect(result).toEqual({ + 'Content-Type': 'application/json', + 'X-RestLi-Method': API_HEADER_METHOD, + 'X-Restli-Protocol-Version': API_PROTOCOL_VERSION, + 'LinkedIn-Version': API_VERSION, + Authorization: `Bearer ${accessToken}`, + }); + }); +}); + +describe('constructPartialStatus', () => { + // The function correctly constructs a map of error messages when given a string containing error messages. + it('should correctly construct a map of error messages when given a string containing error messages', () => { + const errorMessage = 'Index: 1, ERROR :: Error 1\nIndex: 2, ERROR :: Error 2\n'; + const expectedErrorMap = { + 1: 'Error 1', + 2: 'Error 2', + }; + + const result = constructPartialStatus(errorMessage); + + expect(result).toEqual(expectedErrorMap); + }); + + // The function throws an error when given a non-string input. + it('should throw an error when given a non-string input', () => { + const errorMessage = 123; + const result = constructPartialStatus(errorMessage); + expect(result).toEqual({}); + }); +}); + +describe('createResponseArray', () => { + // Returns an array of objects with statusCode, metadata and error properties + it('should return an array of objects with statusCode, metadata and error properties', () => { + // Arrange + const metadata = [{ jobId: 1 }, { jobId: 2 }, { jobId: 3 }]; + const partialStatus = { + 0: 'Partial status message 1', + 2: 'Partial status message 3', + }; + + // Act + const result = createResponseArray(metadata, partialStatus); + + // Assert + expect(result).toEqual([ + { + statusCode: 400, + metadata: { jobId: 1 }, + error: 'Partial status message 1', + }, + { + statusCode: 500, + metadata: { jobId: 2 }, + error: 'success', + }, + { + statusCode: 400, + metadata: { jobId: 3 }, + error: 'Partial status message 3', + }, + ]); + }); +}); + +describe('checkIfPricePresent', () => { + // Returns true if properties object has a 'price' field + it('should return true when properties object has a price field', () => { + const properties = { price: 10 }; + const result = checkIfPricePresent(properties); + expect(result).toBe(true); + }); + + // Returns true if properties object has a 'products' array with an object containing a 'price' field and a 'price' field in the properties object + it('should return true when properties object has a products array with an object containing a price field and a price field in the properties object', () => { + const properties = { products: [{ price: 10 }, { quantity: 3 }], price: 20 }; + const result = checkIfPricePresent(properties); + expect(result).toBe(true); + }); + + // Returns false if properties object does not have a 'price' field or a 'products' array with an object containing a 'price' field + it('should return false when properties object does not have a price field or a products array with an object containing a price field', () => { + const properties = { quantity: 5 }; + const result = checkIfPricePresent(properties); + expect(result).toBe(false); + }); +}); diff --git a/src/cdk/v2/destinations/movable_ink/config.js b/src/cdk/v2/destinations/movable_ink/config.js index 673e94620e..9a0200ab44 100644 --- a/src/cdk/v2/destinations/movable_ink/config.js +++ b/src/cdk/v2/destinations/movable_ink/config.js @@ -1,3 +1,4 @@ module.exports = { - MAX_REQUEST_SIZE_IN_BYTES: 13500, + MAX_REQUEST_SIZE_IN_BYTES: 1000000, + MAX_BATCH_SIZE: 1000, }; diff --git a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml index 43dbb3cbce..394190049b 100644 --- a/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/procWorkflow.yaml @@ -24,6 +24,7 @@ steps: $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting"); $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting"); $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting"); + $.assert(!(messageType === {{$.EventType.TRACK}} && !(.message.event)), "Event name is not present. Aborting"); const userId = .message.().( {{{{$.getGenericPaths("userIdOnly")}}}}; diff --git a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml index 46afb34d53..3ffa49f15b 100644 --- a/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml +++ b/src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml @@ -42,7 +42,7 @@ steps: description: Batches the successfulEvents template: | let batches = $.BatchUtils.chunkArrayBySizeAndLength( - $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items; + $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES, maxItems: $.MAX_BATCH_SIZE}).items; batches@batch.({ "batchedRequest": { diff --git a/src/cdk/v2/destinations/ninetailed/config.js b/src/cdk/v2/destinations/ninetailed/config.js index c38496a415..a59b2a1671 100644 --- a/src/cdk/v2/destinations/ninetailed/config.js +++ b/src/cdk/v2/destinations/ninetailed/config.js @@ -17,10 +17,6 @@ const ConfigCategories = { type: 'identify', name: 'identifyMapping', }, - PAGE: { - type: 'page', - name: 'pageMapping', - }, }; // MAX_BATCH_SIZE : // Maximum number of events to send in a single batch diff --git a/src/cdk/v2/destinations/ninetailed/data/pageMapping.json b/src/cdk/v2/destinations/ninetailed/data/pageMapping.json deleted file mode 100644 index 80ec2f58f1..0000000000 --- a/src/cdk/v2/destinations/ninetailed/data/pageMapping.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "sourceKeys": "properties", - "required": true, - "destKey": "properties" - } -] diff --git a/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml b/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml index 6f5056ce10..383b850a4d 100644 --- a/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml +++ b/src/cdk/v2/destinations/ninetailed/procWorkflow.yaml @@ -16,7 +16,7 @@ steps: template: | let messageType = $.outputs.messageType; $.assert(messageType, "message Type is not present. Aborting"); - $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY,.PAGE])}}, "message type " + messageType + " is not supported"); + $.assert(messageType in {{$.EventType.([.TRACK,.IDENTIFY])}}, "message type " + messageType + " is not supported"); $.assertConfig(.destination.Config.organisationId, "Organisation ID is not present. Aborting"); $.assertConfig(.destination.Config.environment, "Environment is not present. Aborting"); - name: preparePayload diff --git a/src/cdk/v2/destinations/ninetailed/utils.js b/src/cdk/v2/destinations/ninetailed/utils.js index b716422a0e..47b27b3b9d 100644 --- a/src/cdk/v2/destinations/ninetailed/utils.js +++ b/src/cdk/v2/destinations/ninetailed/utils.js @@ -31,12 +31,6 @@ const constructFullPayload = (message) => { config.mappingConfig[config.ConfigCategories.IDENTIFY.name], ); break; - case 'page': - typeSpecifcPayload = constructPayload( - message, - config.mappingConfig[config.ConfigCategories.PAGE.name], - ); - break; default: break; } diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index 17848e6b94..ee4f4f0b33 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -152,7 +152,20 @@ const DestCanonicalNames = { 'the trade desk', ], INTERCOM: ['INTERCOM', 'intercom', 'Intercom'], + GOOGLE_ADWORDS_REMARKETING_LISTS: [ + 'GOOGLE_ADWORDS_REMARKETING_LISTS', + 'google_adwords_remarketing_lists', + 'Google Adwords Remarketing Lists', + 'google adwords remarketing lists', + ], + GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: [ + 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + 'google_adwords_offline_conversions', + 'Google Adwords Offline Conversions', + 'google adwords offline conversions', + ], koala: ['Koala', 'koala', 'KOALA'], + bloomreach: ['Bloomreach', 'bloomreach', 'BLOOMREACH'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/features.json b/src/features.json index 267923fdb4..6d2cac9340 100644 --- a/src/features.json +++ b/src/features.json @@ -67,8 +67,10 @@ "THE_TRADE_DESK": true, "INTERCOM": true, "NINETAILED": true, - "MOVABLE_INK": true, - "KOALA": true + "KOALA": true, + "LINKEDIN_ADS": true, + "BLOOMREACH": true, + "MOVABLE_INK": true }, "regulations": [ "BRAZE", diff --git a/src/middleware.js b/src/middleware.js index 53aabc90e3..543b3af8d1 100644 --- a/src/middleware.js +++ b/src/middleware.js @@ -1,4 +1,5 @@ const Pyroscope = require('@pyroscope/nodejs'); +const { getDestTypeFromContext } = require('@rudderstack/integrations-lib'); const stats = require('./util/stats'); function initPyroscope() { @@ -26,6 +27,7 @@ function durationMiddleware() { method: ctx.method, code: ctx.status, route: ctx.request.url, + destType: getDestTypeFromContext(ctx), }; stats.timing('http_request_duration', startTime, labels); }; diff --git a/src/types/zodTypes.ts b/src/types/zodTypes.ts index 0a65a2bae2..75f12c5e9b 100644 --- a/src/types/zodTypes.ts +++ b/src/types/zodTypes.ts @@ -1,5 +1,5 @@ -import { z } from 'zod'; import { isDefinedAndNotNullAndNotEmpty } from '@rudderstack/integrations-lib'; +import { z } from 'zod'; import { isHttpStatusSuccess } from '../v0/util'; const ProcessorTransformationOutputSchema = z.object({ diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 5de7ac899d..882dff9e75 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -575,6 +575,30 @@ class Prometheus { type: 'gauge', labelNames: ['destination_id'], }, + { + name: 'mixpanel_batch_engage_pack_size', + help: 'mixpanel_batch_engage_pack_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'mixpanel_batch_group_pack_size', + help: 'mixpanel_batch_group_pack_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'mixpanel_batch_track_pack_size', + help: 'mixpanel_batch_track_pack_size', + type: 'gauge', + labelNames: ['destination_id'], + }, + { + name: 'mixpanel_batch_import_pack_size', + help: 'mixpanel_batch_import_pack_size', + type: 'gauge', + labelNames: ['destination_id'], + }, // Histograms { @@ -587,7 +611,7 @@ class Prometheus { name: 'http_request_duration', help: 'Incoming HTTP requests duration in seconds', type: 'histogram', - labelNames: ['method', 'route', 'code'], + labelNames: ['method', 'route', 'code', 'destType'], }, { name: 'tp_batch_size', diff --git a/src/v0/destinations/azure_datalake/transform.js b/src/v0/destinations/azure_datalake/transform.js index 8d29c70e06..6c97e8671f 100644 --- a/src/v0/destinations/azure_datalake/transform.js +++ b/src/v0/destinations/azure_datalake/transform.js @@ -1,25 +1,24 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const azureDatalake = 'azure_datalake'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'azure_datalake'; function getDataTypeOverride() {} function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = azureDatalake; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, getDataTypeOverride, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } -exports.process = process; +module.exports = { + provider, + process, +}; diff --git a/src/v0/destinations/azure_synapse/transform.js b/src/v0/destinations/azure_synapse/transform.js index d98f269475..a80ad7cbdc 100644 --- a/src/v0/destinations/azure_synapse/transform.js +++ b/src/v0/destinations/azure_synapse/transform.js @@ -1,28 +1,25 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const azureSynapse = 'azure_synapse'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'azure_synapse'; function getDataTypeOverride() {} function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = azureSynapse; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, getDataTypeOverride, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/bq/transform.js b/src/v0/destinations/bq/transform.js index e9e496f6a4..8c8be140a9 100644 --- a/src/v0/destinations/bq/transform.js +++ b/src/v0/destinations/bq/transform.js @@ -1,10 +1,6 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const bigquery = 'bq'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'bq'; function getDataTypeOverride() {} @@ -13,8 +9,7 @@ function process(event) { const whIDResolve = event.request.query.whIDResolve === 'true' || false; const whStoreEvent = event.destination.Config.storeFullEvent === true; const destJsonPaths = event.destination?.Config?.jsonPaths || ''; - const provider = bigquery; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, @@ -23,10 +18,12 @@ function process(event) { provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, destJsonPaths, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/clickhouse/transform.js b/src/v0/destinations/clickhouse/transform.js index 24158cc41f..491475419c 100644 --- a/src/v0/destinations/clickhouse/transform.js +++ b/src/v0/destinations/clickhouse/transform.js @@ -1,53 +1,44 @@ const { processWarehouseMessage } = require('../../../warehouse'); const { getDataType } = require('../../../warehouse/index'); -const clickhouse = 'clickhouse'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'clickhouse'; function getDataTypeOverride(key, val, options) { if (options.chEnableArraySupport === 'false') { return 'string'; } - if (Array.isArray(val)) { - // for now returning it as string. confirm this case - if (val.length === 0) { - return 'string'; - } - // check for different data types in the array. if there are different then return array(string) - const firstValueDataType = getDataType(key, val[0], {}); - let finalDataType = firstValueDataType; - for (let i = 1; i < val.length; i += 1) { - const dataType = getDataType(key, val[i], {}); - if (finalDataType !== dataType) { - if (finalDataType === 'string') { - break; - } - if (dataType === 'float' && finalDataType === 'int') { - finalDataType = 'float'; - // eslint-disable-next-line no-continue - continue; - } - if (dataType === 'int' && finalDataType === 'float') { - // eslint-disable-next-line no-continue - continue; - } - finalDataType = 'string'; + if (!Array.isArray(val) || val.length === 0) { + return 'string'; + } + + // check for different data types in the array. if there are different -> return array(string) + let finalDataType = getDataType(key, val[0], {}); + for (let i = 1; i < val.length; i += 1) { + const dataType = getDataType(key, val[i], {}); + if (finalDataType !== dataType) { + if (finalDataType === 'string') { + break; + } + if (dataType === 'float' && finalDataType === 'int') { + finalDataType = 'float'; + // eslint-disable-next-line no-continue + continue; + } + if (dataType === 'int' && finalDataType === 'float') { + // eslint-disable-next-line no-continue + continue; } + finalDataType = 'string'; } - return `array(${finalDataType})`; } - return 'string'; + return `array(${finalDataType})`; } function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = clickhouse; const chEnableArraySupport = event.request.query.chEnableArraySupport || 'false'; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, @@ -55,10 +46,12 @@ function process(event) { provider, chEnableArraySupport, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/deltalake/transform.js b/src/v0/destinations/deltalake/transform.js index 49d40131d4..637b64cf36 100644 --- a/src/v0/destinations/deltalake/transform.js +++ b/src/v0/destinations/deltalake/transform.js @@ -1,25 +1,22 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const deltalake = 'deltalake'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} - -function getDataTypeOverride() {} +const provider = 'deltalake'; function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = deltalake; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, - getDataTypeOverride, + getDataTypeOverride: () => {}, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } -exports.process = process; +module.exports = { + provider, + process, +}; diff --git a/src/v0/destinations/ga4/transform.js b/src/v0/destinations/ga4/transform.js index d8fc531e92..5280a46dab 100644 --- a/src/v0/destinations/ga4/transform.js +++ b/src/v0/destinations/ga4/transform.js @@ -27,6 +27,7 @@ const { const { getItemsArray, validateEventName, + prepareUserConsents, removeInvalidParams, isReservedEventName, getGA4ExclusionList, @@ -238,6 +239,12 @@ const responseBuilder = (message, { Config }) => { rawPayload.user_properties = userProperties; } + // Prepare GA4 consents + const consents = prepareUserConsents(message); + if (!isEmptyObject(consents)) { + rawPayload.consent = consents; + } + payload = removeUndefinedAndNullValues(payload); rawPayload = { ...rawPayload, events: [payload] }; diff --git a/src/v0/destinations/ga4/utils.js b/src/v0/destinations/ga4/utils.js index e4db494727..ce8afda560 100644 --- a/src/v0/destinations/ga4/utils.js +++ b/src/v0/destinations/ga4/utils.js @@ -7,8 +7,10 @@ const { isEmptyObject, extractCustomFields, isDefinedAndNotNull, + getIntegrationsObj, } = require('../../util'); const { mappingConfig, ConfigCategory } = require('./config'); +const { finaliseAnalyticsConsents } = require('../../util/googleUtils'); /** * Reserved event names cannot be used @@ -432,11 +434,30 @@ const prepareUserProperties = (message, piiPropertiesToIgnore = []) => { return validatedUserProperties; }; +/** + * Returns user consents + * Ref : https://developers.google.com/analytics/devguides/collection/protocol/ga4/reference?client_type=gtag#payload_consent + * @param {*} message + * @returns + */ +const prepareUserConsents = (message) => { + const integrationObj = getIntegrationsObj(message, 'ga4') || {}; + const eventLevelConsentsData = integrationObj?.consents || {}; + const consentConfigMap = { + analyticsPersonalizationConsent: 'ad_user_data', + analyticsUserDataConsent: 'ad_personalization', + }; + + const consents = finaliseAnalyticsConsents(consentConfigMap, eventLevelConsentsData); + return consents; +}; + module.exports = { getItem, getItemList, getItemsArray, validateEventName, + prepareUserConsents, removeInvalidParams, isReservedEventName, getGA4ExclusionList, diff --git a/src/v0/destinations/ga4/utils.test.js b/src/v0/destinations/ga4/utils.test.js index 18b3ab5766..501778910f 100644 --- a/src/v0/destinations/ga4/utils.test.js +++ b/src/v0/destinations/ga4/utils.test.js @@ -1,4 +1,9 @@ -const { validateEventName, prepareUserProperties, removeInvalidParams } = require('./utils'); +const { + validateEventName, + removeInvalidParams, + prepareUserConsents, + prepareUserProperties, +} = require('./utils'); const userPropertyData = [ { @@ -447,4 +452,84 @@ describe('Google Analytics 4 utils test', () => { expect(result).toEqual(expected); }); }); + + describe('prepareUserConsents function tests', () => { + it('Should return an empty object when no consents are given', () => { + const message = {}; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return an empty object when no consents are given', () => { + const message = { + integrations: { + GA4: {}, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return an empty object when no consents are given', () => { + const message = { + integrations: { + GA4: { + consents: {}, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return a consents object when consents are given', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({ + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }); + }); + + it('Should return an empty object when invalid consents are given', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'NOT_SPECIFIED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({}); + }); + + it('Should return a valid consents values from consents object', () => { + const message = { + integrations: { + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'DENIED', + }, + }, + }, + }; + const result = prepareUserConsents(message); + expect(result).toEqual({ + ad_user_data: 'DENIED', + }); + }); + }); }); diff --git a/src/v0/destinations/gcs_datalake/transform.js b/src/v0/destinations/gcs_datalake/transform.js index 3e5f1dcfa3..366dcf3483 100644 --- a/src/v0/destinations/gcs_datalake/transform.js +++ b/src/v0/destinations/gcs_datalake/transform.js @@ -1,25 +1,22 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const gcsDatalake = 'gcs_datalake'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} - -function getDataTypeOverride() {} +const provider = 'gcs_datalake'; function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = gcsDatalake; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, - getDataTypeOverride, + getDataTypeOverride: () => {}, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } -exports.process = process; +module.exports = { + provider, + process, +}; diff --git a/src/v0/destinations/google_adwords_offline_conversions/config.js b/src/v0/destinations/google_adwords_offline_conversions/config.js index a02732894f..f065be946c 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/config.js +++ b/src/v0/destinations/google_adwords_offline_conversions/config.js @@ -1,6 +1,6 @@ const { getMappingConfig } = require('../../util'); -const API_VERSION = 'v14'; +const API_VERSION = 'v16'; const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`; @@ -42,6 +42,11 @@ const CONVERSION_CUSTOM_VARIABLE_CACHE_TTL = process.env.CONVERSION_CUSTOM_VARIA const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); +const consentConfigMap = { + personalizationConsent: 'adPersonalization', + userDataConsent: 'adUserData', +}; + module.exports = { trackClickConversionsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_CLICK_CONVERSIONS_CONFIG.name], @@ -58,4 +63,5 @@ module.exports = { MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_CONVERSION_CONFIG_ADD_CONVERSION.name], trackAddStoreAddressConversionsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK_STORE_ADDRESS_IDENTIFIER.name], + consentConfigMap, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 68d4d01fa7..c3be0f7cab 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -3,7 +3,6 @@ const { InstrumentationError, ConfigurationError } = require('@rudderstack/integ const { EventType } = require('../../../constants'); const { getHashFromArrayWithDuplicate, - constructPayload, removeHyphens, getHashFromArray, handleRtTfSingleEventError, @@ -11,16 +10,14 @@ const { getSuccessRespEvents, combineBatchRequestsWithSameJobIds, } = require('../../util'); -const { - CALL_CONVERSION, - trackCallConversionsMapping, - STORE_CONVERSION_CONFIG, -} = require('./config'); +const { CALL_CONVERSION, STORE_CONVERSION_CONFIG } = require('./config'); const { validateDestinationConfig, getStoreConversionPayload, requestBuilder, getClickConversionPayloadAndEndpoint, + getConsentsDataFromIntegrationObj, + getCallConversionPayload, } = require('./utils'); const helper = require('./helper'); @@ -41,12 +38,15 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => const { properties, timestamp, originalTimestamp } = message; const filteredCustomerId = removeHyphens(customerId); + const eventLevelConsentsData = getConsentsDataFromIntegrationObj(message); + if (conversionType === 'click') { // click conversion const convertedPayload = getClickConversionPayloadAndEndpoint( message, Config, filteredCustomerId, + eventLevelConsentsData, ); payload = convertedPayload.payload; endpoint = convertedPayload.endpoint; @@ -55,7 +55,7 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => endpoint = STORE_CONVERSION_CONFIG.replace(':customerId', filteredCustomerId); } else { // call conversions - payload = constructPayload(message, trackCallConversionsMapping); + payload = getCallConversionPayload(message, Config, eventLevelConsentsData); endpoint = CALL_CONVERSION.replace(':customerId', filteredCustomerId); } @@ -119,7 +119,6 @@ const trackResponseBuilder = (message, metadata, destination) => { const process = async (event) => { const { message, metadata, destination } = event; - if (!message.type) { throw new InstrumentationError('Message type is not present. Aborting message.'); } diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 67c0ef31c8..70b42e2157 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -18,6 +18,7 @@ const { isDefinedAndNotNull, getAuthErrCategoryFromStCode, getAccessToken, + getIntegrationsObj, } = require('../../util'); const { SEARCH_STREAM, @@ -27,10 +28,13 @@ const { trackAddStoreAddressConversionsMapping, trackClickConversionsMapping, CLICK_CONVERSION, + trackCallConversionsMapping, + consentConfigMap, } = require('./config'); const { processAxiosResponse } = require('../../../adapters/utils/networkUtils'); const Cache = require('../../util/cache'); const helper = require('./helper'); +const { finaliseConsent } = require('../../util/googleUtils'); const conversionActionIdCache = new Cache(CONVERSION_ACTION_ID_CACHE_TTL); @@ -221,6 +225,17 @@ function getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier) { return result; } +const getCallConversionPayload = (message, Config, eventLevelConsentsData) => { + const payload = constructPayload(message, trackCallConversionsMapping); + // here conversions[0] should be present because there are some mandatory properties mapped in the mapping json. + payload.conversions[0].consent = finaliseConsent( + consentConfigMap, + eventLevelConsentsData, + Config, + ); + return payload; +}; + /** * This Function create the add conversion payload * and returns the payload @@ -277,6 +292,10 @@ const getAddConversionPayload = (message, Config) => { set(payload, 'operations.create.userIdentifiers[0]', {}); } } + // add consent support for store conversions. Note: No event level consent supported. + const consentObject = finaliseConsent(consentConfigMap, {}, Config); + // create property should be present because there are some mandatory properties mapped in the mapping json. + set(payload, 'operations.create.consent', consentObject); return payload; }; @@ -292,7 +311,12 @@ const getStoreConversionPayload = (message, Config, event) => { return payload; }; -const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerId) => { +const getClickConversionPayloadAndEndpoint = ( + message, + Config, + filteredCustomerId, + eventLevelConsent, +) => { const email = getFieldValueFromMessage(message, 'emailOnly'); const phone = getFieldValueFromMessage(message, 'phone'); const { hashUserIdentifier, defaultUserIdentifier, UserIdentifierSource, conversionEnvironment } = @@ -364,9 +388,19 @@ const getClickConversionPayloadAndEndpoint = (message, Config, filteredCustomerI if (!properties.conversionEnvironment && conversionEnvironment !== 'none') { set(payload, 'conversions[0].conversionEnvironment', conversionEnvironment); } + + // add consent support for click conversions + const consentObject = finaliseConsent(consentConfigMap, eventLevelConsent, Config); + // here conversions[0] is expected to be present there are some mandatory properties mapped in the mapping json. + set(payload, 'conversions[0].consent', consentObject); return { payload, endpoint }; }; +const getConsentsDataFromIntegrationObj = (message) => { + const integrationObj = getIntegrationsObj(message, 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS') || {}; + return integrationObj?.consents || {}; +}; + module.exports = { validateDestinationConfig, generateItemListFromProducts, @@ -377,4 +411,6 @@ module.exports = { buildAndGetAddress, getClickConversionPayloadAndEndpoint, getExisitingUserIdentifier, + getConsentsDataFromIntegrationObj, + getCallConversionPayload, }; diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js index 8deaa3ab0a..2d1863413c 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js @@ -2,6 +2,8 @@ const { getClickConversionPayloadAndEndpoint, buildAndGetAddress, getExisitingUserIdentifier, + getConsentsDataFromIntegrationObj, + getCallConversionPayload, } = require('./utils'); const getTestMessage = () => { @@ -161,12 +163,16 @@ describe('getExisitingUserIdentifier util tests', () => { describe('getClickConversionPayloadAndEndpoint util tests', () => { it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present', () => { let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', payload: { conversions: [ { conversionDateTime: '2022-01-01 12:32:45-08:00', conversionEnvironment: 'WEB', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, userIdentifiers: [ { hashedEmail: 'fa922cb41ff930664d4c9ced3c472ce7ecf29a0f8248b7018456e990177fff75', @@ -187,11 +193,15 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { delete fittingPayload.traits.email; delete fittingPayload.properties.email; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', payload: { conversions: [ { conversionDateTime: '2022-01-01 12:32:45-08:00', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, conversionEnvironment: 'WEB', userIdentifiers: [ { @@ -215,7 +225,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { delete fittingPayload.traits.phone; delete fittingPayload.properties.email; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', payload: { conversions: [ { @@ -237,7 +247,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { ).toThrow('Either of email or phone is required for user identifier'); }); - it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present and product list present', () => { + it('finaliseConsent', () => { let fittingPayload = { ...getTestMessage() }; fittingPayload.properties.products = [ { @@ -251,13 +261,17 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { }, ]; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', payload: { conversions: [ { cartData: { items: [{ productId: 1234, quantity: 2, unitPrice: 10 }] }, conversionDateTime: '2022-01-01 12:32:45-08:00', conversionEnvironment: 'WEB', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, userIdentifiers: [ { hashedEmail: 'fa922cb41ff930664d4c9ced3c472ce7ecf29a0f8248b7018456e990177fff75', @@ -273,3 +287,118 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { ); }); }); + +describe('getConsentsDataFromIntegrationObj', () => { + it('should return an empty object when conversionType is "store"', () => { + const message = {}; + const result = getConsentsDataFromIntegrationObj(message); + expect(result).toEqual({}); + }); + it('should return the consent object when conversion type is call', () => { + const message = { + integrations: { + GOOGLE_ADWORDS_OFFLINE_CONVERSIONS: { + consents: { + adUserData: 'GRANTED', + adPersonalization: 'DENIED', + }, + }, + }, + }; + const conversionType = 'call'; + const result = getConsentsDataFromIntegrationObj(message, conversionType); + expect(result).toEqual({ + adPersonalization: 'DENIED', + adUserData: 'GRANTED', + }); + }); +}); + +describe('getCallConversionPayload', () => { + it('should call conversion payload with consent object', () => { + const message = { + properties: { + callerId: '1234', + callStartDateTime: '2022-01-01 12:32:45-08:00', + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + }; + const result = getCallConversionPayload( + message, + { + userDataConsent: 'GRANTED', + personalizationConsent: 'DENIED', + }, + { + adUserData: 'GRANTED', + adPersonalization: 'GRANTED', + }, + ); + expect(result).toEqual({ + conversions: [ + { + callStartDateTime: '2022-01-01 12:32:45-08:00', + callerId: '1234', + consent: { + adPersonalization: 'GRANTED', + adUserData: 'GRANTED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + ], + }); + }); + it('should call conversion payload with consent object', () => { + const message = { + properties: { + callerId: '1234', + callStartDateTime: '2022-01-01 12:32:45-08:00', + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + }; + const result = getCallConversionPayload( + message, + { + userDataConsent: 'GRANTED', + personalizationConsent: 'DENIED', + }, + {}, + ); + expect(result).toEqual({ + conversions: [ + { + callStartDateTime: '2022-01-01 12:32:45-08:00', + callerId: '1234', + consent: { + adPersonalization: 'DENIED', + adUserData: 'GRANTED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + ], + }); + }); + it('should call conversion payload with consent object even if no consent input from UI as well as event level', () => { + const message = { + properties: { + callerId: '1234', + callStartDateTime: '2022-01-01 12:32:45-08:00', + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + }; + const result = getCallConversionPayload(message, {}, {}); + expect(result).toEqual({ + conversions: [ + { + callStartDateTime: '2022-01-01 12:32:45-08:00', + callerId: '1234', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + conversionDateTime: '2022-01-01 12:32:45-08:00', + }, + ], + }); + }); +}); diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js index 5bf0d8a299..0f08b3866d 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/config.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/config.js @@ -16,6 +16,11 @@ const TYPEOFLIST = Object.freeze({ mobileDeviceID: 'mobileId', }); +const consentConfigMap = { + personalizationConsent: 'adPersonalization', + userDataConsent: 'adUserData', +}; + module.exports = { BASE_ENDPOINT, TYPEOFLIST, @@ -23,4 +28,5 @@ module.exports = { hashAttributes, offlineDataJobsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.AUDIENCE_LIST.name], addressInfoMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.ADDRESSINFO.name], + consentConfigMap, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/transform.js b/src/v0/destinations/google_adwords_remarketing_lists/transform.js index 9ab415346a..b0dfaa0c35 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/transform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/transform.js @@ -15,7 +15,7 @@ const { getAccessToken, } = require('../../util'); -const { populateConsentForGoogleDestinations } = require('../../util/googleUtils'); +const { populateConsentFromConfig } = require('../../util/googleUtils'); const { offlineDataJobsMapping, @@ -24,6 +24,7 @@ const { attributeMapping, hashAttributes, TYPEOFLIST, + consentConfigMap, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { MappedToDestinationKey } = require('../../../constants'); @@ -218,7 +219,7 @@ const processEvent = async (metadata, message, destination) => { } Object.values(createdPayload).forEach((data) => { - const consentObj = populateConsentForGoogleDestinations(destination.Config); + const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); response.push(responseBuilder(metadata, data, destination, message, consentObj)); }); return response; diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 10271bebef..a2c40a5672 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -37,6 +37,7 @@ const { batchEvents, trimTraits, generatePageOrScreenCustomEventName, + recordBatchSizeMetrics, } = require('./util'); const { CommonUtils } = require('../../../util/common'); @@ -479,6 +480,13 @@ const process = (event) => processSingleMessage(event.message, event.destination // Ref: https://help.mixpanel.com/hc/en-us/articles/115004613766-Default-Properties-Collected-by-Mixpanel // Ref: https://help.mixpanel.com/hc/en-us/articles/115004561786-Track-UTM-Tags const processRouterDest = async (inputs, reqMetadata) => { + const batchSize = { + engage: 0, + groups: 0, + track: 0, + import: 0, + }; + const groupedEvents = groupEventsByType(inputs); const response = await Promise.all( groupedEvents.map(async (listOfEvents) => { @@ -521,12 +529,20 @@ const processRouterDest = async (inputs, reqMetadata) => { ...importRespList, ]; + batchSize.engage += engageRespList.length; + batchSize.groups += groupsRespList.length; + batchSize.track += trackRespList.length; + batchSize.import += importRespList.length; + return [...batchSuccessRespList, ...batchErrorRespList]; }), ); // Flatten the response array containing batched events from multiple groups const allBatchedEvents = lodash.flatMap(response); + + const { destination } = allBatchedEvents[0]; + recordBatchSizeMetrics(batchSize, destination.ID); return combineBatchRequestsWithSameJobIds(allBatchedEvents); }; diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index f56242d88b..d564e805ad 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -25,6 +25,7 @@ const { mappingConfig, } = require('./config'); const { CommonUtils } = require('../../../util/common'); +const stats = require('../../../util/stats'); const mPIdentifyConfigJson = mappingConfig[ConfigCategory.IDENTIFY.name]; const mPProfileAndroidConfigJson = mappingConfig[ConfigCategory.PROFILE_ANDROID.name]; @@ -342,6 +343,32 @@ const generatePageOrScreenCustomEventName = (message, userDefinedEventTemplate) return eventName; }; +/** + * Records the batch size metrics for different endpoints. + * + * @param {Object} batchSize - The object containing the batch size for different endpoints. + * @param {number} batchSize.engage - The batch size for engage endpoint. + * @param {number} batchSize.groups - The batch size for group endpoint. + * @param {number} batchSize.track - The batch size for track endpoint. + * @param {number} batchSize.import - The batch size for import endpoint. + * @param {string} destinationId - The ID of the destination. + * @returns {void} + */ +const recordBatchSizeMetrics = (batchSize, destinationId) => { + stats.gauge('mixpanel_batch_engage_pack_size', batchSize.engage, { + destination_id: destinationId, + }); + stats.gauge('mixpanel_batch_group_pack_size', batchSize.groups, { + destination_id: destinationId, + }); + stats.gauge('mixpanel_batch_track_pack_size', batchSize.track, { + destination_id: destinationId, + }); + stats.gauge('mixpanel_batch_import_pack_size', batchSize.import, { + destination_id: destinationId, + }); +}; + module.exports = { createIdentifyResponse, isImportAuthCredentialsAvailable, @@ -351,4 +378,5 @@ module.exports = { batchEvents, trimTraits, generatePageOrScreenCustomEventName, + recordBatchSizeMetrics, }; diff --git a/src/v0/destinations/mssql/transform.js b/src/v0/destinations/mssql/transform.js index 2baadebdee..12dd7b40c6 100644 --- a/src/v0/destinations/mssql/transform.js +++ b/src/v0/destinations/mssql/transform.js @@ -1,28 +1,24 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const mssql = 'mssql'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} - +const provider = 'mssql'; function getDataTypeOverride() {} function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = mssql; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, getDataTypeOverride, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/postgres/transform.js b/src/v0/destinations/postgres/transform.js index 32c6b0a069..b57bf4369a 100644 --- a/src/v0/destinations/postgres/transform.js +++ b/src/v0/destinations/postgres/transform.js @@ -1,10 +1,6 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const postgres = 'postgres'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'postgres'; function getDataTypeOverride(key, val, options, jsonKey = false) { if (key === 'violationErrors' || jsonKey) { @@ -17,8 +13,7 @@ function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; const destJsonPaths = event.destination?.Config?.jsonPaths || ''; - const provider = postgres; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, @@ -26,10 +21,12 @@ function process(event) { provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, destJsonPaths, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/rs/transform.js b/src/v0/destinations/rs/transform.js index f051ff49d5..781600a8e2 100644 --- a/src/v0/destinations/rs/transform.js +++ b/src/v0/destinations/rs/transform.js @@ -2,11 +2,7 @@ const { processWarehouseMessage } = require('../../../warehouse'); // redshift destination string limit, if the string length crosses 512 we will change data type to text which is varchar(max) in redshift const RSStringLimit = 512; -const redshift = 'rs'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'rs'; function getDataTypeOverride(key, val, options, jsonKey = false) { if (jsonKey) { @@ -26,8 +22,7 @@ function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; const destJsonPaths = event.destination?.Config?.jsonPaths || ''; - const provider = redshift; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, @@ -35,10 +30,12 @@ function process(event) { provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, destJsonPaths, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/s3_datalake/transform.js b/src/v0/destinations/s3_datalake/transform.js index 7013224faa..8bbfa1556d 100644 --- a/src/v0/destinations/s3_datalake/transform.js +++ b/src/v0/destinations/s3_datalake/transform.js @@ -1,29 +1,25 @@ const { processWarehouseMessage } = require('../../../warehouse'); // use postgres providers for s3-datalake -const s3datalakeProvider = 's3_datalake'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} - +const provider = 's3_datalake'; function getDataTypeOverride() {} function process(event) { const whSchemaVersion = event.request.query.whSchemaVersion || 'v1'; const whStoreEvent = event.destination.Config.storeFullEvent === true; - const provider = s3datalakeProvider; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, getDataTypeOverride, provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/destinations/snapchat_conversion/config.js b/src/v0/destinations/snapchat_conversion/config.js index e0126ea3b1..1cce713fbb 100644 --- a/src/v0/destinations/snapchat_conversion/config.js +++ b/src/v0/destinations/snapchat_conversion/config.js @@ -55,6 +55,7 @@ const eventNameMapping = { save: 'SAVE', subscribe: 'SUBSCRIBE', complete_tutorial: 'COMPLETE_TUTORIAL', + level_complete: 'LEVEL_COMPLETE', invite: 'INVITE', login: 'LOGIN', share: 'SHARE', diff --git a/src/v0/destinations/snowflake/transform.js b/src/v0/destinations/snowflake/transform.js index 7682d13db3..bf53c57978 100644 --- a/src/v0/destinations/snowflake/transform.js +++ b/src/v0/destinations/snowflake/transform.js @@ -1,10 +1,6 @@ const { processWarehouseMessage } = require('../../../warehouse'); -const snowflake = 'snowflake'; - -function processSingleMessage(message, options) { - return processWarehouseMessage(message, options); -} +const provider = 'snowflake'; function getDataTypeOverride(key, val, options, jsonKey = false) { if (key === 'violationErrors' || jsonKey) { @@ -18,8 +14,7 @@ function process(event) { const whIDResolve = event.request.query.whIDResolve === 'true' || false; const whStoreEvent = event.destination.Config.storeFullEvent === true; const destJsonPaths = event.destination?.Config?.jsonPaths || ''; - const provider = snowflake; - return processSingleMessage(event.message, { + return processWarehouseMessage(event.message, { metadata: event.metadata, whSchemaVersion, whStoreEvent, @@ -28,10 +23,12 @@ function process(event) { provider, sourceCategory: event.metadata ? event.metadata.sourceCategory : null, destJsonPaths, + destConfig: event.destination?.Config, }); } module.exports = { + provider, process, getDataTypeOverride, }; diff --git a/src/v0/sources/shopify/transform.js b/src/v0/sources/shopify/transform.js index 4f09984054..93e3ed0c72 100644 --- a/src/v0/sources/shopify/transform.js +++ b/src/v0/sources/shopify/transform.js @@ -154,7 +154,7 @@ const processEvent = async (inputEvent, metricMetadata) => { default: if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) { stats.increment('invalid_shopify_event', { - event: shopifyTopic, + writeKey: metricMetadata.writeKey, source: metricMetadata.source, shopifyTopic: metricMetadata.shopifyTopic, }); diff --git a/src/v0/util/data/GenericFieldMapping.json b/src/v0/util/data/GenericFieldMapping.json index 87dd5e5e55..0a7b309d89 100644 --- a/src/v0/util/data/GenericFieldMapping.json +++ b/src/v0/util/data/GenericFieldMapping.json @@ -116,5 +116,11 @@ "context.traits.address.postal_code", "context.traits.address.postalCode" ], - "sessionId": ["session_id", "context.sessionId"] + "sessionId": ["session_id", "context.sessionId"], + "countryCode": [ + "traits.countryCode", + "traits.address.countryCode", + "context.traits.address.countryCode", + "context.traits.countryCode" + ] } diff --git a/src/v0/util/googleUtils/index.js b/src/v0/util/googleUtils/index.js index de73b0fb05..ef7c244c17 100644 --- a/src/v0/util/googleUtils/index.js +++ b/src/v0/util/googleUtils/index.js @@ -1,36 +1,118 @@ const GOOGLE_ALLOWED_CONSENT_STATUS = ['UNSPECIFIED', 'UNKNOWN', 'GRANTED', 'DENIED']; +const GA4_ALLOWED_CONSENT_STATUS = ['GRANTED', 'DENIED']; + +const UNSPECIFIED_CONSENT = 'UNSPECIFIED'; +const UNKNOWN_CONSENT = 'UNKNOWN'; /** - * Populates the consent object based on the provided properties. + * Populates the consent object based on the provided configuration and consent mapping. * - * @param {object} properties - message.properties containing properties related to consent. - * @returns {object} - An object containing consent information. - * ref : https://developers.google.com/google-ads/api/rest/reference/rest/v15/Consent + * @param {Object} config - The configuration object containing consent values. + * @param {Object} consentConfigMap - The mapping of consent keys to consent types. + * @returns {Object} - The consent object populated with consent values based on the configuration. + * * ref : https://developers.google.com/google-ads/api/rest/reference/rest/v16/Consent */ - -const populateConsentForGoogleDestinations = (config) => { +const populateConsentFromConfig = (config, consentConfigMap) => { const consent = {}; - if (config?.userDataConsent) { - if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(config.userDataConsent)) { - consent.adUserData = config.userDataConsent; + Object.keys(consentConfigMap).forEach((key) => { + const consentType = consentConfigMap[key]; + if (config?.[key]) { + if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(config[key])) { + consent[consentType] = config[key]; + } else { + consent[consentType] = UNKNOWN_CONSENT; + } } else { - consent.adUserData = 'UNKNOWN'; + consent[consentType] = UNSPECIFIED_CONSENT; } - } else { - consent.adUserData = 'UNSPECIFIED'; + }); + + return consent; +}; + +/** + * Generates the final consent object based on the provided consent configuration map, event-level consent, and destination configuration. + * + * @param {Object} consentConfigMap - The map of consent configuration keys and their corresponding consent types. + * @param {Object} [eventLevelConsent={}] - The event-level consent object. + * @param {Object} [destConfig={}] - The destination configuration object. + * @returns {Object} The final consent object. + * ref : + * 1) For click conversion : + * a) https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/uploadClickConversions#ClickConversion + * b) https://developers.google.com/google-ads/api/reference/rpc/v16/ClickConversion#consent + * 2) For Call conversion : + * a) https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/uploadCallConversions#CallConversion + * b) https://developers.google.com/google-ads/api/reference/rpc/v16/CallConversion#consent + * 3) For Store sales conversion : + * a) https://developers.google.com/google-ads/api/reference/rpc/v16/UserData + * b) https://developers.google.com/google-ads/api/reference/rpc/v16/UserData#consent + */ +const finaliseConsent = (consentConfigMap, eventLevelConsent = {}, destConfig = {}) => { + // Initialize defaultConsentBlock with unspecified consent for all keys defined in consentConfigMap + const defaultConsentBlock = Object.keys(consentConfigMap).reduce((acc, key) => { + const consentType = consentConfigMap[key]; + acc[consentType] = UNSPECIFIED_CONSENT; + return acc; + }, {}); + + // If destConfig is provided, update defaultConsentBlock based on it using populateConsentFromConfig + if (Object.keys(destConfig).length > 0) { + const populatedConsent = populateConsentFromConfig(destConfig, consentConfigMap); + Object.assign(defaultConsentBlock, populatedConsent); } - if (config?.personalizationConsent) { - if (GOOGLE_ALLOWED_CONSENT_STATUS.includes(config.personalizationConsent)) { - consent.adPersonalization = config.personalizationConsent; + const consentObj = {}; + + // Iterate through each key in consentConfigMap to determine the final consent + Object.keys(consentConfigMap).forEach((configKey) => { + const consentKey = consentConfigMap[configKey]; // e.g., 'adUserData' + + // Prioritize event-level consent if available + if (eventLevelConsent && eventLevelConsent.hasOwnProperty(consentKey)) { + consentObj[consentKey] = GOOGLE_ALLOWED_CONSENT_STATUS.includes(eventLevelConsent[consentKey]) + ? eventLevelConsent[consentKey] + : UNKNOWN_CONSENT; } else { - consent.adPersonalization = 'UNKNOWN'; + // Fallback to default consent block + consentObj[consentKey] = defaultConsentBlock[consentKey]; } - } else { - consent.adPersonalization = 'UNSPECIFIED'; - } - return consent; + }); + + return consentObj; }; -module.exports = { populateConsentForGoogleDestinations }; +/** + * Populates the consent object based on the provided configuration and consent mapping. + * @param {*} consentConfigMap + * @param {*} eventLevelConsent + * @returns + */ +const finaliseAnalyticsConsents = (consentConfigMap, eventLevelConsent = {}) => { + const consentObj = {}; + // Iterate through each key in consentConfigMap to set the consent + Object.keys(consentConfigMap).forEach((configKey) => { + const consentKey = consentConfigMap[configKey]; // e.g., 'ad_user_data' + + // Set consent only if valid + if ( + eventLevelConsent && + eventLevelConsent.hasOwnProperty(consentKey) && + GA4_ALLOWED_CONSENT_STATUS.includes(eventLevelConsent[consentKey]) + ) { + consentObj[consentKey] = eventLevelConsent[consentKey]; + } + }); + + return consentObj; +}; + +module.exports = { + populateConsentFromConfig, + UNSPECIFIED_CONSENT, + UNKNOWN_CONSENT, + GOOGLE_ALLOWED_CONSENT_STATUS, + finaliseConsent, + finaliseAnalyticsConsents, +}; diff --git a/src/v0/util/googleUtils/index.test.js b/src/v0/util/googleUtils/index.test.js index 9d1aa5e51a..76ec624311 100644 --- a/src/v0/util/googleUtils/index.test.js +++ b/src/v0/util/googleUtils/index.test.js @@ -1,8 +1,16 @@ -const { populateConsentForGoogleDestinations } = require('./index'); +const { + finaliseConsent, + populateConsentFromConfig, + finaliseAnalyticsConsents, +} = require('./index'); -describe('unit test for populateConsentForGoogleDestinations', () => { +describe('unit test for populateConsentFromConfig', () => { + const consentConfigMap = { + personalizationConsent: 'adPersonalization', + userDataConsent: 'adUserData', + }; it('should return an UNSPECIFIED object when no properties are provided', () => { - const result = populateConsentForGoogleDestinations({}); + const result = populateConsentFromConfig({}, consentConfigMap); expect(result).toEqual({ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', @@ -11,18 +19,18 @@ describe('unit test for populateConsentForGoogleDestinations', () => { it('should set adUserData property of consent object when userDataConsent property is provided and its value is one of the allowed consent statuses', () => { const properties = { userDataConsent: 'GRANTED' }; - const result = populateConsentForGoogleDestinations(properties); + const result = populateConsentFromConfig(properties, consentConfigMap); expect(result).toEqual({ adUserData: 'GRANTED', adPersonalization: 'UNSPECIFIED' }); }); it('should set adPersonalization property of consent object when personalizationConsent property is provided and its value is one of the allowed consent statuses', () => { const properties = { personalizationConsent: 'DENIED' }; - const result = populateConsentForGoogleDestinations(properties); + const result = populateConsentFromConfig(properties, consentConfigMap); expect(result).toEqual({ adPersonalization: 'DENIED', adUserData: 'UNSPECIFIED' }); }); it('should return an UNSPECIFIED object when properties parameter is not provided', () => { - const result = populateConsentForGoogleDestinations(); + const result = populateConsentFromConfig(undefined, consentConfigMap); expect(result).toEqual({ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', @@ -30,7 +38,7 @@ describe('unit test for populateConsentForGoogleDestinations', () => { }); it('should return an UNSPECIFIED object when properties parameter is null', () => { - const result = populateConsentForGoogleDestinations(null); + const result = populateConsentFromConfig(null, consentConfigMap); expect(result).toEqual({ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', @@ -38,7 +46,7 @@ describe('unit test for populateConsentForGoogleDestinations', () => { }); it('should return an UNSPECIFIED object when properties parameter is an UNSPECIFIED object', () => { - const result = populateConsentForGoogleDestinations({}); + const result = populateConsentFromConfig({}, consentConfigMap); expect(result).toEqual({ adPersonalization: 'UNSPECIFIED', adUserData: 'UNSPECIFIED', @@ -46,13 +54,242 @@ describe('unit test for populateConsentForGoogleDestinations', () => { }); it('should return UNKNOWN when properties parameter contains adUserData and adPersonalization with non-allowed values', () => { - const result = populateConsentForGoogleDestinations({ - userDataConsent: 'RANDOM', - personalizationConsent: 'RANDOM', + const result = populateConsentFromConfig( + { + userDataConsent: 'RANDOM', + personalizationConsent: 'RANDOM', + }, + consentConfigMap, + ); + expect(result).toEqual({ + adPersonalization: 'UNKNOWN', + adUserData: 'UNKNOWN', }); + }); +}); + +describe('finaliseConsent', () => { + const consentConfigMap = { + personalizationConsent: 'adPersonalization', + userDataConsent: 'adUserData', + }; + // Returns an object containing consent information. + it('should return an object containing consent information when eventLevelConsent, destConfig, and destinationAllowedConsentKeys are provided', () => { + const eventLevelConsent = { + adUserData: 'GRANTED', + adPersonalization: 'DENIED', + }; + const destConfig = { + userDataConsent: 'UNKNOWN', + personalizationConsent: 'GRANTED', + }; + + const result = finaliseConsent(consentConfigMap, eventLevelConsent, destConfig); + expect(result).toEqual({ + adUserData: 'GRANTED', + adPersonalization: 'DENIED', + }); + }); + + it('should return an object containing consent information from destConfig when evenLevelConsent is empty object', () => { + const eventLevelConsent = {}; // for store conversion we will use this + const destConfig = { + userDataConsent: 'UNKNOWN', + personalizationConsent: 'GRANTED', + }; + + const result = finaliseConsent(consentConfigMap, eventLevelConsent, destConfig); + + expect(result).toEqual({ + adUserData: 'UNKNOWN', + adPersonalization: 'GRANTED', + }); + }); + + // If destConfig is not provided, it does not return UNSPECIFIED_CONSENT. + it('should not return UNSPECIFIED_CONSENT when destConfig is not provided but event level consent is provided', () => { + const eventLevelConsent = { + adUserData: 'GRANTED', + adPersonalization: 'DENIED', + }; + const result = finaliseConsent(consentConfigMap, eventLevelConsent, undefined); + + // Assert + expect(result).toEqual({ + adUserData: 'GRANTED', + adPersonalization: 'DENIED', + }); + }); + + it('should return UNSPECIFIED_CONSENT when both destConfig and event level consent is not provided', () => { + const result = finaliseConsent(consentConfigMap, undefined, undefined); + + // Assert + expect(result).toEqual({ + adUserData: 'UNSPECIFIED', + adPersonalization: 'UNSPECIFIED', + }); + }); + + it('should return UNKWOWN_CONSENT when destConfig is provided with wrong consent value', () => { + const destConfig = { + userDataConsent: 'UNKNOWN', + personalizationConsent: 'WRONG CONSENT', + }; + + const result = finaliseConsent(consentConfigMap, undefined, destConfig); + + expect(result).toEqual({ + adUserData: 'UNKNOWN', adPersonalization: 'UNKNOWN', + }); + }); + + it('should return UNKWOWN_CONSENT when destConfig is provided with wrong consent value', () => { + const destConfig = { + userDataConsent: 'UNKNOWN', + personalizationConsent: 'WRONG CONSENT', + }; + + const result = finaliseConsent(consentConfigMap, undefined, destConfig); + + expect(result).toEqual({ + adPersonalization: 'UNKNOWN', + adUserData: 'UNKNOWN', + }); + }); + + it('should return consent block with appropriate fields and values from destConfig', () => { + const consentConfigMap = { + personalizationConsent: 'newKey1', + userDataConsent: 'newKey2', + }; + const destConfig = { + userDataConsent: 'GRANTED', + personalizationConsent: 'GRANTED', + }; + + const result = finaliseConsent(consentConfigMap, undefined, destConfig); + + expect(result).toEqual({ + newKey1: 'GRANTED', + newKey2: 'GRANTED', + }); + }); + + it('should return consent block with appropriate fields from consentConfigMap and values from eventLevel consent', () => { + const consentConfigMap = { + personalizationConsent: 'newKey1', + userDataConsent: 'newKey2', + }; + const destConfig = { + userDataConsent: 'GRANTED', + personalizationConsent: 'GRANTED', + }; + + const eventLevelConsent = { + newKey1: 'UNKNOWN', + newKey2: 'UNSPECIFIED', + }; + + const result = finaliseConsent(consentConfigMap, eventLevelConsent, destConfig); + + expect(result).toEqual({ + newKey1: 'UNKNOWN', + newKey2: 'UNSPECIFIED', + }); + }); + + it('consentConfig and eventLevelConsent should have parity, also the values should be within allowed values otherwise UNKNOWN is returned ', () => { + const consentConfigMap = { + personalizationConsent: 'newKey1', + userDataConsent: 'newKey2', + }; + const destConfig = { + userDataConsent: 'GRANTED', + personalizationConsent: 'GRANTED', + }; + + const eventLevelConsent = { adUserData: 'UNKNOWN', + adPersonalization: 'UNSPECIFIED', + }; + + const result = finaliseConsent(consentConfigMap, eventLevelConsent, destConfig); + + expect(result).toEqual({ + newKey1: 'GRANTED', + newKey2: 'GRANTED', + }); + }); + + it('consentConfig and eventLevelConsent should have parity, otherwise it will take values from destConfig ', () => { + const consentConfigMap = { + personalizationConsent: 'newKey1', + userDataConsent: 'newKey2', + }; + const destConfig = { + userDataConsent: 'GRANTED', + personalizationConsent: 'GRANTED', + }; + + const eventLevelConsent = { + newKey1: 'DENIED', + newKey2: 'RANDOM', + }; + + const result = finaliseConsent(consentConfigMap, eventLevelConsent, destConfig); + + expect(result).toEqual({ + newKey1: 'DENIED', + newKey2: 'UNKNOWN', + }); + }); +}); + +describe('unit test for finaliseAnalyticsConsents', () => { + const consentConfigMap = { + personalizationConsent: 'ad_personalization', + userDataConsent: 'ad_user_data', + }; + it('Should return an empty object when no valid consents are provided', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, {}); + expect(result).toEqual({}); + }); + + it('Should set ad_user_data property of consent object when userDataConsent property is provided and its value is one of the allowed consent statuses', () => { + const properties = { ad_user_data: 'GRANTED' }; + const result = finaliseAnalyticsConsents(consentConfigMap, properties); + expect(result).toEqual({ ad_user_data: 'GRANTED' }); + }); + + it('Should set ad_personalization property of consent object when personalizationConsent property is provided and its value is one of the allowed consent statuses', () => { + const properties = { ad_personalization: 'DENIED' }; + const result = finaliseAnalyticsConsents(consentConfigMap, properties); + expect(result).toEqual({ ad_personalization: 'DENIED' }); + }); + + it('Should return an empty object when properties parameter is not provided', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, undefined); + expect(result).toEqual({}); + }); + + it('Should return an empty object when properties parameter is null', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, null); + expect(result).toEqual({}); + }); + + it('Should return an empty object when properties parameter is an UNSPECIFIED object', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, {}); + expect(result).toEqual({}); + }); + + it('should return empty object when properties parameter contains ad_user_data and ad_personalization with non-allowed values', () => { + const result = finaliseAnalyticsConsents(consentConfigMap, { + userDataConsent: 'RANDOM', + personalizationConsent: 'RANDOM', }); + expect(result).toEqual({}); }); }); diff --git a/src/v1/destinations/bloomreach/networkHandler.js b/src/v1/destinations/bloomreach/networkHandler.js new file mode 100644 index 0000000000..a3c17a167b --- /dev/null +++ b/src/v1/destinations/bloomreach/networkHandler.js @@ -0,0 +1,83 @@ +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { proxyRequest, prepareProxyRequest } = require('../../../adapters/network'); +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); +const tags = require('../../../v0/util/tags'); + +// { +// "results": [ +// { +// "success": true +// }, +// { +// "success": false, +// "errors": [ +// "At least one id should be specified." +// ] +// } +// ], +// "start_time": 1710750816.8504393, +// "end_time": 1710750816.8518236, +// "success": true +// } +const checkIfEventIsAbortableAndExtractErrorMessage = (element) => { + if (element.success) { + return { isAbortable: false, errorMsg: '' }; + } + + const errorMsg = element.errors.join(', '); + return { isAbortable: true, errorMsg }; +}; + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + + const message = '[BLOOMREACH Response V1 Handler] - Request Processed Successfully'; + const responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + if (isHttpStatusSuccess(status)) { + // check for Partial Event failures and Successes + const { results } = response; + results.forEach((event, idx) => { + const proxyOutput = { + statusCode: 200, + metadata: rudderJobMetadata[idx], + error: 'success', + }; + // update status of partial event if abortable + const { isAbortable, errorMsg } = checkIfEventIsAbortableAndExtractErrorMessage(event); + if (isAbortable) { + proxyOutput.statusCode = 400; + proxyOutput.error = errorMsg; + } + responseWithIndividualEvents.push(proxyOutput); + }); + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + throw new TransformerProxyError( + `BLOOMREACH: Error encountered in transformer proxy V1`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + '', + responseWithIndividualEvents, + ); +}; +function networkHandler() { + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.prepareProxy = prepareProxyRequest; + this.responseHandler = responseHandler; +} +module.exports = { networkHandler }; diff --git a/src/v1/destinations/linkedin_ads/networkHandler.js b/src/v1/destinations/linkedin_ads/networkHandler.js new file mode 100644 index 0000000000..8219e18fcb --- /dev/null +++ b/src/v1/destinations/linkedin_ads/networkHandler.js @@ -0,0 +1,112 @@ +const lodash = require('lodash'); +const { TransformerProxyError } = require('../../../v0/util/errorTypes'); +const { prepareProxyRequest, proxyRequest } = require('../../../adapters/network'); +const { isHttpStatusSuccess } = require('../../../v0/util/index'); + +const { + processAxiosResponse, + getDynamicErrorType, +} = require('../../../adapters/utils/networkUtils'); +const tags = require('../../../v0/util/tags'); +const { + constructPartialStatus, + createResponseArray, + getAuthErrCategoryFromStCode, +} = require('../../../cdk/v2/destinations/linkedin_ads/utils'); + +// eslint-disable-next-line consistent-return +// ref : +// 1) https://learn.microsoft.com/en-us/linkedin/shared/api-guide/concepts/error-handling +// 2) https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversions-api?view=li-lms-2024-02&tabs=http#api-error-details +// statusCode : 422 we have found by trial and error, not documented in their doc + +const responseHandler = (responseParams) => { + const { destinationResponse, rudderJobMetadata } = responseParams; + const message = `[LINKEDIN_CONVERSION_API Response V1 Handler] - Request Processed Successfully`; + let responseWithIndividualEvents = []; + const { response, status } = destinationResponse; + + // even if a single event is unsuccessful, the entire batch will fail, we will filter that event out and retry others + if (!isHttpStatusSuccess(status)) { + const errorMessage = response.message || 'unknown error format'; + responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ + statusCode: status, + metadata, + error: errorMessage, + })); + if (status === 401 || status === 403) { + const finalStatus = status === 401 && response.code !== 'REVOKED_ACCESS_TOKEN' ? 500 : 400; + const finalMessage = + status === 401 + ? 'Invalid or expired access token. Retrying' + : 'Lack of permissions to perform the operation. Aborting'; + throw new TransformerProxyError( + `LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. ${finalMessage}`, + finalStatus, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(finalStatus), + }, + destinationResponse, + getAuthErrCategoryFromStCode(destinationResponse), + responseWithIndividualEvents, + ); + } + // if the status is 422, we need to parse the error message and construct the response array + if (status === 422) { + const destPartialStatus = constructPartialStatus(response?.message); + // if the error message is not in the expected format, we will abort all of the events + if (!destPartialStatus || lodash.isEmpty(destPartialStatus)) { + throw new TransformerProxyError( + `LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. Error parsing error message`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); + } + responseWithIndividualEvents = [...createResponseArray(rudderJobMetadata, destPartialStatus)]; + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; + } + throw new TransformerProxyError( + `LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. ${errorMessage}`, + status, + { + [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(status), + }, + destinationResponse, + getAuthErrCategoryFromStCode(status), + responseWithIndividualEvents, + ); + } + + // otherwise all events are successful + responseWithIndividualEvents = rudderJobMetadata.map((metadata) => ({ + statusCode: 200, + metadata, + error: 'success', + })); + + return { + status, + message, + destinationResponse, + response: responseWithIndividualEvents, + }; +}; + +function networkHandler() { + this.prepareProxy = prepareProxyRequest; + this.proxy = proxyRequest; + this.processAxiosResponse = processAxiosResponse; + this.responseHandler = responseHandler; +} + +module.exports = { networkHandler }; diff --git a/src/warehouse/index.js b/src/warehouse/index.js index b3d1c5e4bc..f62f02af79 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -154,7 +154,7 @@ function setDataFromColumnMappingAndComputeColumnTypes( const columnName = utils.safeColumnName(options, key); // do not set column if val is null/empty/object if (typeof val === 'object' || isBlank(val)) { - // delete in output and columnTypes, so as to remove if we user + // delete in output and columnTypes, to remove if the user // has set property with same name // eslint-disable-next-line no-param-reassign delete output[columnName]; @@ -565,8 +565,10 @@ function processWarehouseMessage(message, options) { : {}; const responses = []; const eventType = message.type?.toLowerCase(); - const skipTracksTable = options.integrationOptions.skipTracksTable || false; - const skipUsersTable = options.integrationOptions.skipUsersTable || false; + const skipTracksTable = + options.destConfig?.skipTracksTable || options.integrationOptions.skipTracksTable || false; + const skipUsersTable = + options.destConfig?.skipUsersTable || options.integrationOptions.skipUsersTable || false; const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; diff --git a/test/__tests__/data/warehouse/dest_config_scenarios.js b/test/__tests__/data/warehouse/dest_config_scenarios.js new file mode 100644 index 0000000000..0e290b7153 --- /dev/null +++ b/test/__tests__/data/warehouse/dest_config_scenarios.js @@ -0,0 +1,223 @@ +const _ = require("lodash"); + +const trackMessage = { + destination: { Config: {} }, + message: { + type: "track", + messageId: "my-track-message-id-1", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33", + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + os: { + name: "", + version: "" + }, + screen: { density: 2 }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + userAgent: "Mozilla/5.0 Chrome/79.0.3945.117 Safari/537.36" + }, + event: "groups", + integrations: { All: true }, + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { currency: "USD" }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30" + }, + request: { query: { whSchemaVersion: "v1" } } +}; + +const identifyMessage = { + destination: { Config: {} }, + message: { + type: "identify", + messageId: "my-identify-message-id-1", + sentAt: "2021-01-03T17:02:53.195Z", + userId: "user123", + channel: "web", + integrations: { All: true }, + context: { + os: { + "name": "android", + "version": "1.12.3" + }, + app: { + name: "RudderLabs JavaScript SDK", + build: "1.0.0", + version: "1.1.11", + namespace: "com.rudderlabs.javascript" + }, + traits: { + email: "user123@email.com", + phone: "+917836362334", + userId: "user123" + }, + locale: "en-US", + device: { + token: "token", + id: "id", + type: "ios" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.1.11" + }, + userAgent: "Gecko/20100101 Firefox/84.0" + }, + rudderId: "8f8fa6b5-8e24-489c-8e22-61f23f2e364f", + anonymousId: "97c46c81-3140-456d-b2a9-690d70aaca35", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + timestamp: "2020-01-24T11:59:02.403+05:30" + }, + request: { query: { whSchemaVersion: "v1" } } +}; + +const scenarios = [ + { + name: "track event is not skipped when options are not provided", + skipUsersTable: null, + skipTracksTable: null, + event: _.cloneDeep(trackMessage), + expected: [ + { + id: "my-track-message-id-1", + table: "tracks" + }, + { + id: "my-track-message-id-1", + table: "_groups" + } + ] + }, + { + name: "track event is not skipped when skipTracksTable is false", + skipUsersTable: null, + skipTracksTable: false, + event: _.cloneDeep(trackMessage), + expected: [ + { + id: "my-track-message-id-1", + table: "tracks" + }, + { + id: "my-track-message-id-1", + table: "_groups" + } + ] + }, + { + name: "track event is skipped when skipTracksTable is true", + skipUsersTable: null, + skipTracksTable: true, + event: _.cloneDeep(trackMessage), + expected: [ + { + id: "my-track-message-id-1", + table: "_groups" + } + ] + }, + { + name: "track event is not affected by skipUsersTable", + skipUsersTable: true, + skipTracksTable: null, + event: _.cloneDeep(trackMessage), + expected: [ + { + id: "my-track-message-id-1", + table: "tracks" + }, + { + id: "my-track-message-id-1", + table: "_groups" + } + ] + }, + { + name: "user event is not skipped when options are not provided", + skipUsersTable: null, + skipTracksTable: null, + event: _.cloneDeep(identifyMessage), + expected: [ + { + id: "my-identify-message-id-1", + table: "identifies" + }, + { + id: "user123", + table: "users" + } + ] + }, + { + name: "user event is not skipped when skipUsersTable is false", + skipUsersTable: false, + skipTracksTable: null, + event: _.cloneDeep(identifyMessage), + expected: [ + { + id: "my-identify-message-id-1", + table: "identifies" + }, + { + id: "user123", + table: "users" + } + ] + }, + { + name: "user event is skipped when skipUsersTable is true", + skipUsersTable: true, + skipTracksTable: null, + event: _.cloneDeep(identifyMessage), + expected: [ + { + id: "my-identify-message-id-1", + table: "identifies" + } + ] + }, + { + name: "user event is not affected by skipTracksTable", + skipUsersTable: null, + skipTracksTable: true, + event: _.cloneDeep(identifyMessage), + expected: [ + { + id: "my-identify-message-id-1", + table: "identifies" + }, + { + id: "user123", + table: "users" + } + ] + }, +]; + +module.exports = { + scenarios: function() { + return _.cloneDeep(scenarios); + } +}; \ No newline at end of file diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 772e59e65a..2c89120686 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -8,6 +8,7 @@ const { opOutput } = require(`./data/warehouse/integration_options_events.js`); const { names } = require(`./data/warehouse/names.js`); +const destConfig = require(`./data/warehouse/dest_config_scenarios.js`); const { largeNoOfColumnsevent } = require(`./data/warehouse/event_columns_length`); @@ -1009,6 +1010,28 @@ describe("Add receivedAt for events missing it", () => { }); describe("Integration options", () => { + describe("Destination config options", () => { + destConfig.scenarios().forEach(scenario => { + it(scenario.name, () => { + if (scenario.skipUsersTable !== null) { + scenario.event.destination.Config.skipUsersTable = scenario.skipUsersTable + } + if (scenario.skipTracksTable !== null) { + scenario.event.destination.Config.skipTracksTable = scenario.skipTracksTable + } + + transformers.forEach((transformer, index) => { + const received = transformer.process(scenario.event); + expect(received).toHaveLength(scenario.expected.length); + for (const i in received) { + const evt = received[i]; + expect(evt.data.id ? evt.data.id : evt.data.ID).toEqual(scenario.expected[i].id); + expect(evt.metadata.table.toLowerCase()).toEqual(scenario.expected[i].table); + } + }); + }); + }); + }); describe("track", () => { it("should generate two events for every track call", () => { const i = opInput("track"); diff --git a/test/integrations/destinations/bloomreach/common.ts b/test/integrations/destinations/bloomreach/common.ts new file mode 100644 index 0000000000..798e744cbc --- /dev/null +++ b/test/integrations/destinations/bloomreach/common.ts @@ -0,0 +1,99 @@ +import { Destination } from '../../../../src/types'; + +const destType = 'bloomreach'; +const destTypeInUpperCase = 'BLOOMREACH'; +const displayName = 'bloomreach'; +const channel = 'web'; +const destination: Destination = { + Config: { + apiBaseUrl: 'https://demoapp-api.bloomreach.com', + apiKey: 'test-api-key', + apiSecret: 'test-api-secret', + projectToken: 'test-project-token', + hardID: 'registered', + softID: 'cookie', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; + +const traits = { + email: 'test@example.com', + firstName: 'John', + lastName: 'Doe', + phone: '1234567890', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, +}; + +const properties = { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + category: 'Games', + name: 'Cones of Dunshire', + brand: 'Wyatt Games', + variant: 'expansion pack', + price: 49.99, + quantity: 5, + coupon: 'PREORDER15', + currency: 'USD', + position: 1, + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.webp', + key1: 'value1', +}; +const endpoint = 'https://demoapp-api.bloomreach.com/track/v2/projects/test-project-token/batch'; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const proxyV1RetryableErrorStatTags = { + ...RouterInstrumentationErrorStatTags, + errorCategory: 'network', + errorType: 'retryable', + feature: 'dataDelivery', + implementation: 'native', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1hcGkta2V5OnRlc3QtYXBpLXNlY3JldA==', +}; + +export { + destType, + channel, + destination, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + traits, + headers, + properties, + endpoint, + proxyV1RetryableErrorStatTags, +}; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/business.ts b/test/integrations/destinations/bloomreach/dataDelivery/business.ts new file mode 100644 index 0000000000..9e71b7a2fd --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/business.ts @@ -0,0 +1,195 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload, generateMetadata } from '../../../testUtils'; +import { destType, headers, properties, endpoint } from '../common'; + +const customerProperties = { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, +}; + +const metadataArray = [generateMetadata(1), generateMetadata(2)]; + +// https://documentation.bloomreach.com/engagement/reference/tips-and-best-practices +export const businessProxyV1: ProxyV1TestData[] = [ + { + id: 'bloomreach_v1_business_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid request - where the destination responds with 200 with error for request 2 in a batch', + successCriteria: 'Should return 200 with partial failures within the response payload', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: { + commands: [ + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: customerProperties, + }, + }, + { + name: 'customers', + data: { + customer_ids: {}, + }, + }, + ], + }, + endpoint, + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + results: [ + { + success: true, + }, + { + success: false, + errors: ['At least one id should be specified.'], + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 400, + metadata: generateMetadata(2), + error: 'At least one id should be specified.', + }, + ], + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_business_scenario_2', + name: destType, + description: + '[Proxy v1 API] :: Test for a valid request - where the destination responds with 200 without any error', + successCriteria: 'Should return 200 with no error with destination response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + headers, + params: {}, + JSON: { + commands: [ + { + name: 'customers/events', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + timestamp: 1709566376, + properties, + event_type: 'test_event', + }, + }, + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: customerProperties, + }, + }, + ], + }, + endpoint, + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: '[BLOOMREACH Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + results: [ + { + success: true, + }, + { + success: true, + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + { + statusCode: 200, + metadata: generateMetadata(2), + error: 'success', + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/data.ts b/test/integrations/destinations/bloomreach/dataDelivery/data.ts new file mode 100644 index 0000000000..5099eafce7 --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/data.ts @@ -0,0 +1,3 @@ +import { businessProxyV1 } from './business'; +import { otherProxyV1 } from './other'; +export const data = [...businessProxyV1, ...otherProxyV1]; diff --git a/test/integrations/destinations/bloomreach/dataDelivery/other.ts b/test/integrations/destinations/bloomreach/dataDelivery/other.ts new file mode 100644 index 0000000000..f0dd9cc09a --- /dev/null +++ b/test/integrations/destinations/bloomreach/dataDelivery/other.ts @@ -0,0 +1,212 @@ +import { ProxyV1TestData } from '../../../testTypes'; +import { generateProxyV1Payload, generateMetadata } from '../../../testUtils'; +import { destType, proxyV1RetryableErrorStatTags } from '../common'; + +const metadataArray = [generateMetadata(1)]; + +// https://documentation.bloomreach.com/engagement/reference/tips-and-best-practices +export const otherProxyV1: ProxyV1TestData[] = [ + { + id: 'bloomreach_v1_other_scenario_1', + name: destType, + description: + '[Proxy v1 API] :: Scenario for testing Service Unavailable error from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_service_not_available', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: + '{"error":{"message":"Service Unavailable","description":"The server is currently unable to handle the request due to temporary overloading or maintenance of the server. Please try again later."}}', + statusCode: 503, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 503, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_2', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing Internal Server error from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_internal_server_error', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '"Internal Server Error"', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_3', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing Gateway Time Out error from destination', + successCriteria: 'Should return 504 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_gateway_time_out', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '"Gateway Timeout"', + statusCode: 504, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 504, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_4', + name: destType, + description: '[Proxy v1 API] :: Scenario for testing null response from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_null_response', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '""', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, + { + id: 'bloomreach_v1_other_scenario_5', + name: destType, + description: + '[Proxy v1 API] :: Scenario for testing null and no status response from destination', + successCriteria: 'Should return 500 status code with error message', + scenario: 'Framework', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload( + { + endpoint: 'https://random_test_url/test_for_null_and_no_status', + }, + metadataArray, + ), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + response: [ + { + error: '""', + statusCode: 500, + metadata: generateMetadata(1), + }, + ], + statTags: proxyV1RetryableErrorStatTags, + message: 'BLOOMREACH: Error encountered in transformer proxy V1', + status: 500, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/mocks.ts b/test/integrations/destinations/bloomreach/mocks.ts new file mode 100644 index 0000000000..ba3b22b52a --- /dev/null +++ b/test/integrations/destinations/bloomreach/mocks.ts @@ -0,0 +1,5 @@ +import * as config from '../../../../src/cdk/v2/destinations/bloomreach/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 3 as typeof config.MAX_BATCH_SIZE); +}; diff --git a/test/integrations/destinations/bloomreach/network.ts b/test/integrations/destinations/bloomreach/network.ts new file mode 100644 index 0000000000..b20ff881b8 --- /dev/null +++ b/test/integrations/destinations/bloomreach/network.ts @@ -0,0 +1,124 @@ +import { destType, headers, properties, endpoint } from './common'; + +export const networkCallsData = [ + { + httpReq: { + url: endpoint, + data: { + commands: [ + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + }, + }, + { + name: 'customers', + data: { + customer_ids: {}, + }, + }, + ], + }, + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: { + results: [ + { + success: true, + }, + { + success: false, + errors: ['At least one id should be specified.'], + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + statusText: 'Ok', + }, + }, + { + httpReq: { + url: endpoint, + data: { + commands: [ + { + name: 'customers/events', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + timestamp: 1709566376, + properties, + event_type: 'test_event', + }, + }, + { + name: 'customers', + data: { + customer_ids: { + cookie: '97c46c81-3140-456d-b2a9-690d70aaca35', + }, + update_timestamp: 1709405952, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + }, + }, + ], + }, + params: { destination: destType }, + headers, + method: 'POST', + }, + httpRes: { + data: { + results: [ + { + success: true, + }, + { + success: true, + }, + ], + start_time: 1710771351.9885373, + end_time: 1710771351.9891083, + success: true, + }, + status: 200, + statusText: 'Ok', + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/data.ts b/test/integrations/destinations/bloomreach/processor/data.ts new file mode 100644 index 0000000000..a3633ad0dd --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/data.ts @@ -0,0 +1,5 @@ +import { validation } from './validation'; +import { identify } from './identify'; +import { track } from './track'; +import { page } from './page'; +export const data = [...identify, ...track, ...page, ...validation]; diff --git a/test/integrations/destinations/bloomreach/processor/identify.ts b/test/integrations/destinations/bloomreach/processor/identify.ts new file mode 100644 index 0000000000..2a79cb57e3 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/identify.ts @@ -0,0 +1,156 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, traits, headers, endpoint } from '../common'; + +export const identify: ProcessorTestData[] = [ + { + id: 'bloomreach-identify-test-1', + name: destType, + description: 'Identify call to create/update customer properties', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { registered: 'userId123', cookie: 'anonId123' }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-identify-test-2', + name: destType, + description: 'Identify call with multiple hard and soft identifiers using integration object', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain multiple hard and soft identifiers and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + integrations: { + All: true, + bloomreach: { + hardID: { + hardID1: 'value1', + }, + softID: { + google_analytics: 'gaId123', + softID2: 'value2', + }, + }, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { + registered: 'userId123', + cookie: 'anonId123', + hardID1: 'value1', + google_analytics: 'gaId123', + softID2: 'value2', + }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/page.ts b/test/integrations/destinations/bloomreach/processor/page.ts new file mode 100644 index 0000000000..0c2d27989d --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/page.ts @@ -0,0 +1,72 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, headers, endpoint } from '../common'; + +const properties = { + category: 'Docs', + path: '', + referrer: '', + search: '', + title: '', + url: '', +}; + +export const page: ProcessorTestData[] = [ + { + id: 'bloomreach-page-test-1', + name: destType, + description: 'Page call with category, name', + scenario: 'Framework+Business', + successCriteria: + 'Response should contain event_name = "Viewed {{ category }} {{ name }} Page" and properties and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'page', + anonymousId: 'anonId123', + name: 'Integration', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Viewed Docs Integration Page', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/track.ts b/test/integrations/destinations/bloomreach/processor/track.ts new file mode 100644 index 0000000000..a369f508b2 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/track.ts @@ -0,0 +1,173 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, transformResultBuilder } from '../../../testUtils'; +import { destType, destination, headers, properties, endpoint } from '../common'; + +export const track: ProcessorTestData[] = [ + { + id: 'bloomreach-track-test-1', + name: destType, + description: 'Track call with anonymous user', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + event: 'Product Viewed', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Product Viewed', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-track-test-2', + name: destType, + description: 'Track call with known user', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Product Added', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { registered: 'userId123', cookie: 'anonId123' }, + properties, + timestamp: 1709566376, + event_type: 'Product Added', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'bloomreach-track-test-3', + name: destType, + description: 'Track call with no properties', + scenario: 'Framework+Business', + successCriteria: 'Response should contain all the mapping and status code should be 200', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + event: 'test_event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint, + headers, + JSON: { + data: { + customer_ids: { cookie: 'anonId123' }, + timestamp: 1709566376, + event_type: 'test_event', + }, + name: 'customers/events', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/processor/validation.ts b/test/integrations/destinations/bloomreach/processor/validation.ts new file mode 100644 index 0000000000..ff959d74c6 --- /dev/null +++ b/test/integrations/destinations/bloomreach/processor/validation.ts @@ -0,0 +1,131 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata } from '../../../testUtils'; +import { destType, destination, processorInstrumentationErrorStatTags } from '../common'; + +export const validation: ProcessorTestData[] = [ + { + id: 'bloomreach-validation-test-1', + name: destType, + description: 'Missing userId and anonymousId', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Either one of userId or anonymousId is required. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Either one of userId or anonymousId is required. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'bloomreach-validation-test-2', + name: destType, + description: 'Unsupported message type -> group', + scenario: 'Framework', + successCriteria: 'Instrumentation Error for Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'group', + userId: 'userId123', + channel: 'mobile', + anonymousId: 'anon_123', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'bloomreach-validation-test-3', + name: destType, + description: 'Missing required field -> timestamp', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'identify', + integrations: { + All: true, + }, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Timestamp is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Timestamp is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/bloomreach/router/data.ts b/test/integrations/destinations/bloomreach/router/data.ts new file mode 100644 index 0000000000..e99d0cc8cd --- /dev/null +++ b/test/integrations/destinations/bloomreach/router/data.ts @@ -0,0 +1,220 @@ +import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; +import { + destType, + destination, + traits, + properties, + headers, + endpoint, + RouterInstrumentationErrorStatTags, +} from '../common'; + +const routerRequest = { + input: [ + { + message: { + type: 'track', + anonymousId: 'anonId1', + event: 'test_event_1A', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'identify', + anonymousId: 'anonId1', + userId: 'userId1', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, + { + message: { + type: 'track', + anonymousId: 'anonId2', + event: 'test_event_2A', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(3), + destination, + }, + { + message: { + type: 'track', + anonymousId: 'anonId1', + userId: 'userId1', + event: 'test_event_1B', + properties, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(4), + destination, + }, + { + message: { + type: 'identify', + traits, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(5), + destination, + }, + ], + destType, +}; +export const data = [ + { + id: 'bloomreach-router-test-1', + name: destType, + description: 'Basic Router Test to test multiple payloads', + scenario: 'Framework', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + JSON: { + commands: [ + { + data: { + customer_ids: { cookie: 'anonId1' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_1A', + }, + name: 'customers/events', + }, + { + data: { + customer_ids: { + registered: 'userId1', + cookie: 'anonId1', + }, + properties: { + email: 'test@example.com', + first_name: 'John', + last_name: 'Doe', + phone: '1234567890', + city: 'New York', + country: 'USA', + address: { + city: 'New York', + country: 'USA', + pinCode: '123456', + }, + }, + update_timestamp: 1709566376, + }, + name: 'customers', + }, + { + data: { + customer_ids: { cookie: 'anonId2' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_2A', + }, + name: 'customers/events', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params: {}, + body: { + JSON: { + commands: [ + { + data: { + customer_ids: { registered: 'userId1', cookie: 'anonId1' }, + properties, + timestamp: 1709566376, + event_type: 'test_event_1B', + }, + name: 'customers/events', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(5)], + batched: false, + statusCode: 400, + error: 'Either one of userId or anonymousId is required. Aborting', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/fullstory/processor/data.ts b/test/integrations/destinations/fullstory/processor/data.ts index d206b4a84f..9c8d29c7e8 100644 --- a/test/integrations/destinations/fullstory/processor/data.ts +++ b/test/integrations/destinations/fullstory/processor/data.ts @@ -149,7 +149,7 @@ export const data = [ id: 's001', }, user: { - id: 'u001', + uid: 'u001', }, }, JSON_ARRAY: {}, diff --git a/test/integrations/destinations/ga4/processor/data.ts b/test/integrations/destinations/ga4/processor/data.ts index f96ca9e74a..4465ec9e2c 100644 --- a/test/integrations/destinations/ga4/processor/data.ts +++ b/test/integrations/destinations/ga4/processor/data.ts @@ -14900,4 +14900,313 @@ export const data = [ }, mockFns: defaultMockFns, }, + { + name: 'ga4', + description: '(gtag) send consents setting to ga4 with login event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-26T05:17:09Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + adTrackingEnabled: 'false', + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + externalId: [ + { + type: 'ga4AppInstanceId', + id: 'dummyGA4AppInstanceId', + }, + { + type: 'ga4ClientId', + id: 'client_id', + }, + ], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 'login', + properties: { + method: 'Google', + }, + integrations: { + All: true, + GA4: { + consents: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + }, + }, + sentAt: '2022-04-20T15:20:57Z', + }, + destination: { + Config: { + apiSecret: 'dummyApiSecret', + measurementId: 'G-123456', + firebaseAppId: '', + blockPageViewEvent: false, + typesOfClient: 'gtag', + extendPageViewParams: false, + sendUserId: false, + eventFilteringOption: 'disable', + blacklistedEvents: [ + { + eventName: '', + }, + ], + whitelistedEvents: [ + { + eventName: '', + }, + ], + enableServerSideIdentify: false, + sendLoginSignup: false, + generateLead: false, + }, + Enabled: true, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-123456', + }, + body: { + JSON: { + client_id: 'client_id', + consent: { + ad_personalization: 'GRANTED', + ad_user_data: 'GRANTED', + }, + timestamp_micros: 1650950229000000, + non_personalized_ads: true, + events: [ + { + name: 'login', + params: { + method: 'Google', + engagement_time_msec: 1, + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, + { + name: 'ga4', + description: '(gtag) send consents setting to ga4 with login event', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-26T05:17:09Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + adTrackingEnabled: 'false', + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + ip: '0.0.0.0', + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + externalId: [ + { + type: 'ga4AppInstanceId', + id: 'dummyGA4AppInstanceId', + }, + { + type: 'ga4ClientId', + id: 'client_id', + }, + ], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 'login', + properties: { + method: 'Google', + }, + integrations: { + All: true, + GA4: { + consents: { + ad_personalization: 'NOT_SPECIFIED', + ad_user_data: 'DENIED', + }, + }, + }, + sentAt: '2022-04-20T15:20:57Z', + }, + destination: { + Config: { + apiSecret: 'dummyApiSecret', + measurementId: 'G-123456', + firebaseAppId: '', + blockPageViewEvent: false, + typesOfClient: 'gtag', + extendPageViewParams: false, + sendUserId: false, + eventFilteringOption: 'disable', + blacklistedEvents: [ + { + eventName: '', + }, + ], + whitelistedEvents: [ + { + eventName: '', + }, + ], + enableServerSideIdentify: false, + sendLoginSignup: false, + generateLead: false, + }, + Enabled: true, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://www.google-analytics.com/mp/collect', + headers: { + HOST: 'www.google-analytics.com', + 'Content-Type': 'application/json', + }, + params: { + api_secret: 'dummyApiSecret', + measurement_id: 'G-123456', + }, + body: { + JSON: { + client_id: 'client_id', + consent: { + ad_user_data: 'DENIED', + }, + timestamp_micros: 1650950229000000, + non_personalized_ads: true, + events: [ + { + name: 'login', + params: { + method: 'Google', + engagement_time_msec: 1, + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: defaultMockFns, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts index fbeaf7f250..87aafea0af 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts @@ -235,7 +235,7 @@ export const testScenariosForV0API = [ params: params.param1, JSON: invalidArgumentRequestPayload, endpoint: - 'https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs', }), method: 'POST', }, @@ -253,7 +253,7 @@ export const testScenariosForV0API = [ code: 400, details: [ { - '@type': 'type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure', + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', errors: [ { errorCode: { @@ -309,7 +309,7 @@ export const testScenariosForV0API = [ headers: headers.header1, params: params.param1, JSON: validRequestPayload1, - endpoint: 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', }), method: 'POST', }, @@ -350,7 +350,7 @@ export const testScenariosForV0API = [ params: params.param2, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', }), method: 'POST', }, @@ -400,7 +400,7 @@ export const testScenariosForV0API = [ params: params.param3, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', }), method: 'POST', }, @@ -453,7 +453,7 @@ export const testScenariosForV1API = [ params: params.param1, JSON: invalidArgumentRequestPayload, endpoint: - 'https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs', }, metadataArray, ), @@ -500,7 +500,7 @@ export const testScenariosForV1API = [ params: params.param1, JSON: validRequestPayload1, endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', }, metadataArray, ), @@ -545,7 +545,7 @@ export const testScenariosForV1API = [ params: params.param2, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', }, metadataArray, ), @@ -591,7 +591,7 @@ export const testScenariosForV1API = [ params: params.param3, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', }, metadataArray, ), @@ -637,7 +637,7 @@ export const testScenariosForV1API = [ params: params.param4, JSON: notAllowedToAccessFeatureRequestPayload, endpoint: - 'https://googleads.googleapis.com/v14/customers/1234567893:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/1234567893:uploadClickConversions', }, metadataArray, ), diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts index 15a150d0e5..e14e4109f0 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts @@ -95,7 +95,7 @@ export const v0oauthScenarios = [ request: { body: generateProxyV0Payload({ ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', }), method: 'POST', }, @@ -138,7 +138,7 @@ export const v0oauthScenarios = [ request: { body: generateProxyV0Payload({ ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v14/customers/1234/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs', }), method: 'POST', }, @@ -184,7 +184,7 @@ export const v1oauthScenarios = [ { ...commonRequestParameters, endpoint: - 'https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', }, metadataArray, ), @@ -230,7 +230,7 @@ export const v1oauthScenarios = [ body: generateProxyV1Payload( { ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v14/customers/1234/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs', }, metadataArray, ), diff --git a/test/integrations/destinations/google_adwords_offline_conversions/network.ts b/test/integrations/destinations/google_adwords_offline_conversions/network.ts index 7dc7f97933..4dad9e0d1b 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/network.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/network.ts @@ -1,7 +1,7 @@ export const networkCallsData = [ { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -30,7 +30,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1112223333/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v16/customers/1112223333/googleAds:searchStream', data: { query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'`, }, @@ -63,7 +63,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations', + url: 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations', data: { enable_partial_failure: false, enable_warnings: false, @@ -108,7 +108,7 @@ export const networkCallsData = [ status: 'INVALID_ARGUMENT', details: [ { - '@type': 'type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure', + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', errors: [ { errorCode: { @@ -144,7 +144,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -173,7 +173,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations', + url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations', data: { enable_partial_failure: false, enable_warnings: false, @@ -216,7 +216,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run', + url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run', data: { validate_only: false }, params: { destination: 'google_adwords_offline_conversion' }, headers: { @@ -238,7 +238,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request with invalid authentication credentials', httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/customerid/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -274,7 +274,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request with invalid authentication scopes', httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -307,7 +307,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567890/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v16/customers/1234567890/googleAds:searchStream', data: { query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'`, }, @@ -335,7 +335,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567891/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v16/customers/1234567891/googleAds:searchStream', data: { query: "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", @@ -368,7 +368,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567891/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v16/customers/1234567891/googleAds:searchStream', data: { query: 'SELECT conversion_custom_variable.name FROM conversion_custom_variable' }, headers: { Authorization: 'Bearer abcd1234', @@ -402,7 +402,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + url: 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', data: { conversions: [ { @@ -469,7 +469,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567891:uploadClickConversions', + url: 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', data: { conversions: [ { @@ -530,7 +530,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567893/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v16/customers/1234567893/googleAds:searchStream', data: { query: "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", @@ -563,7 +563,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v14/customers/1234567893:uploadClickConversions', + url: 'https://googleads.googleapis.com/v16/customers/1234567893:uploadClickConversions', data: { conversions: [ { @@ -615,7 +615,7 @@ export const networkCallsData = [ 'Customer is not allowlisted for accessing this feature., at conversions[0].conversion_environment', details: [ { - '@type': 'type.googleapis.com/google.ads.googleads.v14.errors.GoogleAdsFailure', + '@type': 'type.googleapis.com/google.ads.googleads.v16.errors.GoogleAdsFailure', errors: [ { errorCode: { diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index f47deaef67..decb1e58c7 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -176,7 +176,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -263,6 +263,10 @@ export const data = [ conversionValue: 1, currencyCode: 'GBP', orderId: 'PL-123QR', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, }, ], partialFailure: true, @@ -465,7 +469,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -520,6 +524,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gbraid: 'gbraid', wbraid: 'wbraid', externalAttributionData: { @@ -754,7 +762,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -809,6 +817,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gbraid: 'gbraid', wbraid: 'wbraid', externalAttributionData: { @@ -1043,7 +1055,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -1099,6 +1111,10 @@ export const data = [ conversions: [ { callerId: 'callerId', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callStartDateTime: '2022-08-28 15:01:30+05:30', conversionDateTime: '2022-01-01 12:32:45-08:00', conversionValue: 1, @@ -2015,7 +2031,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2072,6 +2088,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gbraid: 'gbraid', wbraid: 'wbraid', externalAttributionData: { @@ -2128,7 +2148,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2185,6 +2205,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callerId: 'callerId', callStartDateTime: '2022-08-28 15:01:30+05:30', conversionDateTime: '2022-01-01 12:32:45-08:00', @@ -2350,7 +2374,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2379,6 +2403,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', userIdentifiers: [ @@ -2546,7 +2574,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2575,6 +2603,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callerId: 'callerId', callStartDateTime: '2022-08-28 15:01:30+05:30', conversionDateTime: '2022-01-01 12:32:45-08:00', @@ -2778,7 +2810,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2832,6 +2864,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callerId: 'callerId', callStartDateTime: '2022-08-28 15:01:30+05:30', conversionDateTime: '2022-09-20 08:50:04+05:30', @@ -2997,7 +3033,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3025,6 +3061,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gclid: 'gclid', conversionDateTime: '2022-01-01 12:32:45-08:00', userIdentifiers: [ @@ -3501,7 +3541,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3532,6 +3572,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, cartData: { items: [ { @@ -3861,7 +3905,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3892,6 +3936,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, cartData: { items: [ { @@ -4049,7 +4097,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4077,6 +4125,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -4382,7 +4434,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4409,6 +4461,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -4577,7 +4633,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4604,6 +4660,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -4775,7 +4835,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4802,6 +4862,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -4935,7 +4999,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4962,6 +5026,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -5091,7 +5159,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5118,6 +5186,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', @@ -5245,7 +5317,323 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': 'logincustomerid', + }, + params: { + event: 'Sign-up - click', + customerId: '1112223333', + }, + body: { + JSON: { + event: '1112223333', + isStoreConversion: true, + createJobPayload: { + job: { + type: 'STORE_SALES_UPLOAD_FIRST_PARTY', + storeSalesMetadata: { + loyaltyFraction: '1', + transaction_upload_fraction: '1', + }, + }, + }, + addConversionPayload: { + operations: { + create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + transaction_attribute: { + store_attribute: { + store_code: 'store code', + }, + transaction_amount_micros: '100000000', + currency_code: 'INR', + transaction_date_time: '2019-10-14 16:45:18+05:30', + }, + userIdentifiers: [{}], + }, + }, + enable_partial_failure: false, + enable_warnings: false, + validate_only: false, + }, + executeJobPayload: { + validate_only: false, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: timestampMock, + }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 26 : store conversion consent mapped from UI config', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + traits: {}, + }, + event: 'Product Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + item_id: 'item id', + merchant_id: 'merchant id', + currency: 'INR', + revenue: '100', + store_code: 'store code', + gclid: 'gclid', + conversionDateTime: '2019-10-14T11:15:18.299Z', + product_id: '123445', + quantity: 123, + }, + integrations: { + All: true, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + isCustomerAllowed: false, + customerId: '111-222-3333', + subAccount: true, + loginCustomerId: 'login-customer-id', + userDataConsent: 'GRANTED', + personalizationConsent: 'DENIED', + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Product Clicked', + to: 'store', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Product Clicked', + to: 'Sign-up - click', + }, + ], + hashUserIdentifier: true, + defaultUserIdentifier: 'phone', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNkgr4t3u4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + headers: { + Authorization: 'Bearer abcd1234', + 'Content-Type': 'application/json', + 'developer-token': 'ijkl91011', + 'login-customer-id': 'logincustomerid', + }, + params: { + event: 'Sign-up - click', + customerId: '1112223333', + }, + body: { + JSON: { + event: '1112223333', + isStoreConversion: true, + createJobPayload: { + job: { + type: 'STORE_SALES_UPLOAD_FIRST_PARTY', + storeSalesMetadata: { + loyaltyFraction: '1', + transaction_upload_fraction: '1', + }, + }, + }, + addConversionPayload: { + operations: { + create: { + consent: { + adPersonalization: 'DENIED', + adUserData: 'GRANTED', + }, + transaction_attribute: { + store_attribute: { + store_code: 'store code', + }, + transaction_amount_micros: '100000000', + currency_code: 'INR', + transaction_date_time: '2019-10-14 16:45:18+05:30', + }, + userIdentifiers: [{}], + }, + }, + enable_partial_failure: false, + enable_warnings: false, + validate_only: false, + }, + executeJobPayload: { + validate_only: false, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + statusCode: 200, + }, + ], + }, + }, + mockFns: timestampMock, + }, + { + name: 'google_adwords_offline_conversions', + description: + 'Test 27 : store conversion consent mapped from UI config even though integration object is present', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + traits: {}, + }, + event: 'Product Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + item_id: 'item id', + merchant_id: 'merchant id', + currency: 'INR', + revenue: '100', + store_code: 'store code', + gclid: 'gclid', + conversionDateTime: '2019-10-14T11:15:18.299Z', + product_id: '123445', + quantity: 123, + }, + integrations: { + google_adwords_offline_conversion: { + consent: { + adUserdata: 'UNSPECIFIED', + adPersonalization: 'GRANTED', + }, + }, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + isCustomerAllowed: false, + customerId: '111-222-3333', + subAccount: true, + loginCustomerId: 'login-customer-id', + userDataConsent: 'GRANTED', + personalizationConsent: 'DENIED', + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Product Clicked', + to: 'store', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Product Clicked', + to: 'Sign-up - click', + }, + ], + hashUserIdentifier: true, + defaultUserIdentifier: 'phone', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNkgr4t3u4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: + 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5272,6 +5660,10 @@ export const data = [ addConversionPayload: { operations: { create: { + consent: { + adPersonalization: 'DENIED', + adUserData: 'GRANTED', + }, transaction_attribute: { store_attribute: { store_code: 'store code', diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index a38980f0e9..596e7550e5 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -487,7 +487,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/7693729833/offlineUserDataJobs', + 'https://googleads.googleapis.com/v16/customers/7693729833/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -511,6 +511,10 @@ export const data = [ operations: [ { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code' }, transaction_amount_micros: '100000000', @@ -528,6 +532,10 @@ export const data = [ }, { create: { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, transaction_attribute: { store_attribute: { store_code: 'store code2' }, transaction_amount_micros: '100000000', @@ -561,7 +569,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/7693729833:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/7693729833:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -589,6 +597,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callerId: '1234', callStartDateTime: '2019-10-14T11:15:18.299Z', conversionDateTime: '2019-10-14 16:45:18+05:30', @@ -673,7 +685,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -722,6 +734,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, gbraid: 'gbraid', wbraid: 'wbraid', externalAttributionData: { @@ -806,7 +822,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v14/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -855,6 +871,10 @@ export const data = [ JSON: { conversions: [ { + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, callerId: 'callerId', callStartDateTime: '2022-08-28 15:01:30+05:30', conversionDateTime: '2022-01-01 12:32:45-08:00', diff --git a/test/integrations/destinations/linkedin_ads/dataDelivery/business.ts b/test/integrations/destinations/linkedin_ads/dataDelivery/business.ts new file mode 100644 index 0000000000..ff4fa4455f --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/dataDelivery/business.ts @@ -0,0 +1,188 @@ +import { generateProxyV1Payload } from '../../../testUtils'; +import { ProxyV1TestData } from '../../../testTypes'; + +export const element = { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '0', + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + }, +}; + +export const wrongFormatElement = { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + userInfo: { + city: 'San Francisco', + }, + }, +}; + +export const testJSONData = { + elements: [{ ...element }], +}; + +export const wrongFormattedTestJSONData = { + elements: [{ ...wrongFormatElement }], +}; + +export const testJSONDataWithDifferentTypeConversion = { + elements: [ + { + ...element, + conversion: 'urn:li:partner:differentConversion', + }, + ], +}; + +export const statTags = { + destType: 'LINKEDIN_ADS', + errorCategory: 'network', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', +}; + +export const metadata = { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, +}; +export const headerBlockWithCorrectAccessToken = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; + +const commonRequestParametersWithWrongElemet = { + headers: headerBlockWithCorrectAccessToken, + JSON: wrongFormattedTestJSONData, +}; + +const commonRequestParametersWithDifferentConversion = { + headers: headerBlockWithCorrectAccessToken, + JSON: testJSONDataWithDifferentTypeConversion, +}; + +export const testScenariosForV1API: ProxyV1TestData[] = [ + { + id: 'linkedin_ads_v1_scenario_1', + name: 'linkedin_ads', + description: 'Event fails due to wrong process followed while creating a conversion', + successCriteria: 'Should return 400 and aborted', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + ...commonRequestParametersWithDifferentConversion, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + message: + "LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + response: [ + { + error: + '{"message":"Incorrect conversions information provided. Conversion\'s method should be CONVERSIONS_API, indices [0] (0-indexed)","status":400}', + statusCode: 400, + metadata, + }, + ], + statTags, + status: 400, + }, + }, + }, + }, + }, + { + id: 'linkedin_ads_v1_scenario_2', + name: 'linkedin_ads', + description: 'Event fails due to wrong format payload sent to linkedin', + successCriteria: 'Should return 400 with appropriate reason of failure', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + ...commonRequestParametersWithWrongElemet, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + destinationResponse: { + response: { + message: + 'Index: 0, ERROR :: /conversionValue/amount :: field is required but not found and has no default value\nERROR :: /user/userInfo/firstName :: field is required but not found and has no default value\nERROR :: /user/userInfo/lastName :: field is required but not found and has no default value\n', + status: 422, + }, + status: 422, + }, + message: + '[LINKEDIN_CONVERSION_API Response V1 Handler] - Request Processed Successfully', + response: [ + { + error: + '/conversionValue/amount :: field is required but not found and has no default value', + statusCode: 400, + metadata, + }, + ], + status: 422, + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_ads/dataDelivery/data.ts b/test/integrations/destinations/linkedin_ads/dataDelivery/data.ts new file mode 100644 index 0000000000..5bb0a7ef6e --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/dataDelivery/data.ts @@ -0,0 +1,4 @@ +import { testScenariosForV1API, statTags as baseStatTags } from './business'; +import { oauthScenariosV1 } from './oauth'; + +export const data = [...testScenariosForV1API, ...oauthScenariosV1]; diff --git a/test/integrations/destinations/linkedin_ads/dataDelivery/oauth.ts b/test/integrations/destinations/linkedin_ads/dataDelivery/oauth.ts new file mode 100644 index 0000000000..5cc643d972 --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/dataDelivery/oauth.ts @@ -0,0 +1,207 @@ +import { generateMetadata, generateProxyV1Payload } from '../../../testUtils'; +import { ProxyV1TestData } from '../../../testTypes'; + +export const testJSONData = { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '0', + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + }, + }, + ], +}; +export const statTags = { + destType: 'LINKEDIN_ADS', + errorCategory: 'network', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + errorType: 'aborted', + feature: 'dataDelivery', + implementation: 'native', + module: 'destination', +}; + +export const metadata = { + jobId: 1, + attemptNum: 1, + userId: 'default-userId', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', + sourceId: 'default-sourceId', + secret: { + accessToken: 'default-accessToken', + }, + dontBatch: false, +}; + +export const headerBlockWithCorrectAccessToken = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; + +const commonRequestParameters = { + headers: headerBlockWithCorrectAccessToken, + JSON: testJSONData, +}; +const commonRequestParametersWithInvalidAccess = { + headers: { ...headerBlockWithCorrectAccessToken, Authorization: 'Bearer invalidToken' }, + JSON: testJSONData, + accessToken: 'invalidToken', +}; + +const commonRequestParametersWithRevokedAccess = { + headers: { ...headerBlockWithCorrectAccessToken, Authorization: 'Bearer revokedToken' }, + JSON: testJSONData, + accessToken: 'revokedToken', +}; + +export const oauthScenariosV1: ProxyV1TestData[] = [ + { + id: 'linkedin_ads_v1_oauth_scenario_1', + name: 'linkedin_ads', + description: 'app event fails due to revoked access token error', + successCriteria: 'Should return 400 with revoked access token error', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + ...commonRequestParametersWithRevokedAccess, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 400, + body: { + output: { + response: [ + { + error: + '{"status":401,"serviceErrorCode":65601,"code":"REVOKED_ACCESS_TOKEN","message":"The token used in the request has been revoked by the user"}', + statusCode: 400, + metadata: { ...metadata, secret: { accessToken: 'revokedToken' } }, + }, + ], + statTags, + authErrorCategory: 'AUTH_STATUS_INACTIVE', + message: + 'LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. Invalid or expired access token. Retrying', + status: 400, + }, + }, + }, + }, + }, + { + id: 'linkedin_ads_v1_oauth_scenario_2', + name: 'linkedin_ads', + description: 'app event fails due to invalid access token error', + successCriteria: 'Should return 500 with invalid access token error', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + ...commonRequestParametersWithInvalidAccess, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 500, + body: { + output: { + response: [ + { + error: + '{"status":401,"serviceErrorCode":65600,"code":"INVALID_ACCESS_TOKEN","message":"Invalid access token"}', + statusCode: 500, + metadata: { ...metadata, secret: { accessToken: 'invalidToken' } }, + }, + ], + statTags: { ...statTags, errorType: 'retryable' }, + authErrorCategory: 'REFRESH_TOKEN', + message: + 'LinkedIn Conversion API: Error transformer proxy v1 during LinkedIn Conversion API response transformation. Invalid or expired access token. Retrying', + status: 500, + }, + }, + }, + }, + }, + { + id: 'linkedin_ads_v1_oauth_scenario_3', + name: 'linkedin_ads', + description: 'success case', + successCriteria: 'Should return 200 response', + scenario: 'Business', + feature: 'dataDelivery', + module: 'destination', + version: 'v1', + input: { + request: { + body: generateProxyV1Payload({ + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + ...commonRequestParameters, + }), + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: { + status: 200, + message: + '[LINKEDIN_CONVERSION_API Response V1 Handler] - Request Processed Successfully', + destinationResponse: { + response: { + elements: [ + { + status: 201, + }, + { + status: 201, + }, + ], + }, + status: 200, + }, + response: [ + { + statusCode: 200, + metadata: generateMetadata(1), + error: 'success', + }, + ], + }, + }, + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_ads/network.ts b/test/integrations/destinations/linkedin_ads/network.ts new file mode 100644 index 0000000000..890ad48589 --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/network.ts @@ -0,0 +1,186 @@ +export const headerBlockWithCorrectAccessToken = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; +export const element = { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '0', + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + }, +}; + +export const testJSONData = { + elements: [{ ...element }], +}; + +export const testJSONDataWithDifferentTypeConversion = { + elements: [ + { + ...element, + conversion: 'urn:li:partner:differentConversion', + }, + ], +}; + +export const wrongFormatElement = { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + userInfo: { + city: 'San Francisco', + }, + }, +}; + +export const wrongFormattedTestJSONData = { + elements: [{ ...wrongFormatElement }], +}; + +// MOCK DATA +const businessMockData = [ + { + description: 'Mock response from destination depicting request with a revoked access token', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: { ...headerBlockWithCorrectAccessToken, Authorization: 'Bearer revokedToken' }, + data: testJSONData, + }, + httpRes: { + data: { + status: 401, + serviceErrorCode: 65601, + code: 'REVOKED_ACCESS_TOKEN', + message: 'The token used in the request has been revoked by the user', + }, + status: 401, + statusText: 'OK', + }, + }, + { + description: 'Mock response from destination depicting request with an invalid access token', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: { ...headerBlockWithCorrectAccessToken, Authorization: 'Bearer invalidToken' }, + data: testJSONData, + }, + httpRes: { + data: { + status: 401, + serviceErrorCode: 65600, + code: 'INVALID_ACCESS_TOKEN', + message: 'Invalid access token', + }, + status: 401, + statusText: 'OK', + }, + }, + { + description: + 'Mock response from destination depicting a correct request with a valid access token', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: headerBlockWithCorrectAccessToken, + data: testJSONData, + }, + httpRes: { + data: { + elements: [ + { + status: 201, + }, + { + status: 201, + }, + ], + }, + status: 200, + statusText: 'OK', + }, + }, + { + description: + 'Mock response from destination depicting request with a conversion created differently than choosing direct API', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: headerBlockWithCorrectAccessToken, + data: testJSONDataWithDifferentTypeConversion, + }, + httpRes: { + data: { + message: + "Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + status: 400, + }, + status: 400, + statusText: 'OK', + }, + }, + { + description: + 'Mock response from destination depicting request with a conversion created differently than choosing direct API', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: headerBlockWithCorrectAccessToken, + data: testJSONDataWithDifferentTypeConversion, + }, + httpRes: { + data: { + message: + "Incorrect conversions information provided. Conversion's method should be CONVERSIONS_API, indices [0] (0-indexed)", + status: 400, + }, + status: 400, + statusText: 'OK', + }, + }, + { + description: + 'Mock response from destination depicting request with a conversion created differently than choosing direct API', + httpReq: { + method: 'post', + url: 'https://api.linkedin.com/rest/conversionEvents', + headers: headerBlockWithCorrectAccessToken, + data: wrongFormattedTestJSONData, + }, + httpRes: { + data: { + message: + 'Index: 0, ERROR :: /conversionValue/amount :: field is required but not found and has no default value\nERROR :: /user/userInfo/firstName :: field is required but not found and has no default value\nERROR :: /user/userInfo/lastName :: field is required but not found and has no default value\n', + status: 422, + }, + status: 422, + statusText: 'OK', + }, + }, +]; + +export const networkCallsData = [...businessMockData]; diff --git a/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts b/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts new file mode 100644 index 0000000000..287e35e5a7 --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/processor/configLevelFeaturesTestData.ts @@ -0,0 +1,219 @@ +import { + generateMetadata, + generateTrackPayload, + overrideDestination, + transformResultBuilder, +} from '../../../testUtils'; +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; + +const commonDestination: Destination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: { + hashData: true, + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + { + from: 'spin_result', + to: '23456', + }, + { + from: 'ABC Searched', + to: '34567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + Enabled: true, +}; + +const commonUserTraits = { + email: 'abc@gmail.com', + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + event_id: '12345', +}; + +const commonUserProperties = { + price: 400, + additional_bet_index: 0, + eventId: '12345', +}; + +const commonTimestamp = new Date('2023-10-14'); + +const commonHeader = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; + +export const configLevelFeaturesTestData: ProcessorTestData[] = [ + { + id: 'linkedin_ads-config-test-1', + name: 'linkedin_ads', + description: 'Track call : hashData is set to false and no deduplication key is provided', + scenario: 'Business', + successCriteria: 'email provided will not be hashed and eventId will be mapped from messageId', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonUserProperties, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { hashData: false }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'abc@gmail.com', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-config-test-2', + name: 'linkedin_ads', + description: 'Track call : hashData is set to true and deduplication key is provided', + scenario: 'Business', + successCriteria: + 'email provided will be hashed and eventId will be mapped from deduplication key properties.eventId', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonUserProperties, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { + deduplicationKey: `properties.eventId`, + }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_ads/processor/data.ts b/test/integrations/destinations/linkedin_ads/processor/data.ts new file mode 100644 index 0000000000..edd6d1f1b5 --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/processor/data.ts @@ -0,0 +1,11 @@ +import { trackTestData } from './trackTestData'; +import { validationTestData } from './validationTestData'; +import { configLevelFeaturesTestData } from './configLevelFeaturesTestData'; + +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2023-10-15')); +}; +export const data = [...trackTestData, ...validationTestData, ...configLevelFeaturesTestData].map( + (d) => ({ ...d, mockFns }), +); diff --git a/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts b/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts new file mode 100644 index 0000000000..f9dfc528db --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/processor/trackTestData.ts @@ -0,0 +1,718 @@ +import { generateMetadata, generateTrackPayload, transformResultBuilder } from '../../../testUtils'; +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; + +const commonDestination: Destination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: { + hashData: true, + deduplicationKey: 'properties.eventId', + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + { + from: 'spin_result', + to: '23456', + }, + { + from: 'ABC Searched', + to: '34567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + Enabled: true, +}; + +const commonUserTraits = { + email: 'abc@gmail.com', + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + event_id: '12345', +}; + +const commonUserProperties = { + price: 400, + additional_bet_index: 0, + eventId: '12345', +}; + +const commonPropertiesWithProducts = { + revenue: 400, + additional_bet_index: 0, + eventId: '12345', + products: [ + { + product_id: '123', + name: 'abc', + category: 'def', + brand: 'xyz', + variant: 'pqr', + price: 100, + quantity: 2, + }, + { + product_id: '456', + name: 'def', + category: 'abc', + brand: 'pqr', + variant: 'xyz', + price: 200, + quantity: 3, + }, + ], +}; + +const commonPropertiesWithProductsPriceNotPresentInAll = { + revenue: 400, + additional_bet_index: 0, + eventId: '12345', + products: [ + { + product_id: '123', + name: 'abc', + category: 'def', + brand: 'xyz', + variant: 'pqr', + quantity: 2, + }, + { + product_id: '456', + name: 'def', + category: 'abc', + brand: 'pqr', + variant: 'xyz', + price: 200, + quantity: 3, + }, + ], +}; + +const commonTimestamp = new Date('2023-10-14'); + +const commonStatTags = { + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + implementation: 'cdkV2', + destType: 'LINKEDIN_ADS', + module: 'destination', + feature: 'processor', + workspaceId: 'default-workspaceId', +}; + +const commonHeader = { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202402', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', +}; + +export const trackTestData: ProcessorTestData[] = [ + { + id: 'linkedin_ads-track-test-1', + name: 'linkedin_ads', + description: 'Track call : particular track event mapped to a specific conversion rule', + scenario: 'Business', + successCriteria: + 'event will respect the UI mapping and create a conversion event with the mapped conversion rule', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonUserProperties, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-2', + name: 'linkedin_ads', + description: 'Track call : event is mapped with more than one conversion rules ', + scenario: 'Business', + successCriteria: + 'event will respect the UI mapping and create a conversion event with the mapped conversion rule and club the two conversions in a single elements array', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'ABC Searched', + properties: commonUserProperties, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:1234567', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + { + conversion: 'urn:lla:llaPartnerConversion:34567', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-3', + name: 'linkedin_ads', + description: 'Track call : track event containing multiple allowed user identifiqers', + scenario: 'Business', + successCriteria: + 'event will respect the UI mapping and create a conversion event with the mapped conversion rule', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonUserProperties, + externalId: [ + { + id: 'test@rudderlabs.com', + type: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + }, + { + id: 'test@rudderlabs.com', + type: 'ACXIOM_ID', + }, + { + id: 'test@rudderlabs.com', + type: 'ORACLE_MOAT_ID', + }, + ], + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + { + idType: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ACXIOM_ID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ORACLE_MOAT_ID', + idValue: 'test@rudderlabs.com', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-4', + name: 'linkedin_ads', + description: 'Track call : event not containing any of the allowed user identifiers', + scenario: 'Business', + successCriteria: + 'Error will be thrown as the event does not contain any of the allowed user identifiers', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'ABC Searched', + properties: commonUserProperties, + context: { + traits: { + firstName: 'John', + }, + }, + timestamp: commonTimestamp, + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + '[LinkedIn Conversion API] no matching user id found. Please provide at least one of the following: email, linkedinFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: [LinkedIn Conversion API] no matching user id found. Please provide at least one of the following: email, linkedinFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + metadata: generateMetadata(1), + statTags: commonStatTags, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-5', + name: 'linkedin_ads', + description: 'Track call : track event containing product array', + scenario: 'Business', + successCriteria: + 'the amount will be summation of product * quantity for all the products in the array', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonPropertiesWithProducts, + externalId: [ + { + id: 'test@rudderlabs.com', + type: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + }, + { + id: 'test@rudderlabs.com', + type: 'ACXIOM_ID', + }, + { + id: 'test@rudderlabs.com', + type: 'ORACLE_MOAT_ID', + }, + ], + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '800', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + { + idType: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ACXIOM_ID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ORACLE_MOAT_ID', + idValue: 'test@rudderlabs.com', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-6', + name: 'linkedin_ads', + description: 'Track call : track event containing first name and last name in traits', + scenario: 'Business', + successCriteria: + 'output event will contain userInfo object only because first name and last name are present in traits', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonUserProperties, + context: { + traits: { ...commonUserTraits, firstName: 'John', lastName: 'Doe' }, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '400', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'John', + lastName: 'Doe', + }, + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-track-test-7', + name: 'linkedin_ads', + description: + 'Track call : track event containing product array where not all products contains price field', + scenario: 'Business', + successCriteria: + 'the amount will be summation of product * quantity for all the products in the array', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: commonPropertiesWithProductsPriceNotPresentInAll, + externalId: [ + { + id: 'test@rudderlabs.com', + type: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + }, + { + id: 'test@rudderlabs.com', + type: 'ACXIOM_ID', + }, + { + id: 'test@rudderlabs.com', + type: 'ORACLE_MOAT_ID', + }, + ], + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: commonDestination, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://api.linkedin.com/rest/conversionEvents`, + headers: commonHeader, + params: {}, + FORM: {}, + files: {}, + JSON: { + elements: [ + { + conversion: 'urn:lla:llaPartnerConversion:23456', + conversionHappenedAt: 1697241600000, + conversionValue: { + amount: '600', + currencyCode: 'USD', + }, + eventId: '12345', + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + { + idType: 'LINKEDIN_FIRST_PARTY_ADS_TRACKING_UUID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ACXIOM_ID', + idValue: 'test@rudderlabs.com', + }, + { + idType: 'ORACLE_MOAT_ID', + idValue: 'test@rudderlabs.com', + }, + ], + }, + }, + ], + }, + userId: '', + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts new file mode 100644 index 0000000000..4579cf68ee --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/processor/validationTestData.ts @@ -0,0 +1,323 @@ +import { generateMetadata, generateTrackPayload, overrideDestination } from '../../../testUtils'; +import { Destination } from '../../../../../src/types'; +import { ProcessorTestData } from '../../../testTypes'; + +const commonDestination: Destination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: { + hashData: true, + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + { + from: 'spin_result', + to: '23456', + }, + { + from: 'ABC Searched', + to: '34567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], + }, + Enabled: true, +}; + +const commonUserTraits = { + email: 'abc@gmail.com', + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + event_id: '12345', +}; + +const commonUserProperties = { + additional_bet_index: 0, + eventId: '12345', +}; + +const commonUserPropertiesWithProductWithoutPrice = { + additional_bet_index: 0, + eventId: '12345', + products: [ + { + productId: '12345', + }, + { + productId: '123456', + }, + ], +}; + +const commonStats = { + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + implementation: 'cdkV2', + destType: 'LINKEDIN_ADS', + module: 'destination', + feature: 'processor', + workspaceId: 'default-workspaceId', +}; + +const commonTimestamp = new Date('2023-10-14'); +const olderTimestamp = new Date('2023-07-13'); + +export const validationTestData: ProcessorTestData[] = [ + { + id: 'linkedin_ads-validation-test-1', + name: 'linkedin_ads', + description: 'Track call : event is older than 90 days', + scenario: 'Business', + successCriteria: 'shoud throw error with status code 400 and error message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'spin_result', + properties: { ...commonUserProperties, price: 400 }, + context: { + traits: commonUserTraits, + }, + timestamp: olderTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { hashData: false }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Events must be sent within ninety days of their occurrence.: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: Events must be sent within ninety days of their occurrence.', + metadata: generateMetadata(1), + statTags: { + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + implementation: 'cdkV2', + destType: 'LINKEDIN_ADS', + module: 'destination', + feature: 'processor', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-validation-test-2', + name: 'linkedin_ads', + description: 'Track call : event not mapped to conversion rule in UI', + scenario: 'Business', + successCriteria: + 'should throw error with status code 400 and error message no matching conversion rule found for random event. Please provide a conversion rule. Aborting', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'random event', + properties: { ...commonUserProperties, price: 400 }, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { + deduplicationKey: `properties.eventId`, + }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + '[LinkedIn Conversion API] no matching conversion rule found for random event. Please provide a conversion rule. Aborting: Workflow: procWorkflow, Step: deduceConversionEventRules, ChildStep: undefined, OriginalError: [LinkedIn Conversion API] no matching conversion rule found for random event. Please provide a conversion rule. Aborting', + metadata: generateMetadata(1), + statTags: { ...commonStats, errorType: 'configuration' }, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-validation-test-3', + name: 'linkedin_ads', + description: '[Error]: Check for unsupported message type', + scenario: 'Framework', + successCriteria: + 'Response should contain error message and status code should be 400, as we are sending a message type which is not supported by linkedin_ads destination and the error message should be Event type random is not supported', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: commonDestination, + metadata: generateMetadata(1), + message: { + userId: 'user123', + type: 'random', + groupId: 'XUepkK', + traits: { + subscribe: true, + }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '+12 345 678 900', + consent: 'email', + }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'message type random is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type random is not supported', + metadata: generateMetadata(1), + statTags: commonStats, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-validation-test-4', + name: 'linkedin_ads', + description: 'Track call : properties without product array and no price', + scenario: 'Business', + successCriteria: + 'should throw error with status code 400 and error message regarding price is a mandatory field for linkedin conversions', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'random event', + properties: commonUserProperties, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { + deduplicationKey: `properties.eventId`, + }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + '[LinkedIn Conversion API]: Cannot map price for event random event. Aborting: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: [LinkedIn Conversion API]: Cannot map price for event random event. Aborting', + metadata: generateMetadata(1), + statTags: commonStats, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_ads-validation-test-5', + name: 'linkedin_ads', + description: 'Track call : properties with product array and no price', + scenario: 'Business', + successCriteria: + 'should throw error with status code 400 and error message regarding price is a mandatory field for linkedin conversions', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateTrackPayload({ + event: 'random event', + properties: commonUserPropertiesWithProductWithoutPrice, + context: { + traits: commonUserTraits, + }, + timestamp: commonTimestamp, + messageId: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + }), + metadata: generateMetadata(1), + destination: overrideDestination(commonDestination, { + deduplicationKey: `properties.eventId`, + }), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + '[LinkedIn Conversion API]: Cannot map price for event random event. Aborting: Workflow: procWorkflow, Step: commonFields, ChildStep: undefined, OriginalError: [LinkedIn Conversion API]: Cannot map price for event random event. Aborting', + metadata: generateMetadata(1), + statTags: commonStats, + statusCode: 400, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_ads/router/data.ts b/test/integrations/destinations/linkedin_ads/router/data.ts new file mode 100644 index 0000000000..cf7defe6af --- /dev/null +++ b/test/integrations/destinations/linkedin_ads/router/data.ts @@ -0,0 +1,462 @@ +export const mockFns = (_) => { + // @ts-ignore + jest.useFakeTimers().setSystemTime(new Date('2023-10-15')); +}; + +const config = { + hashData: true, + deduplicationKey: 'properties.eventId', + conversionMapping: [ + { + from: 'ABC Searched', + to: '1234567', + }, + { + from: 'spin_result', + to: '23456', + }, + { + from: 'ABC Searched', + to: '34567', + }, + ], + oneTrustCookieCategories: [ + { + oneTrustCookieCategory: 'Marketing', + }, + ], +}; + +const commonDestination = { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: config, + Enabled: true, +}; + +export const data = [ + { + id: 'linkedin_ads-track-test-1', + name: 'linkedin_ads', + description: 'Track call : custom event calls with simple user properties and traits', + scenario: 'Business', + successCriteria: + 'event not respecting the internal mapping and as well as UI mapping should be considered as a custom event and should be sent as it is', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + type: 'track', + event: 'ABC Searched', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + properties: { + tax: 2, + total: 27.5, + coupon: 'hasbros', + revenue: 48, + price: 25, + quantity: 2, + currency: 'USD', + discount: 2.5, + order_id: '50314b8e9bcf000000000000', + requestIP: '123.0.0.0', + optOutType: 'LDP', + clickId: 'dummy_clickId', + + shipping: 3, + subtotal: 22.5, + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + { + message: { + type: 'track', + event: 'ABC Searched', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + email: 'abc@gmail.com', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + properties: { + tax: 2, + total: 27.5, + coupon: 'hasbros', + revenue: 48, + price: 25, + quantity: 2, + currency: 'USD', + discount: 2.5, + order_id: '50314b8e9bcf000000000000', + requestIP: '123.0.0.0', + optOutType: 'LDP', + clickId: 'dummy_clickId', + + shipping: 3, + subtotal: 22.5, + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 2, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + { + message: { + type: 'track', + event: 'spin_result', + sentAt: '2020-08-14T05: 30: 30.118Z', + channel: 'web', + context: { + source: 'test', + userAgent: 'chrome', + traits: { + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + phone: '+1234589947', + gender: 'non-binary', + db: '19950715', + lastname: 'Rudderlabs', + firstName: 'Test', + address: { + city: 'Kolkata', + state: 'WB', + zip: '700114', + country: 'IN', + }, + }, + device: { + advertisingId: 'abc123', + }, + library: { + name: 'rudder-sdk-ruby-sync', + version: '1.0.6', + }, + }, + messageId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + timestamp: '2024-02-10T12:16:07.251Z', + properties: { + tax: 2, + total: 27.5, + coupon: 'hasbros', + revenue: 48, + price: 25, + quantity: 2, + currency: 'USD', + discount: 2.5, + order_id: '50314b8e9bcf000000000000', + requestIP: '123.0.0.0', + optOutType: 'LDP', + clickId: 'dummy_clickId', + + shipping: 3, + subtotal: 22.5, + affiliation: 'Google Store', + checkout_id: 'fksdjfsdjfisjf9sdfjsd9f', + }, + anonymousId: '50be5c78-6c3f-4b60-be84-97805a316fb1', + integrations: { + All: true, + }, + }, + metadata: { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 3, + secret: { + accessToken: 'dummyToken', + }, + }, + destination: commonDestination, + }, + ], + destType: 'linkedin_ads', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 3, + secret: { + accessToken: 'dummyToken', + }, + }, + ], + destination: { + ID: '12335', + Name: 'sample-destination', + DestinationDefinition: { + ID: '123', + Name: 'linkedin_ads', + DisplayName: 'LinkedIn Ads', + Config: { + cdkV2Enabled: true, + }, + }, + WorkspaceID: '123', + Transformations: [], + Config: config, + Enabled: true, + }, + batched: false, + statusCode: 400, + error: + '[LinkedIn Conversion API] no matching user id found. Please provide at least one of the following: email, linkedinFirstPartyAdsTrackingUUID, acxiomId, oracleMoatId', + statTags: { + destType: 'LINKEDIN_ADS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + }, + }, + { + batchedRequest: { + body: { + JSON: { + elements: [ + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:1234567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:34567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:1234567', + }, + { + conversionHappenedAt: 1707567367251, + eventId: '7208bbb6-2c4e-45bb-bf5b-ad426f3593e9', + conversionValue: { + currencyCode: 'USD', + amount: '50', + }, + user: { + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08', + }, + ], + userInfo: { + firstName: 'Test', + lastName: 'Rudderlabs', + }, + }, + conversion: 'urn:lla:llaPartnerConversion:34567', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.linkedin.com/rest/conversionEvents', + headers: { + 'Content-Type': 'application/json', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + 'LinkedIn-Version': '202402', + Authorization: 'Bearer dummyToken', + }, + params: {}, + files: {}, + }, + metadata: [ + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 1, + secret: { + accessToken: 'dummyToken', + }, + }, + { + sourceType: '', + destinationType: '', + namespace: '', + jobId: 2, + secret: { + accessToken: 'dummyToken', + }, + }, + ], + batched: true, + statusCode: 200, + destination: commonDestination, + }, + ], + }, + }, + }, + }, +].map((d) => ({ ...d, mockFns })); diff --git a/test/integrations/destinations/movable_ink/common.ts b/test/integrations/destinations/movable_ink/common.ts index f7eaa7af39..29fe76852c 100644 --- a/test/integrations/destinations/movable_ink/common.ts +++ b/test/integrations/destinations/movable_ink/common.ts @@ -110,6 +110,186 @@ const trackTestProperties = { position: 2, category: 'Games', }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358033', + sku: '8472-998-0112', + name: 'Cones of Dunshire', + price: 40, + position: 1, + category: 'Games', + url: 'https://www.website.com/product/path', + image_url: 'https://www.website.com/product/path.jpg', + }, + { + product_id: '577c6f5d5cf86a4c7735ba03', + sku: '3309-483-2201', + name: 'Five Crowns', + price: 5, + position: 2, + category: 'Games', + }, + { + product_id: '122c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Ticket to Ride', + price: 20, + position: 3, + category: 'Games', + }, + { + product_id: '222c6f5d5cf86a4c77358033', + sku: '9472-998-0112', + name: 'Catan', + price: 30, + position: 4, + category: 'Games', + }, + { + product_id: '322c6f5d5cf86a4c77358033', + sku: '7472-998-0112', + name: 'Pandemic', + price: 25, + position: 5, + category: 'Games', + }, + { + product_id: '422c6f5d5cf86a4c77358033', + sku: '8472-998-0113', + name: 'Exploding Kittens', + price: 15, + position: 6, + category: 'Games', + }, + { + product_id: '522c6f5d5cf86a4c77358033', + sku: '8472-998-0114', + name: 'Codenames', + price: 18, + position: 7, + category: 'Games', + }, + { + product_id: '622c6f5d5cf86a4c77358034', + sku: '8472-998-0115', + name: 'Scythe', + price: 35, + position: 8, + category: 'Games', + }, ], }, 'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' }, diff --git a/test/integrations/destinations/movable_ink/mocks.ts b/test/integrations/destinations/movable_ink/mocks.ts new file mode 100644 index 0000000000..2468f51315 --- /dev/null +++ b/test/integrations/destinations/movable_ink/mocks.ts @@ -0,0 +1,6 @@ +import config from '../../../../src/cdk/v2/destinations/movable_ink/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_REQUEST_SIZE_IN_BYTES', 5000); + jest.replaceProperty(config, 'MAX_BATCH_SIZE', 2); +}; diff --git a/test/integrations/destinations/movable_ink/processor/identify.ts b/test/integrations/destinations/movable_ink/processor/identify.ts index 27186da05c..e5bbf5a9a7 100644 --- a/test/integrations/destinations/movable_ink/processor/identify.ts +++ b/test/integrations/destinations/movable_ink/processor/identify.ts @@ -1,6 +1,6 @@ import { ProcessorTestData } from '../../../testTypes'; import { generateMetadata, transformResultBuilder } from '../../../testUtils'; -import { destType, channel, destination, traits, headers } from '../common'; +import { destType, destination, traits, headers } from '../common'; export const identify: ProcessorTestData[] = [ { diff --git a/test/integrations/destinations/movable_ink/processor/track.ts b/test/integrations/destinations/movable_ink/processor/track.ts index 5f30a3de83..890de11a0c 100644 --- a/test/integrations/destinations/movable_ink/processor/track.ts +++ b/test/integrations/destinations/movable_ink/processor/track.ts @@ -23,6 +23,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -49,6 +50,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -84,6 +86,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Order Completed', properties: trackTestProperties['Order Completed'], integrations: { All: true, @@ -110,6 +113,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Order Completed', properties: trackTestProperties['Order Completed'], integrations: { All: true, @@ -145,6 +149,7 @@ export const track: ProcessorTestData[] = [ channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Custom Event', properties: trackTestProperties['Custom Event'], integrations: { All: true, @@ -171,6 +176,7 @@ export const track: ProcessorTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Custom Event', properties: trackTestProperties['Custom Event'], integrations: { All: true, diff --git a/test/integrations/destinations/movable_ink/processor/validation.ts b/test/integrations/destinations/movable_ink/processor/validation.ts index ab6b123eb7..6aafb5e2c0 100644 --- a/test/integrations/destinations/movable_ink/processor/validation.ts +++ b/test/integrations/destinations/movable_ink/processor/validation.ts @@ -214,4 +214,48 @@ export const validation: ProcessorTestData[] = [ }, }, }, + { + id: 'MovableInk-validation-test-6', + name: destType, + description: 'Missing event name', + scenario: 'Framework', + successCriteria: 'Instrumentation Error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination, + message: { + type: 'track', + anonymousId: 'anonId123', + userId: 'userId123', + properties: {}, + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Event name is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Event name is not present. Aborting', + metadata: generateMetadata(1), + statTags: processorInstrumentationErrorStatTags, + statusCode: 400, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/movable_ink/router/data.ts b/test/integrations/destinations/movable_ink/router/data.ts index 72df3d7074..afadfec56e 100644 --- a/test/integrations/destinations/movable_ink/router/data.ts +++ b/test/integrations/destinations/movable_ink/router/data.ts @@ -1,6 +1,7 @@ import { RouterTestData } from '../../../testTypes'; import { RouterTransformationRequest } from '../../../../../src/types'; import { generateMetadata } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; import { destType, channel, @@ -43,6 +44,7 @@ const routerRequest: RouterTransformationRequest = { channel, anonymousId: 'anonId123', userId: 'userId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -58,19 +60,73 @@ const routerRequest: RouterTransformationRequest = { channel, anonymousId: 'anonId123', userId: 'userId123', - properties: trackTestProperties['Custom Event'], + event: 'Custom Event', integrations: { All: true, }, + originalTimestamp: '2024-03-04T15:32:56.409Z', }, metadata: generateMetadata(4), destination, }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Custom Event', + properties: trackTestProperties['Custom Event'], + integrations: { + All: true, + }, + }, + metadata: generateMetadata(5), + destination, + }, + ], + destType, +}; + +// >5KB payload +const routerRequest2: RouterTransformationRequest = { + input: [ + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Order Completed', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(1), + destination, + }, + { + message: { + type: 'track', + channel, + anonymousId: 'anonId123', + userId: 'userId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(2), + destination, + }, ], destType, }; -export const data: RouterTestData[] = [ +export const data = [ { id: 'MovableInk-router-test-1', name: destType, @@ -118,6 +174,7 @@ export const data: RouterTestData[] = [ channel, userId: 'userId123', anonymousId: 'anonId123', + event: 'Product Added', properties: trackTestProperties['Product Added'], integrations: { All: true, @@ -138,6 +195,42 @@ export const data: RouterTestData[] = [ statusCode: 200, destination, }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4)], + batched: true, + statusCode: 200, + destination, + }, { metadata: [generateMetadata(2)], batched: false, @@ -147,7 +240,7 @@ export const data: RouterTestData[] = [ destination, }, { - metadata: [generateMetadata(4)], + metadata: [generateMetadata(5)], batched: false, statusCode: 400, error: 'Timestamp is not present. Aborting', @@ -158,5 +251,105 @@ export const data: RouterTestData[] = [ }, }, }, + mockFns: defaultMockFns, + }, + { + id: 'MovableInk-router-test-2', + name: destType, + description: 'Basic Router Test to test Max Request Size', + scenario: 'Framework', + successCriteria: + 'Some events should be transformed successfully and some should fail for missing fields and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest2, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Order Completed', + properties: trackTestProperties['Order Completed'], + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: destination.Config.endpoint, + headers, + params: {}, + body: { + JSON: { + events: [ + { + type: 'track', + channel, + userId: 'userId123', + anonymousId: 'anonId123', + event: 'Custom Event', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + timestamp: 1709566376409, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(2)], + batched: true, + statusCode: 200, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, }, ]; diff --git a/test/integrations/destinations/ninetailed/processor/data.ts b/test/integrations/destinations/ninetailed/processor/data.ts index 4e5fa72365..9d3cd217cd 100644 --- a/test/integrations/destinations/ninetailed/processor/data.ts +++ b/test/integrations/destinations/ninetailed/processor/data.ts @@ -1,5 +1,4 @@ import { validationFailures } from './validation'; import { track } from './track'; -import { page } from './page'; import { identify } from './identify'; -export const data = [...identify, ...page, ...track, ...validationFailures]; +export const data = [...identify, ...track, ...validationFailures]; diff --git a/test/integrations/destinations/ninetailed/processor/page.ts b/test/integrations/destinations/ninetailed/processor/page.ts deleted file mode 100644 index 93a086ceea..0000000000 --- a/test/integrations/destinations/ninetailed/processor/page.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { destination, context, commonProperties, metadata } from '../commonConfig'; -import { transformResultBuilder } from '../../../testUtils'; -export const page = [ - { - id: 'ninetailed-test-page-success-1', - name: 'ninetailed', - description: 'page call with all mappings available', - scenario: 'Framework+Buisness', - successCriteria: 'Response should contain all the mappings and status code should be 200', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - destination, - message: { - context, - type: 'page', - event: 'product purchased', - userId: 'sajal12', - channel: 'mobile', - messageId: 'dummy_msg_id', - properties: commonProperties, - anonymousId: 'anon_123', - integrations: { - All: true, - }, - originalTimestamp: '2021-01-25T15:32:56.409Z', - }, - metadata, - }, - ], - }, - }, - output: { - response: { - status: 200, - body: [ - { - metadata: { - destinationId: 'dummyDestId', - }, - output: transformResultBuilder({ - method: 'POST', - endpoint: - 'https://experience.ninetailed.co/v2/organizations/dummyOrganisationId/environments/main/events', - JSON: { - events: [ - { - context: { - app: { - name: 'RudderLabs JavaScript SDK', - version: '1.0.0', - }, - campaign: { - name: 'campign_123', - source: 'social marketing', - medium: 'facebook', - term: '1 year', - }, - library: { - name: 'RudderstackSDK', - version: 'Ruddderstack SDK version', - }, - locale: 'en-US', - page: { - path: '/signup', - referrer: 'https://rudderstack.medium.com/', - search: '?type=freetrial', - url: 'https://app.rudderstack.com/signup?type=freetrial', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - location: { - coordinates: { - latitude: 40.7128, - longitude: -74.006, - }, - city: 'San Francisco', - postalCode: '94107', - region: 'CA', - regionCode: 'CA', - country: ' United States', - countryCode: 'United States of America', - continent: 'North America', - timezone: 'America/Los_Angeles', - }, - }, - type: 'page', - channel: 'mobile', - messageId: 'dummy_msg_id', - properties: commonProperties, - anonymousId: 'anon_123', - originalTimestamp: '2021-01-25T15:32:56.409Z', - }, - ], - }, - userId: '', - }), - statusCode: 200, - }, - ], - }, - }, - }, -]; diff --git a/test/integrations/destinations/ninetailed/router/data.ts b/test/integrations/destinations/ninetailed/router/data.ts index 05105f4aed..1bf664d1c4 100644 --- a/test/integrations/destinations/ninetailed/router/data.ts +++ b/test/integrations/destinations/ninetailed/router/data.ts @@ -31,15 +31,6 @@ export const data = [ metadata: { jobId: 1, userId: 'u1' }, destination, }, - { - message: { - ...commonInput, - type: 'page', - properties: pageProperties, - }, - metadata: { jobId: 2, userId: 'u1' }, - destination, - }, { message: { type: 'identify', @@ -80,11 +71,6 @@ export const data = [ event: 'product list viewed', properties: trackProperties, }, - { - ...commonOutput, - type: 'page', - properties: pageProperties, - }, { type: 'identify', ...commonOutput, @@ -103,7 +89,6 @@ export const data = [ }, metadata: [ { jobId: 1, userId: 'u1' }, - { jobId: 2, userId: 'u1' }, { jobId: 3, userId: 'u1' }, ], batched: true, @@ -142,21 +127,9 @@ export const data = [ { message: { ...commonInput, - type: 'page', - properties: { - title: 'Sample Page', - url: 'https://example.com/?utm_campaign=example_campaign&utm_content=example_content', - path: '/', - hash: '', - search: '?utm_campaign=example_campaign&utm_content=example_content', - width: '1920', - height: '1080', - query: { - utm_campaign: 'example_campaign', - utm_content: 'example_content', - }, - referrer: '', - }, + type: 'track', + event: 'product added', + properties: trackProperties, }, metadata: { jobId: 2, userId: 'u1' }, destination, @@ -210,8 +183,9 @@ export const data = [ }, { ...commonOutput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, ], }, @@ -264,8 +238,9 @@ export const data = [ { message: { ...commonInput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, metadata: { jobId: 2, userId: 'u1' }, destination, @@ -330,8 +305,9 @@ export const data = [ }, { ...commonOutput, - type: 'page', - properties: pageProperties, + type: 'track', + event: 'product added', + properties: trackProperties, }, ], }, diff --git a/test/integrations/destinations/snapchat_conversion/processor/data.ts b/test/integrations/destinations/snapchat_conversion/processor/data.ts index b0d14208cc..7de7ed9b8d 100644 --- a/test/integrations/destinations/snapchat_conversion/processor/data.ts +++ b/test/integrations/destinations/snapchat_conversion/processor/data.ts @@ -4600,6 +4600,140 @@ export const data = [ }, }, }, + { + name: 'snapchat_conversion', + description: 'test event mapping from destination config', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + messageId: 'ec5481b6-a926-4d2e-b293-0b3a77c4d3be', + originalTimestamp: '2022-04-22T10:57:58Z', + anonymousId: 'ea5cfab2-3961-4d8a-8187-3d1858c99090', + context: { + traits: { + email: 'test@email.com', + phone: '+91 2111111 ', + }, + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + device: { + advertisingId: 'T0T0T072-5e28-45a1-9eda-ce22a3e36d1a', + id: '3f034872-5e28-45a1-9eda-ce22a3e36d1a', + manufacturer: 'Google', + name: 'generic_x86_arm', + type: 'ios', + attTrackingStatus: 3, + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.0.0', + }, + locale: 'en-US', + os: { + name: 'iOS', + version: '14.4.1', + }, + screen: { + density: 2, + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + }, + type: 'track', + event: 'Custom Event', + properties: { + query: 't-shirts', + event_conversion_type: 'web', + }, + integrations: { + All: true, + }, + sentAt: '2022-04-22T10:57:58Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: false, + }, + }, + Config: { + pixelId: 'dummyPixelId', + apiKey: 'dummyApiKey', + rudderEventsToSnapEvents: [ + { + from: 'Custom Event', + to: 'level_complete', + }, + ], + }, + }, + metadata: { + jobId: 47, + destinationId: 'd2', + workspaceId: 'w2', + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 47, + destinationId: 'd2', + workspaceId: 'w2', + }, + output: { + version: '1', + type: 'REST', + userId: '', + method: 'POST', + endpoint: 'https://tr.snapchat.com/v2/conversion', + headers: { + Authorization: 'Bearer dummyApiKey', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + event_type: 'LEVEL_COMPLETE', + hashed_email: '73062d872926c2a556f17b36f50e328ddf9bff9d403939bd14b6c3b7f5a33fc2', + hashed_phone_number: + 'bc77d64d7045fe44795ed926df37231a0cfb6ec6b74588c512790e9f143cc492', + hashed_mobile_ad_id: + 'f9779d734aaee50f16ee0011260bae7048f1d9a128c62b6a661077875701edd2', + hashed_idfv: '54bd0b26a3d39dad90f5149db49b9fd9ba885f8e35d1d94cae69273f5e657b9f', + user_agent: + 'mozilla/5.0 (macintosh; intel mac os x 10_15_2) applewebkit/537.36 (khtml, like gecko) chrome/79.0.3945.88 safari/537.36', + timestamp: '1650625078', + event_conversion_type: 'OFFLINE', + pixel_id: 'dummyPixelId', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + statusCode: 200, + }, + ], + }, + }, + }, ].map((tc) => ({ ...tc, mockFns: (_) => { diff --git a/test/integrations/destinations/webhook/processor/data.ts b/test/integrations/destinations/webhook/processor/data.ts index dbe83a79a5..e9629041fe 100644 --- a/test/integrations/destinations/webhook/processor/data.ts +++ b/test/integrations/destinations/webhook/processor/data.ts @@ -2758,4 +2758,221 @@ export const data = [ }, }, }, + { + name: 'webhook', + description: 'Test POST method with track message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + event: 'spin_result', + integrations: { + All: true, + }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + additional_bet_index: 0, + level: 1, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + }, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + headers: [ + { + from: 'Content-Type', + to: 'application/xml', + }, + { + from: 'test2', + to: 'value2', + }, + ], + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + JSON: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + event: 'spin_result', + integrations: { + All: true, + }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + additional_bet_index: 0, + level: 1, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + type: 'REST', + method: 'POST', + endpoint: 'http://6b0e6a60.ngrok.io', + headers: { + 'content-type': 'application/xml', + test2: 'value2', + }, + params: {}, + files: {}, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'webhook', + description: 'Test method PATCH', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + context: { + traits: { + address: { + city: 'Dhaka', + country: 'Bangladesh', + }, + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + }, + }, + event: 'spin_result', + integrations: { + All: true, + }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + prop1: 1, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + user_properties: { + coin_balance: 9466052, + }, + }, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + webhookMethod: 'PATCH', + headers: [ + { + from: 'test2', + to: 'value2', + }, + ], + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + body: { + JSON: { + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + context: { + traits: { + address: { + city: 'Dhaka', + country: 'Bangladesh', + }, + anonymousId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + }, + }, + event: 'spin_result', + integrations: { + All: true, + }, + message_id: 'a80f82be-9bdc-4a9f-b2a5-15621ee41df8', + properties: { + prop1: 1, + }, + timestamp: '2019-09-01T15:46:51.693229+05:30', + type: 'track', + user_properties: { + coin_balance: 9466052, + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + userId: 'c82cbdff-e5be-4009-ac78-cdeea09ab4b1', + type: 'REST', + method: 'PATCH', + endpoint: 'http://6b0e6a60.ngrok.io', + headers: { + 'content-type': 'application/json', + test2: 'value2', + }, + params: {}, + files: {}, + }, + metadata: { + destinationId: 'destId', + workspaceId: 'wspId', + }, + statusCode: 200, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/webhook/router/data.ts b/test/integrations/destinations/webhook/router/data.ts index a082eb12f1..6c738ee8a7 100644 --- a/test/integrations/destinations/webhook/router/data.ts +++ b/test/integrations/destinations/webhook/router/data.ts @@ -419,4 +419,391 @@ export const data = [ }, }, }, + { + name: 'webhook', + description: 'Identify payload with 3 events in 1 batch', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + anonymousId: '234234234234234', + channel: 'mobile', + context: { + app: { + build: '1', + name: 'AMTestProject', + namespace: 'com.rudderstack.android.rudderstack.sampleAndroidApp', + version: '1.0', + }, + device: { + id: '8d872292709c6fbe', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'android', + }, + library: { + name: 'com.rudderstack.android.sdk.core', + version: '1.0.2', + }, + locale: 'en-US', + network: { + carrier: 'Android', + bluetooth: false, + cellular: true, + wifi: true, + }, + os: { + name: 'Android', + version: '9', + }, + screen: { + density: 420, + height: 1794, + width: 1080, + }, + timezone: 'Asia/Kolkata', + traits: { + address: { + city: 'Kolkata', + country: 'India', + postalcode: '700096', + state: 'West bengal', + street: 'Park Street', + }, + age: '30', + anonymousId: '8d872292709c6fbe', + birthday: '2020-05-26', + createdat: '18th March 2020', + description: 'Premium User for 3 years', + email: 'identify@test.com', + firstname: 'John', + userId: 'sample_user_id', + lastname: 'Sparrow', + name: 'John Sparrow', + id: 'sample_user_id', + phone: '9876543210', + username: 'john_sparrow', + quantity: '5', + price: '56.0', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + event: 'identify', + integrations: { + All: true, + }, + messageId: '1590431830865-3be680d6-7dcd-4b05-8460-f3acc30046d9', + originalTimestamp: '2020-05-25T18:37:10.865Z', + sentAt: '2020-05-25T18:37:10.917Z', + type: 'identify', + userId: 'sample_user_id', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + headers: [ + { from: '', to: '' }, + { from: 'test2', to: 'value2' }, + ], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + { + message: { + anonymousId: '1231241234123', + channel: 'mobile', + context: { + timezone: 'Asia/Kolkata', + traits: { + address: { + city: 'Kolkata', + country: 'India', + postalcode: '700096', + state: 'West bengal', + street: 'Park Street', + }, + age: '30', + anonymousId: '8d872292709c6fbe', + birthday: '2020-05-26', + createdat: '18th March 2020', + description: 'Premium User for 3 years', + email: 'identify2@test.com', + firstname: 'John', + userId: 'sample_user_id', + lastname: 'Sparrow', + name: 'John Sparrow', + id: 'sample_user_id', + phone: '9876543210', + username: 'john_sparrow', + quantity: '5', + price: '56.0', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + event: 'identify', + integrations: { + All: true, + }, + messageId: '23432324-3be680d6-7dcd-4b05-8460-f3acc30046d9', + type: 'identify', + userId: 'sample_user_id', + }, + metadata: { jobId: 3, userId: 'u1' }, + destination: { + Config: { webhookUrl: 'https://6b0e6a60.ngrok.io/n' }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + { + message: { + anonymousId: '345345435', + channel: 'mobile', + event: 'identify', + integrations: { + All: true, + }, + messageId: '23432324-3be680d6-7dcd-4b05-8460-f3acc30046d9', + type: 'identify', + userId: 'sample_user_id', + }, + metadata: { jobId: 4, userId: 'u1' }, + destination: { + Config: { webhookUrl: 'https://6b0e6a60.ngrok.io/n' }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + ], + destType: 'webhook', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + anonymousId: '234234234234234', + channel: 'mobile', + context: { + app: { + build: '1', + name: 'AMTestProject', + namespace: 'com.rudderstack.android.rudderstack.sampleAndroidApp', + version: '1.0', + }, + device: { + id: '8d872292709c6fbe', + manufacturer: 'Google', + model: 'AOSP on IA Emulator', + name: 'generic_x86_arm', + type: 'android', + }, + library: { + name: 'com.rudderstack.android.sdk.core', + version: '1.0.2', + }, + locale: 'en-US', + network: { + carrier: 'Android', + bluetooth: false, + cellular: true, + wifi: true, + }, + os: { + name: 'Android', + version: '9', + }, + screen: { + density: 420, + height: 1794, + width: 1080, + }, + timezone: 'Asia/Kolkata', + traits: { + address: { + city: 'Kolkata', + country: 'India', + postalcode: '700096', + state: 'West bengal', + street: 'Park Street', + }, + age: '30', + anonymousId: '8d872292709c6fbe', + birthday: '2020-05-26', + createdat: '18th March 2020', + description: 'Premium User for 3 years', + email: 'identify@test.com', + firstname: 'John', + userId: 'sample_user_id', + lastname: 'Sparrow', + name: 'John Sparrow', + id: 'sample_user_id', + phone: '9876543210', + username: 'john_sparrow', + quantity: '5', + price: '56.0', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + event: 'identify', + integrations: { + All: true, + }, + messageId: '1590431830865-3be680d6-7dcd-4b05-8460-f3acc30046d9', + originalTimestamp: '2020-05-25T18:37:10.865Z', + sentAt: '2020-05-25T18:37:10.917Z', + type: 'identify', + userId: 'sample_user_id', + }, + FORM: {}, + }, + files: {}, + endpoint: 'http://6b0e6a60.ngrok.io', + userId: '234234234234234', + headers: { 'content-type': 'application/json', test2: 'value2' }, + version: '1', + params: {}, + type: 'REST', + method: 'POST', + }, + metadata: [{ jobId: 2, userId: 'u1' }], + batched: false, + statusCode: 200, + destination: { + Config: { + webhookUrl: 'http://6b0e6a60.ngrok.io', + headers: [ + { from: '', to: '' }, + { from: 'test2', to: 'value2' }, + ], + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + { + batchedRequest: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + anonymousId: '1231241234123', + channel: 'mobile', + context: { + timezone: 'Asia/Kolkata', + traits: { + address: { + city: 'Kolkata', + country: 'India', + postalcode: '700096', + state: 'West bengal', + street: 'Park Street', + }, + age: '30', + anonymousId: '8d872292709c6fbe', + birthday: '2020-05-26', + createdat: '18th March 2020', + description: 'Premium User for 3 years', + email: 'identify2@test.com', + firstname: 'John', + userId: 'sample_user_id', + lastname: 'Sparrow', + name: 'John Sparrow', + id: 'sample_user_id', + phone: '9876543210', + username: 'john_sparrow', + quantity: '5', + price: '56.0', + }, + userAgent: + 'Dalvik/2.1.0 (Linux; U; Android 9; AOSP on IA Emulator Build/PSR1.180720.117)', + }, + event: 'identify', + integrations: { + All: true, + }, + messageId: '23432324-3be680d6-7dcd-4b05-8460-f3acc30046d9', + type: 'identify', + userId: 'sample_user_id', + }, + FORM: {}, + }, + files: {}, + endpoint: 'https://6b0e6a60.ngrok.io/n', + userId: '1231241234123', + headers: { 'content-type': 'application/json' }, + version: '1', + params: {}, + type: 'REST', + method: 'POST', + }, + metadata: [{ jobId: 3, userId: 'u1' }], + batched: false, + statusCode: 200, + destination: { + Config: { + webhookUrl: 'https://6b0e6a60.ngrok.io/n', + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + { + batchedRequest: { + body: { + XML: {}, + JSON_ARRAY: {}, + JSON: { + anonymousId: '345345435', + channel: 'mobile', + event: 'identify', + integrations: { + All: true, + }, + messageId: '23432324-3be680d6-7dcd-4b05-8460-f3acc30046d9', + type: 'identify', + userId: 'sample_user_id', + }, + FORM: {}, + }, + files: {}, + endpoint: 'https://6b0e6a60.ngrok.io/n', + userId: '345345435', + headers: { 'content-type': 'application/json' }, + version: '1', + params: {}, + type: 'REST', + method: 'POST', + }, + metadata: [{ jobId: 4, userId: 'u1' }], + batched: false, + statusCode: 200, + destination: { + Config: { + webhookUrl: 'https://6b0e6a60.ngrok.io/n', + }, + DestinationDefinition: { Config: { cdkV2Enabled: true } }, + }, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 13a76702f9..c16aeff98c 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -484,7 +484,7 @@ export const generateProxyV1Payload = ( workspaceId: 'default-workspaceId', sourceId: 'default-sourceId', secret: { - accessToken: 'default-accessToken', + accessToken: payloadParameters.accessToken || 'default-accessToken', }, dontBatch: false, },