fix: pinterest, mixpanel, klaviyo updates (#2288)

rudderlabs · Jun 20, 2023 · 8a18b90 · 8a18b90
2 parents a9f2925 + 50f07d3
commit 8a18b90
Show file tree

Hide file tree

Showing 17 changed files with 927 additions and 153 deletions.
diff --git a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml
@@ -93,10 +93,10 @@ steps:
         "db": {{{{$.getGenericPaths("birthday")}}}},
         "ln": {{{{$.getGenericPaths("lastName")}}}},
         "fn": {{{{$.getGenericPaths("firstName")}}}},
-        "ct": (.traits.address.city ?? .context.traits.address.city).toLowerCase(),
-        "st": (.traits.address.state ?? .context.traits.address.state).toLowerCase(),
+        "ct": .traits.address.city ?? .context.traits.address.city,
+        "st": .traits.address.state ?? .context.traits.address.state,
         "zp": .traits.address.zip ?? .context.traits.address.zip,
-        "country": (.traits.address.country ?? .context.traits.address.country).toLowerCase(),
+        "country": .traits.address.country ?? .context.traits.address.country,
         "hashed_maids": .context.device.advertisingId,
         "client_ip_address": .context.ip ?? .request_ip,
         "client_user_agent": .context.userAgent,

diff --git a/src/v0/destinations/af/transform.js b/src/v0/destinations/af/transform.js
@@ -32,7 +32,9 @@ function responseBuilderSimple(payload, message, destination) {
   } else if (os && isAppleFamily(os) && appleAppId) {
     endpoint = `${ENDPOINT}id${appleAppId}`;
   } else {
-    throw new ConfigurationError('Invalid app endpoint');
+    throw new ConfigurationError(
+      'os name is required along with the respective appId eg. (os->android & Android App Id is required) or (os->ios & Apple App Id is required)',
+    );
   }
   // if (androidAppId) {
   //   endpoint = `${ENDPOINT}${androidAppId}`;

diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js
@@ -70,6 +70,9 @@ const identifyRequestHandler = async (message, category, destination) => {
 
   propertyPayload = removeUndefinedAndNullValues(propertyPayload);
   if (enforceEmailAsPrimary) {
+    if (!propertyPayload.email && !propertyPayload.phone_number) {
+      throw new InstrumentationError('None of email and phone are present in the payload');
+    }
     delete propertyPayload.external_id;
     customPropertyPayload = {
       ...customPropertyPayload,
@@ -78,8 +81,10 @@ const identifyRequestHandler = async (message, category, destination) => {
   }
   const data = {
     type: 'profile',
-    attributes: propertyPayload,
-    properties: removeUndefinedAndNullValues(customPropertyPayload),
+    attributes: {
+      ...propertyPayload,
+      properties: removeUndefinedAndNullValues(customPropertyPayload),
+    },
   };
   const payload = {
     data: removeUndefinedAndNullValues(data),
@@ -125,8 +130,6 @@ const trackRequestHandler = (message, category, destination) => {
     const eventName = eventNameMapping[event];
     const eventMap = jsonNameMapping[eventName];
     attributes.metric = { name: eventName };
-    // using identify to create customer properties
-    attributes.profile = createCustomerProperties(message);
     const categ = CONFIG_CATEGORIES[eventMap];
     attributes.properties = constructPayload(message.properties, MAPPING_CONFIG[categ.name]);
     attributes.properties = {
@@ -186,8 +189,9 @@ const trackRequestHandler = (message, category, destination) => {
       ...attributes.properties,
       ...populateCustomFieldsFromTraits(message),
     };
-    attributes.profile = createCustomerProperties(message);
   }
+  // Map user properties to profile object
+  attributes.profile = createCustomerProperties(message, destination.Config);
   if (message.timestamp) {
     attributes.time = message.timestamp;
   }

diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js
@@ -15,7 +15,7 @@ const {
 
 const { BASE_ENDPOINT, MAPPING_CONFIG, CONFIG_CATEGORIES, MAX_BATCH_SIZE } = require('./config');
 const { JSON_MIME_TYPE } = require('../../util/constant');
-const { NetworkError } = require('../../util/errorTypes');
+const { NetworkError, InstrumentationError } = require('../../util/errorTypes');
 const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils');
 const tags = require('../../util/tags');
 const { handleHttpRequest } = require('../../../adapters/network');
@@ -135,12 +135,23 @@ const subscribeUserToList = (message, traitsInfo, destination) => {
 };
 
 // This function is used for creating and returning customer properties using mapping json
-const createCustomerProperties = (message) => {
+const createCustomerProperties = (message, Config) => {
+  const { enforceEmailAsPrimary } = Config;
   let customerProperties = constructPayload(
     message,
     MAPPING_CONFIG[CONFIG_CATEGORIES.PROFILE.name],
   );
-  customerProperties.$id = getFieldValueFromMessage(message, 'userId');
+  if (!enforceEmailAsPrimary) {
+    customerProperties.$id = getFieldValueFromMessage(message, 'userId');
+  } else {
+    if (!customerProperties.$email && !customerProperties.$phone_number) {
+      throw new InstrumentationError('None of email and phone are present in the payload');
+    }
+    customerProperties = {
+      ...customerProperties,
+      _id: getFieldValueFromMessage(message, 'userId'),
+    };
+  }
   customerProperties = removeUndefinedAndNullValues(customerProperties);
   return customerProperties;
 };

diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js
@@ -305,6 +305,7 @@ const processGroupEvents = (message, type, destination) => {
           $set: {
             [groupKey]: groupKeyVal,
           },
+          $ip: get(message, 'context.ip'),
         };
 
         if (destination?.Config.identityMergeApi === 'simplified') {

diff --git a/src/v0/destinations/pinterest_tag/data/pinterestUserConfig.json b/src/v0/destinations/pinterest_tag/data/pinterestUserConfig.json
@@ -53,18 +53,12 @@
   {
     "destKey": "ct",
     "sourceKeys": ["traits.address.city", "context.traits.address.city"],
-    "required": false,
-    "metadata": {
-      "type": "toLower"
-    }
+    "required": false
   },
   {
     "destKey": "st",
     "sourceKeys": ["traits.address.state", "context.traits.address.state"],
-    "required": false,
-    "metadata": {
-      "type": "toLower"
-    }
+    "required": false
   },
   {
     "destKey": "zp",
@@ -74,10 +68,7 @@
   {
     "destKey": "country",
     "sourceKeys": ["traits.address.country", "context.traits.address.country"],
-    "required": false,
-    "metadata": {
-      "type": "toLower"
-    }
+    "required": false
   },
   {
     "destKey": "hashed_maids",

diff --git a/src/v0/destinations/pinterest_tag/utils.js b/src/v0/destinations/pinterest_tag/utils.js
@@ -30,6 +30,42 @@ const ecomEventMaps = [
 
 const USER_NON_ARRAY_PROPERTIES = ['client_user_agent', 'client_ip_address'];
 
+const getHashedValue = (key, value) => {
+  switch (key) {
+    case 'em':
+    case 'ct':
+    case 'st':
+    case 'country':
+    case 'ln':
+    case 'fn':
+    case 'ge':
+      value = Array.isArray(value)
+        ? value.map((val) => val.toString().toLowerCase())
+        : value.toString().toLowerCase();
+      break;
+    case 'ph':
+      // phone numbers should only contain digits & should not contain leading zeros
+      value = Array.isArray(value)
+        ? value.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, ''))
+        : value.toString().replace(/\D/g, '').replace(/^0+/, '');
+      break;
+    case 'zp':
+      // zip fields should only contain digits
+      value = Array.isArray(value)
+        ? value.map((val) => val.toString().replace(/\D/g, ''))
+        : value.toString().replace(/\D/g, '');
+      break;
+    case 'hashed_maids':
+    case 'external_id':
+    case 'db':
+      // no action needed on value
+      break;
+    default:
+      return String(value);
+  }
+  return Array.isArray(value) ? value.map((val) => sha256(val)) : [sha256(value)];
+};
+
 /**
  *
  * @param {*} userPayload Payload mapped from user fields
@@ -38,37 +74,8 @@ const USER_NON_ARRAY_PROPERTIES = ['client_user_agent', 'client_ip_address'];
  * Ref: https://s.pinimg.com/ct/docs/conversions_api/dist/v3.html
  */
 const processUserPayload = (userPayload) => {
-  let formatValue = '';
   Object.keys(userPayload).forEach((key) => {
-    switch (key) {
-      case 'em':
-        formatValue = userPayload[key].toString().toLowerCase();
-        userPayload[key] = [sha256(formatValue)];
-        break;
-      case 'ph':
-      case 'zp':
-        // zip fields should only contain digits
-        formatValue = userPayload[key].toString().replace(/\D/g, '');
-        if (key === 'ph') {
-          // phone numbers should not contain leading zeros
-          formatValue = formatValue.replace(/^0+/, '');
-        }
-        userPayload[key] = [sha256(formatValue)];
-        break;
-      case 'ct':
-      case 'st':
-      case 'country':
-      case 'ge':
-      case 'db':
-      case 'ln':
-      case 'fn':
-      case 'hashed_maids':
-      case 'external_id':
-        userPayload[key] = [sha256(userPayload[key])];
-        break;
-      default:
-        userPayload[key] = String(userPayload[key]);
-    }
+    userPayload[key] = getHashedValue(key, userPayload[key]);
   });
   return userPayload;
 };
@@ -286,15 +293,21 @@ const processHashedUserPayload = (userPayload, message) => {
   const processedHashedUserPayload = {};
   Object.keys(userPayload).forEach((key) => {
     if (!USER_NON_ARRAY_PROPERTIES.includes(key)) {
-      processedHashedUserPayload[key] = [userPayload[key]];
+      if (Array.isArray(userPayload[key])) {
+        processedHashedUserPayload[key] = [...userPayload[key]];
+      } else {
+        processedHashedUserPayload[key] = [userPayload[key]];
+      }
     } else {
       processedHashedUserPayload[key] = userPayload[key];
     }
   });
   // multiKeyMap will works on only specific values like m, male, MALE, f, F, Female
   // if hashed data is sent from the user, it is directly set over here
   const gender = message.traits?.gender || message.context?.traits?.gender;
-  if (gender) {
+  if (gender && Array.isArray(gender)) {
+    processedHashedUserPayload.ge = [...gender];
+  } else if (gender) {
     processedHashedUserPayload.ge = [gender];
   }
   return processedHashedUserPayload;

diff --git a/src/v0/destinations/pinterest_tag/utils.test.js b/src/v0/destinations/pinterest_tag/utils.test.js
@@ -0,0 +1,117 @@
+const { processUserPayload } = require('../../../../src/v0/destinations/pinterest_tag/utils');
+
+const userFields = [
+  {
+    em: '[email protected]',
+    ph: '+1234589947',
+    ge: 'male',
+    db: '19960314',
+    ln: 'Ganguly',
+    fn: 'Shrouti',
+    ct: 'Kolkata',
+    st: 'WB',
+    zp: '700114',
+    country: 'IN',
+    hashed_maids: 'abc123',
+    client_ip_address: '0.0.0.0',
+    client_user_agent: 'chrome',
+    external_id: '50be5c78-6c3f-4b60-be84-97805a316fb1',
+  },
+  {
+    em: ['[email protected]', '[email protected]'],
+    ph: ['+1234589947', '+1234589948'],
+    ln: ['Ganguly', 'Xu'],
+    db: ['19960314', '19960315'],
+    ge: ['female', 'male'],
+    fn: ['Shrouti', 'Alex'],
+    ct: ['Kolkata', 'Mumbai'],
+    st: ['WB', 'MH'],
+    zp: ['700114', '700115'],
+    country: ['IN', 'IN'],
+    hashed_maids: ['abc123', 'def123'],
+    client_ip_address: '0.0.0.0',
+    client_user_agent: 'chrome',
+    external_id: ['50be5c78-6c3f-4b60-be84-97805a316fb1', '50be5c78-6c3f-4b60-be84-97805a316fb2'],
+  },
+];
+
+const expectedOutput = [
+  {
+    em: ['48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08'],
+    ph: ['d164bbe036663cb5c96835e9ccc6501e9a521127ea62f6359744928ba932413b'],
+    ln: ['b507b6eb07a0166a64a6c06f5c684c732116d2b9c966e1176c3d7fcc1295bcc8'],
+    fn: ['d03a692ebd9ab84a8147d666baf05673c8113fa436f92e658a25ee306f383776'],
+    ct: ['6689106ca7922c30b2fd2c175c85bc7fc2d52cc4941bdd7bb622c6cdc6284a85'],
+    st: ['3b45022ab36728cdae12e709e945bba267c50ee8a91e6e4388539a8e03a3fdcd'],
+    zp: ['1a4292e00780e18d00e76fde9850aee5344e939ba593333cd5e4b4aa2cd33b0c'],
+    country: ['582967534d0f909d196b97f9e6921342777aea87b46fa52df165389db1fb8ccf'],
+    hashed_maids: ['6ca13d52ca70c883e0f0bb101e425a89e8624de51db2d2392593af6a84118090'],
+    ge: ['0d248e82c62c9386878327d491c762a002152d42ab2c391a31c44d9f62675ddf'],
+    external_id: ['3217d71a74c219d6e31e28267b313a7ceb6a2c032db1a091c9416b25b2ae2bc8'],
+    db: ['25f0dd07093aa653929e6e4d7710eeee84073ce654786612c09d1db67114c7ef'],
+    client_user_agent: 'chrome',
+    client_ip_address: '0.0.0.0',
+  },
+  {
+    em: [
+      '48ddb93f0b30c475423fe177832912c5bcdce3cc72872f8051627967ef278e08',
+      'c392e50ebeca7bea4405e9c545023451ac56620031f81263f681269bde14218b',
+    ],
+    ph: [
+      'd164bbe036663cb5c96835e9ccc6501e9a521127ea62f6359744928ba932413b',
+      '22bdde2594851294f2a6f4c34af704e68b398b03129ea9ceb58f0ffe33f6db52',
+    ],
+    ln: [
+      'b507b6eb07a0166a64a6c06f5c684c732116d2b9c966e1176c3d7fcc1295bcc8',
+      '9c2f138690fca4890c3c4a6691610fbbbdf32091cc001f7355cfdf574baa52b9',
+    ],
+    db: [
+      '25f0dd07093aa653929e6e4d7710eeee84073ce654786612c09d1db67114c7ef',
+      '66874ab18465ad8260b5276de8a6fc20ef56b5710e54be77ba2b7b0fb8d99540',
+    ],
+    ge: [
+      '9f165139a8c2894a47aea23b77d330eca847264224a44d5a17b19db8b9a72c08',
+      '0d248e82c62c9386878327d491c762a002152d42ab2c391a31c44d9f62675ddf',
+    ],
+    fn: [
+      'd03a692ebd9ab84a8147d666baf05673c8113fa436f92e658a25ee306f383776',
+      '4135aa9dc1b842a653dea846903ddb95bfb8c5a10c504a7fa16e10bc31d1fdf0',
+    ],
+    ct: [
+      '6689106ca7922c30b2fd2c175c85bc7fc2d52cc4941bdd7bb622c6cdc6284a85',
+      'd209bcc17778fd19fd2bc0c99a3868bf011da5162d3a75037a605768ebc276e2',
+    ],
+    st: [
+      '3b45022ab36728cdae12e709e945bba267c50ee8a91e6e4388539a8e03a3fdcd',
+      '1b0316ed1cfed044035c55363e02ccafab26d66b1c2746b94d17285f043324aa',
+    ],
+    zp: [
+      '1a4292e00780e18d00e76fde9850aee5344e939ba593333cd5e4b4aa2cd33b0c',
+      '4d6755aa1e85517191f06cc91448696c173e1195ae51f94a1670116ac7b5c47b',
+    ],
+    country: [
+      '582967534d0f909d196b97f9e6921342777aea87b46fa52df165389db1fb8ccf',
+      '582967534d0f909d196b97f9e6921342777aea87b46fa52df165389db1fb8ccf',
+    ],
+    hashed_maids: [
+      '6ca13d52ca70c883e0f0bb101e425a89e8624de51db2d2392593af6a84118090',
+      '78ecf2b02cdfee12360cd83f65d5a8bbecbb15de93705d9df1d4d100ca5a6030',
+    ],
+    client_ip_address: '0.0.0.0',
+    client_user_agent: 'chrome',
+    external_id: [
+      '3217d71a74c219d6e31e28267b313a7ceb6a2c032db1a091c9416b25b2ae2bc8',
+      '309dbb489ab5951ea3a1ef2a78f62ce902693fa56a474fe2407d564dbc21f83c',
+    ],
+  },
+];
+
+describe('format and hash the user fields', () => {
+  it('Should return the formatted and hashed (not for all fields) data in required format (values are all string)', () => {
+    expect(processUserPayload(userFields[0])).toEqual(expectedOutput[0]);
+  });
+
+  it('Should return the formatted and hashed (not for all fields) data in required format (values are array of strings)', () => {
+    expect(processUserPayload(userFields[1])).toEqual(expectedOutput[1]);
+  });
+});
-Original file line number
+Diff line change
@@ Expand Up @@
               $set: {
                 [groupKey]: groupKeyVal,
               },
+              $ip: get(message, 'context.ip'),
             };
             if (destination?.Config.identityMergeApi === 'simplified') {
@@ Expand Down @@