add object diff to log with duplicate entity report

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to
 
 # Unreleased
 
+# 15.2.0 - 2024-12-04
+
+- runtime: Add diff object in the duplicate entity report for rawData and
+  properties mismatch
+
 # 15.1.0 - 2024-11-21
 
 - http-client: added retryCalculateDelay method to allow for custom delay

packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts

@@ -1,6 +1,7 @@
 import {
   InMemoryDuplicateKeyTracker,
   createDuplicateEntityReport,
+  diffObjects,
 } from './duplicateKeyTracker';
 import { InMemoryGraphObjectStore } from '../storage';
 import { Entity } from '@jupiterone/integration-sdk-core';
@@ -132,6 +133,10 @@ describe('createDuplicateEntityReport', () => {
       _key: 'test-key',
       propertiesMatch: false,
       rawDataMatch: true,
+      propertiesDiff: JSON.stringify({
+        _class: { type: 'value_mismatch' },
+        _type: { type: 'value_mismatch' },
+      }),
     });
   });
 
@@ -185,6 +190,9 @@ describe('createDuplicateEntityReport', () => {
       _key: 'test-key',
       propertiesMatch: true,
       rawDataMatch: false,
+      rawDataDiff: JSON.stringify({
+        data: { type: 'missing_in_src' },
+      }),
     });
   });
 
@@ -241,6 +249,108 @@ describe('createDuplicateEntityReport', () => {
       _key: 'test-key',
       propertiesMatch: true,
       rawDataMatch: false,
+      rawDataDiff: JSON.stringify({
+        data: { type: 'missing_in_src' },
+      }),
     });
   });
 });
+
+describe('diffObjects', () => {
+  test('returns an empty diff for identical objects', () => {
+    const src = { name: 'Alice', age: 30 };
+    const dest = { name: 'Alice', age: 30 };
+
+    expect(diffObjects(src, dest)).toEqual({});
+  });
+
+  test('detects missing keys in src', () => {
+    const src = { name: 'Alice' };
+    const dest = { name: 'Alice', age: 30 };
+
+    expect(diffObjects(src, dest)).toEqual({
+      age: { type: 'missing_in_src' },
+    });
+  });
+
+  test('detects missing keys in dest', () => {
+    const src = { name: 'Alice', age: 30 };
+    const dest = { name: 'Alice' };
+
+    expect(diffObjects(src, dest)).toEqual({
+      age: { type: 'missing_in_dest' },
+    });
+  });
+
+  test('detects type mismatches', () => {
+    const src = { age: 30 };
+    const dest = { age: '30' };
+
+    expect(diffObjects(src, dest)).toEqual({
+      age: {
+        type: 'type_mismatch',
+        valueTypes: { src: 'number', dest: 'string' },
+      },
+    });
+  });
+
+  test('detects value mismatches', () => {
+    const src = { age: 30 };
+    const dest = { age: 31 };
+
+    expect(diffObjects(src, dest)).toEqual({
+      age: { type: 'value_mismatch' },
+    });
+  });
+
+  test('handles nested object differences', () => {
+    const src = { user: { name: 'Alice', age: 30 } };
+    const dest = { user: { name: 'Alice', age: 31 } };
+
+    expect(diffObjects(src, dest)).toEqual({
+      'user.age': { type: 'value_mismatch' },
+    });
+  });
+
+  test('handles missing nested keys in src', () => {
+    const src = { user: { name: 'Alice' } };
+    const dest = { user: { name: 'Alice', age: 30 } };
+
+    expect(diffObjects(src, dest)).toEqual({
+      'user.age': { type: 'missing_in_src' },
+    });
+  });
+
+  test('handles missing nested keys in dest', () => {
+    const src = { user: { name: 'Alice', age: 30 } };
+    const dest = { user: { name: 'Alice' } };
+
+    expect(diffObjects(src, dest)).toEqual({
+      'user.age': { type: 'missing_in_dest' },
+    });
+  });
+
+  test('handles array comparison', () => {
+    const src = { tags: ['a', 'b', 'c'], other: ['a', 'b', 'c'] };
+    const dest = { tags: ['a', 'b', 'd'], other: ['a', 'b', 'c'] };
+
+    expect(diffObjects(src, dest)).toEqual({
+      tags: {
+        type: 'value_mismatch',
+      },
+    });
+  });
+
+  test('handles empty objects', () => {
+    const src = {};
+    const dest = {};
+
+    expect(diffObjects(src, dest)).toEqual({});
+  });
+
+  test('handles null and undefined objects', () => {
+    expect(diffObjects(null, undefined)).toEqual({});
+    expect(diffObjects(undefined, {})).toEqual({});
+    expect(diffObjects({}, null)).toEqual({});
+  });
+});

packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts

@@ -132,8 +132,98 @@ export type DuplicateEntityReport = {
   _key: string;
   rawDataMatch: boolean;
   propertiesMatch: boolean;
+  rawDataDiff?: string;
+  propertiesDiff?: string;
 };
 
+type DiffType =
+  | 'missing_in_src'
+  | 'missing_in_dest'
+  | 'type_mismatch'
+  | 'value_mismatch';
+
+interface ObjectDiff {
+  [key: string]: {
+    type: DiffType;
+    valueTypes?: { src: string; dest: string };
+  };
+}
+
+/**
+ * Compares two objects and returns the differences between them.
+ *
+ * @param {unknown} src - The source object to compare.
+ * @param {unknown} dest - The destination object to compare.
+ * @param {string} [path=''] - The base path for keys, used for tracking nested object differences.
+ * @returns {ObjectDiff} An object representing the differences between `src` and `dest`.
+ *   Each key corresponds to a path in the objects, with details about the type of difference.
+ *
+ * @example
+ * const src = { a: 1, b: { c: 2 } };
+ * const dest = { a: 1, b: { c: 3 }, d: 4 };
+ * const result = diffObjects(src, dest);
+ * console.log(result);
+ * // Output:
+ * // {
+ * //   "b.c": { type: "value_mismatch" },
+ * //   "d": { type: "missing_in_src" }
+ * // }
+ */
+export function diffObjects(
+  src: unknown,
+  dest: unknown,
+  path: string = '',
+): ObjectDiff {
+  const diff = {};
+
+  // Helper to add differences
+  const addDiff = (
+    key: string,
+    type: DiffType,
+    valueTypes?: { src: string; dest: string },
+  ) => {
+    diff[key] = { type, valueTypes };
+  };
+
+  // Iterate through the keys of both objects
+  const allKeys = new Set([
+    ...Object.keys(src || {}),
+    ...Object.keys(dest || {}),
+  ]);
+
+  const isObject = (val: unknown): val is Record<string, unknown> =>
+    typeof val === 'object' && val !== null;
+
+  for (const key of allKeys) {
+    const fullPath = path ? `${path}.${key}` : key;
+    const valSrc = src?.[key];
+    const valDest = dest?.[key];
+
+    if (valSrc === undefined) {
+      addDiff(fullPath, 'missing_in_src');
+    } else if (valDest === undefined) {
+      addDiff(fullPath, 'missing_in_dest');
+    } else if (typeof valSrc !== typeof valDest) {
+      addDiff(fullPath, 'type_mismatch', {
+        src: typeof valSrc,
+        dest: typeof valDest,
+      });
+    } else if (Array.isArray(valSrc) && Array.isArray(valDest)) {
+      if (JSON.stringify(valSrc) !== JSON.stringify(valDest)) {
+        addDiff(fullPath, 'value_mismatch');
+      }
+    } else if (isObject(valSrc) && isObject(valDest)) {
+      // Recursive comparison for nested objects
+      const nestedDiff = diffObjects(valSrc, valDest, fullPath);
+      Object.assign(diff, nestedDiff);
+    } else if (valSrc !== valDest) {
+      addDiff(fullPath, 'value_mismatch');
+    }
+  }
+
+  return diff;
+}
+
 /**
  * compareEntities compares two entities and produces a DuplicateEntityReport describing their
  * similarities and differences.
@@ -144,12 +234,38 @@ export type DuplicateEntityReport = {
 function compareEntities(a: Entity, b: Entity): DuplicateEntityReport {
   const aClone = JSON.parse(JSON.stringify(a));
   const bClone = JSON.parse(JSON.stringify(b));
-  aClone._rawData = undefined;
-  bClone._rawData = undefined;
+  delete aClone._rawData;
+  delete bClone._rawData;
+
+  const rawDataMatch = isDeepStrictEqual(a._rawData, b._rawData);
+  const propertiesMatch = isDeepStrictEqual(aClone, bClone);
+
+  let rawDataDiff: ObjectDiff | undefined;
+  if (!rawDataMatch) {
+    try {
+      rawDataDiff = diffObjects(
+        a._rawData?.[0].rawData,
+        b._rawData?.[0].rawData,
+      );
+    } catch (e) {
+      // ignore
+    }
+  }
+
+  let propertiesDiff: ObjectDiff | undefined;
+  if (!propertiesMatch) {
+    try {
+      propertiesDiff = diffObjects(aClone, bClone);
+    } catch (e) {
+      // ignore
+    }
+  }
 
   return {
     _key: a._key,
-    rawDataMatch: isDeepStrictEqual(a._rawData, b._rawData),
-    propertiesMatch: isDeepStrictEqual(aClone, bClone),
+    rawDataMatch,
+    propertiesMatch,
+    ...(rawDataDiff && { rawDataDiff: JSON.stringify(rawDataDiff) }),
+    ...(propertiesDiff && { propertiesDiff: JSON.stringify(propertiesDiff) }),
   };
 }