Merge pull request #1140 from JupiterOne/add-diff-duplicate-entity-re…

…port

add object diff to log with duplicate entity report

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to
 
 # Unreleased
 
+# 15.2.0 - 2024-12-04
+
+- runtime: Add diff object in the duplicate entity report for rawData and
+  properties mismatch
+
 # 15.1.0 - 2024-11-21
 
 - http-client: added retryCalculateDelay method to allow for custom delay

packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts

@@ -1,6 +1,7 @@
 import {
   InMemoryDuplicateKeyTracker,
   createDuplicateEntityReport,
+  diffObjects,
 } from './duplicateKeyTracker';
 import { InMemoryGraphObjectStore } from '../storage';
 import { Entity } from '@jupiterone/integration-sdk-core';
@@ -73,7 +74,7 @@ describe('createDuplicateEntityReport', () => {
 
     expect(der).toMatchObject({
       _key: 'test-key',
-      propertiesMatch: true,
+      entityPropertiesMatch: true,
       rawDataMatch: true,
     });
   });
@@ -130,8 +131,12 @@ describe('createDuplicateEntityReport', () => {
 
     expect(der).toMatchObject({
       _key: 'test-key',
-      propertiesMatch: false,
+      entityPropertiesMatch: false,
       rawDataMatch: true,
+      entityPropertiesDiff: JSON.stringify({
+        _class: { diffType: 'array_values_mismatch' },
+        _type: { diffType: 'value_mismatch' },
+      }),
     });
   });
 
@@ -183,8 +188,11 @@ describe('createDuplicateEntityReport', () => {
 
     expect(der).toMatchObject({
       _key: 'test-key',
-      propertiesMatch: true,
+      entityPropertiesMatch: true,
       rawDataMatch: false,
+      rawDataDiff: JSON.stringify({
+        data: { diffType: 'missing_in_original' },
+      }),
     });
   });
 
@@ -239,8 +247,110 @@ describe('createDuplicateEntityReport', () => {
 
     expect(der).toMatchObject({
       _key: 'test-key',
-      propertiesMatch: true,
+      entityPropertiesMatch: true,
       rawDataMatch: false,
+      rawDataDiff: JSON.stringify({
+        data: { diffType: 'missing_in_original' },
+      }),
     });
   });
 });
+
+describe('diffObjects', () => {
+  test('returns an empty diff for identical objects', () => {
+    const original = { name: 'Alice', age: 30 };
+    const duplicate = { name: 'Alice', age: 30 };
+
+    expect(diffObjects(original, duplicate)).toEqual({});
+  });
+
+  test('detects missing keys in original', () => {
+    const original = { name: 'Alice' };
+    const duplicate = { name: 'Alice', age: 30 };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      age: { diffType: 'missing_in_original' },
+    });
+  });
+
+  test('detects missing keys in duplicate', () => {
+    const original = { name: 'Alice', age: 30 };
+    const duplicate = { name: 'Alice' };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      age: { diffType: 'missing_in_duplicate' },
+    });
+  });
+
+  test('detects type mismatches', () => {
+    const original = { age: 30 };
+    const duplicate = { age: '30' };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      age: {
+        diffType: 'type_mismatch',
+        valueTypes: { original: 'number', duplicate: 'string' },
+      },
+    });
+  });
+
+  test('detects value mismatches', () => {
+    const original = { age: 30 };
+    const duplicate = { age: 31 };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      age: { diffType: 'value_mismatch' },
+    });
+  });
+
+  test('handles nested object differences', () => {
+    const original = { user: { name: 'Alice', age: 30 } };
+    const duplicate = { user: { name: 'Alice', age: 31 } };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      'user.age': { diffType: 'value_mismatch' },
+    });
+  });
+
+  test('handles missing nested keys in original', () => {
+    const original = { user: { name: 'Alice' } };
+    const duplicate = { user: { name: 'Alice', age: 30 } };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      'user.age': { diffType: 'missing_in_original' },
+    });
+  });
+
+  test('handles missing nested keys in duplicate', () => {
+    const original = { user: { name: 'Alice', age: 30 } };
+    const duplicate = { user: { name: 'Alice' } };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      'user.age': { diffType: 'missing_in_duplicate' },
+    });
+  });
+
+  test('handles array comparison', () => {
+    const original = { tags: ['a', 'b', 'c'], other: ['a', 'b', 'c'] };
+    const duplicate = { tags: ['a', 'b', 'd'], other: ['a', 'b', 'c'] };
+
+    expect(diffObjects(original, duplicate)).toEqual({
+      tags: {
+        diffType: 'array_values_mismatch',
+      },
+    });
+  });
+
+  test('handles empty objects', () => {
+    const original = {};
+    const duplicate = {};
+
+    expect(diffObjects(original, duplicate)).toEqual({});
+  });
+
+  test('handles null and undefined objects', () => {
+    expect(diffObjects(null, undefined)).toEqual({});
+    expect(diffObjects(undefined, {})).toEqual({});
+    expect(diffObjects({}, null)).toEqual({});
+  });
+});

packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts

@@ -131,9 +131,101 @@ function isDeepStrictEqual(a: any, b: any): boolean {
 export type DuplicateEntityReport = {
   _key: string;
   rawDataMatch: boolean;
-  propertiesMatch: boolean;
+  entityPropertiesMatch: boolean;
+  rawDataDiff?: string;
+  entityPropertiesDiff?: string;
+  diffErrors?: { rawData?: string; entityProperties?: string };
 };
 
+type DiffType =
+  | 'missing_in_original'
+  | 'missing_in_duplicate'
+  | 'type_mismatch'
+  | 'value_mismatch'
+  | 'array_values_mismatch';
+
+interface ObjectDiff {
+  [key: string]: {
+    type: DiffType;
+    valueTypes?: { src: string; dest: string };
+  };
+}
+
+/**
+ * Compares two objects and returns the differences between them.
+ *
+ * @param {unknown} originalObject - The source object to compare.
+ * @param {unknown} duplicateObject - The destination object to compare.
+ * @param {string} [path=''] - The base path for keys, used for tracking nested object differences.
+ * @returns {ObjectDiff} An object representing the differences between `original` and `duplicate`.
+ *   Each key corresponds to a path in the objects, with details about the type of difference.
+ *
+ * @example
+ * const originalObj = { a: 1, b: { c: 2 } };
+ * const duplicateObj = { a: 1, b: { c: 3 }, d: 4 };
+ * const result = diffObjects(originalObj, duplicateObj);
+ * console.log(result);
+ * // Output:
+ * // {
+ * //   "b.c": { type: "value_mismatch" },
+ * //   "d": { type: "missing_in_original" }
+ * // }
+ */
+export function diffObjects(
+  originalObject: unknown,
+  duplicateObject: unknown,
+  path: string = '',
+): ObjectDiff {
+  const diff = {};
+
+  // Helper to add differences
+  const addDiff = (
+    key: string,
+    diffType: DiffType,
+    valueTypes?: { original: string; duplicate: string },
+  ) => {
+    diff[key] = { diffType, valueTypes };
+  };
+
+  // Iterate through the keys of both objects
+  const allKeys = new Set([
+    ...Object.keys(originalObject || {}),
+    ...Object.keys(duplicateObject || {}),
+  ]);
+
+  const isObject = (val: unknown): val is Record<string, unknown> =>
+    typeof val === 'object' && val !== null;
+
+  for (const key of allKeys) {
+    const fullPath = path ? `${path}.${key}` : key;
+    const valOriginal = originalObject?.[key];
+    const valDuplicate = duplicateObject?.[key];
+
+    if (valOriginal === undefined) {
+      addDiff(fullPath, 'missing_in_original');
+    } else if (valDuplicate === undefined) {
+      addDiff(fullPath, 'missing_in_duplicate');
+    } else if (typeof valOriginal !== typeof valDuplicate) {
+      addDiff(fullPath, 'type_mismatch', {
+        original: typeof valOriginal,
+        duplicate: typeof valDuplicate,
+      });
+    } else if (Array.isArray(valOriginal) && Array.isArray(valDuplicate)) {
+      if (JSON.stringify(valOriginal) !== JSON.stringify(valDuplicate)) {
+        addDiff(fullPath, 'array_values_mismatch');
+      }
+    } else if (isObject(valOriginal) && isObject(valDuplicate)) {
+      // Recursive comparison for nested objects
+      const nestedDiff = diffObjects(valOriginal, valDuplicate, fullPath);
+      Object.assign(diff, nestedDiff);
+    } else if (valOriginal !== valDuplicate) {
+      addDiff(fullPath, 'value_mismatch');
+    }
+  }
+
+  return diff;
+}
+
 /**
  * compareEntities compares two entities and produces a DuplicateEntityReport describing their
  * similarities and differences.
@@ -144,12 +236,43 @@ export type DuplicateEntityReport = {
 function compareEntities(a: Entity, b: Entity): DuplicateEntityReport {
   const aClone = JSON.parse(JSON.stringify(a));
   const bClone = JSON.parse(JSON.stringify(b));
-  aClone._rawData = undefined;
-  bClone._rawData = undefined;
+  delete aClone._rawData;
+  delete bClone._rawData;
+
+  const rawDataMatch = isDeepStrictEqual(a._rawData, b._rawData);
+  const entityPropertiesMatch = isDeepStrictEqual(aClone, bClone);
+
+  const diffErrors: { rawData?: string; entityProperties?: string } = {};
+
+  let rawDataDiff: ObjectDiff | undefined;
+  if (!rawDataMatch) {
+    try {
+      rawDataDiff = diffObjects(
+        a._rawData?.[0].rawData,
+        b._rawData?.[0].rawData,
+      );
+    } catch (e) {
+      diffErrors.rawData = e.message;
+    }
+  }
+
+  let entityPropertiesDiff: ObjectDiff | undefined;
+  if (!entityPropertiesMatch) {
+    try {
+      entityPropertiesDiff = diffObjects(aClone, bClone);
+    } catch (e) {
+      diffErrors.entityProperties = e.message;
+    }
+  }
 
   return {
     _key: a._key,
-    rawDataMatch: isDeepStrictEqual(a._rawData, b._rawData),
-    propertiesMatch: isDeepStrictEqual(aClone, bClone),
+    rawDataMatch,
+    entityPropertiesMatch,
+    ...(rawDataDiff && { rawDataDiff: JSON.stringify(rawDataDiff) }),
+    ...(entityPropertiesDiff && {
+      entityPropertiesDiff: JSON.stringify(entityPropertiesDiff),
+    }),
+    ...(Object.keys(diffErrors).length > 0 && { diffErrors }),
   };
 }