diff --git a/CHANGELOG.md b/CHANGELOG.md index bab17024..fd379d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to # Unreleased +# 15.2.0 - 2024-12-04 + +- runtime: Add diff object in the duplicate entity report for rawData and + properties mismatch + # 15.1.0 - 2024-11-21 - http-client: added retryCalculateDelay method to allow for custom delay diff --git a/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts b/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts index 23de3cb2..0a08e53d 100644 --- a/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts +++ b/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.test.ts @@ -1,6 +1,7 @@ import { InMemoryDuplicateKeyTracker, createDuplicateEntityReport, + diffObjects, } from './duplicateKeyTracker'; import { InMemoryGraphObjectStore } from '../storage'; import { Entity } from '@jupiterone/integration-sdk-core'; @@ -73,7 +74,7 @@ describe('createDuplicateEntityReport', () => { expect(der).toMatchObject({ _key: 'test-key', - propertiesMatch: true, + entityPropertiesMatch: true, rawDataMatch: true, }); }); @@ -130,8 +131,12 @@ describe('createDuplicateEntityReport', () => { expect(der).toMatchObject({ _key: 'test-key', - propertiesMatch: false, + entityPropertiesMatch: false, rawDataMatch: true, + entityPropertiesDiff: JSON.stringify({ + _class: { diffType: 'array_values_mismatch' }, + _type: { diffType: 'value_mismatch' }, + }), }); }); @@ -183,8 +188,11 @@ describe('createDuplicateEntityReport', () => { expect(der).toMatchObject({ _key: 'test-key', - propertiesMatch: true, + entityPropertiesMatch: true, rawDataMatch: false, + rawDataDiff: JSON.stringify({ + data: { diffType: 'missing_in_original' }, + }), }); }); @@ -239,8 +247,110 @@ describe('createDuplicateEntityReport', () => { expect(der).toMatchObject({ _key: 'test-key', - propertiesMatch: true, + entityPropertiesMatch: true, rawDataMatch: false, + rawDataDiff: JSON.stringify({ + data: { diffType: 'missing_in_original' }, + }), }); }); }); + +describe('diffObjects', () => { + test('returns an empty diff for identical objects', () => { + const original = { name: 'Alice', age: 30 }; + const duplicate = { name: 'Alice', age: 30 }; + + expect(diffObjects(original, duplicate)).toEqual({}); + }); + + test('detects missing keys in original', () => { + const original = { name: 'Alice' }; + const duplicate = { name: 'Alice', age: 30 }; + + expect(diffObjects(original, duplicate)).toEqual({ + age: { diffType: 'missing_in_original' }, + }); + }); + + test('detects missing keys in duplicate', () => { + const original = { name: 'Alice', age: 30 }; + const duplicate = { name: 'Alice' }; + + expect(diffObjects(original, duplicate)).toEqual({ + age: { diffType: 'missing_in_duplicate' }, + }); + }); + + test('detects type mismatches', () => { + const original = { age: 30 }; + const duplicate = { age: '30' }; + + expect(diffObjects(original, duplicate)).toEqual({ + age: { + diffType: 'type_mismatch', + valueTypes: { original: 'number', duplicate: 'string' }, + }, + }); + }); + + test('detects value mismatches', () => { + const original = { age: 30 }; + const duplicate = { age: 31 }; + + expect(diffObjects(original, duplicate)).toEqual({ + age: { diffType: 'value_mismatch' }, + }); + }); + + test('handles nested object differences', () => { + const original = { user: { name: 'Alice', age: 30 } }; + const duplicate = { user: { name: 'Alice', age: 31 } }; + + expect(diffObjects(original, duplicate)).toEqual({ + 'user.age': { diffType: 'value_mismatch' }, + }); + }); + + test('handles missing nested keys in original', () => { + const original = { user: { name: 'Alice' } }; + const duplicate = { user: { name: 'Alice', age: 30 } }; + + expect(diffObjects(original, duplicate)).toEqual({ + 'user.age': { diffType: 'missing_in_original' }, + }); + }); + + test('handles missing nested keys in duplicate', () => { + const original = { user: { name: 'Alice', age: 30 } }; + const duplicate = { user: { name: 'Alice' } }; + + expect(diffObjects(original, duplicate)).toEqual({ + 'user.age': { diffType: 'missing_in_duplicate' }, + }); + }); + + test('handles array comparison', () => { + const original = { tags: ['a', 'b', 'c'], other: ['a', 'b', 'c'] }; + const duplicate = { tags: ['a', 'b', 'd'], other: ['a', 'b', 'c'] }; + + expect(diffObjects(original, duplicate)).toEqual({ + tags: { + diffType: 'array_values_mismatch', + }, + }); + }); + + test('handles empty objects', () => { + const original = {}; + const duplicate = {}; + + expect(diffObjects(original, duplicate)).toEqual({}); + }); + + test('handles null and undefined objects', () => { + expect(diffObjects(null, undefined)).toEqual({}); + expect(diffObjects(undefined, {})).toEqual({}); + expect(diffObjects({}, null)).toEqual({}); + }); +}); diff --git a/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts b/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts index 676cd59f..227e27d1 100644 --- a/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts +++ b/packages/integration-sdk-runtime/src/execution/duplicateKeyTracker.ts @@ -131,9 +131,101 @@ function isDeepStrictEqual(a: any, b: any): boolean { export type DuplicateEntityReport = { _key: string; rawDataMatch: boolean; - propertiesMatch: boolean; + entityPropertiesMatch: boolean; + rawDataDiff?: string; + entityPropertiesDiff?: string; + diffErrors?: { rawData?: string; entityProperties?: string }; }; +type DiffType = + | 'missing_in_original' + | 'missing_in_duplicate' + | 'type_mismatch' + | 'value_mismatch' + | 'array_values_mismatch'; + +interface ObjectDiff { + [key: string]: { + type: DiffType; + valueTypes?: { src: string; dest: string }; + }; +} + +/** + * Compares two objects and returns the differences between them. + * + * @param {unknown} originalObject - The source object to compare. + * @param {unknown} duplicateObject - The destination object to compare. + * @param {string} [path=''] - The base path for keys, used for tracking nested object differences. + * @returns {ObjectDiff} An object representing the differences between `original` and `duplicate`. + * Each key corresponds to a path in the objects, with details about the type of difference. + * + * @example + * const originalObj = { a: 1, b: { c: 2 } }; + * const duplicateObj = { a: 1, b: { c: 3 }, d: 4 }; + * const result = diffObjects(originalObj, duplicateObj); + * console.log(result); + * // Output: + * // { + * // "b.c": { type: "value_mismatch" }, + * // "d": { type: "missing_in_original" } + * // } + */ +export function diffObjects( + originalObject: unknown, + duplicateObject: unknown, + path: string = '', +): ObjectDiff { + const diff = {}; + + // Helper to add differences + const addDiff = ( + key: string, + diffType: DiffType, + valueTypes?: { original: string; duplicate: string }, + ) => { + diff[key] = { diffType, valueTypes }; + }; + + // Iterate through the keys of both objects + const allKeys = new Set([ + ...Object.keys(originalObject || {}), + ...Object.keys(duplicateObject || {}), + ]); + + const isObject = (val: unknown): val is Record => + typeof val === 'object' && val !== null; + + for (const key of allKeys) { + const fullPath = path ? `${path}.${key}` : key; + const valOriginal = originalObject?.[key]; + const valDuplicate = duplicateObject?.[key]; + + if (valOriginal === undefined) { + addDiff(fullPath, 'missing_in_original'); + } else if (valDuplicate === undefined) { + addDiff(fullPath, 'missing_in_duplicate'); + } else if (typeof valOriginal !== typeof valDuplicate) { + addDiff(fullPath, 'type_mismatch', { + original: typeof valOriginal, + duplicate: typeof valDuplicate, + }); + } else if (Array.isArray(valOriginal) && Array.isArray(valDuplicate)) { + if (JSON.stringify(valOriginal) !== JSON.stringify(valDuplicate)) { + addDiff(fullPath, 'array_values_mismatch'); + } + } else if (isObject(valOriginal) && isObject(valDuplicate)) { + // Recursive comparison for nested objects + const nestedDiff = diffObjects(valOriginal, valDuplicate, fullPath); + Object.assign(diff, nestedDiff); + } else if (valOriginal !== valDuplicate) { + addDiff(fullPath, 'value_mismatch'); + } + } + + return diff; +} + /** * compareEntities compares two entities and produces a DuplicateEntityReport describing their * similarities and differences. @@ -144,12 +236,43 @@ export type DuplicateEntityReport = { function compareEntities(a: Entity, b: Entity): DuplicateEntityReport { const aClone = JSON.parse(JSON.stringify(a)); const bClone = JSON.parse(JSON.stringify(b)); - aClone._rawData = undefined; - bClone._rawData = undefined; + delete aClone._rawData; + delete bClone._rawData; + + const rawDataMatch = isDeepStrictEqual(a._rawData, b._rawData); + const entityPropertiesMatch = isDeepStrictEqual(aClone, bClone); + + const diffErrors: { rawData?: string; entityProperties?: string } = {}; + + let rawDataDiff: ObjectDiff | undefined; + if (!rawDataMatch) { + try { + rawDataDiff = diffObjects( + a._rawData?.[0].rawData, + b._rawData?.[0].rawData, + ); + } catch (e) { + diffErrors.rawData = e.message; + } + } + + let entityPropertiesDiff: ObjectDiff | undefined; + if (!entityPropertiesMatch) { + try { + entityPropertiesDiff = diffObjects(aClone, bClone); + } catch (e) { + diffErrors.entityProperties = e.message; + } + } return { _key: a._key, - rawDataMatch: isDeepStrictEqual(a._rawData, b._rawData), - propertiesMatch: isDeepStrictEqual(aClone, bClone), + rawDataMatch, + entityPropertiesMatch, + ...(rawDataDiff && { rawDataDiff: JSON.stringify(rawDataDiff) }), + ...(entityPropertiesDiff && { + entityPropertiesDiff: JSON.stringify(entityPropertiesDiff), + }), + ...(Object.keys(diffErrors).length > 0 && { diffErrors }), }; }