diff --git a/packages/shared/src/models/dataPipe/operators/filter.spec.ts b/packages/shared/src/models/dataPipe/operators/filter.spec.ts index d1ee031cf..8a14f998f 100644 --- a/packages/shared/src/models/dataPipe/operators/filter.spec.ts +++ b/packages/shared/src/models/dataPipe/operators/filter.spec.ts @@ -23,10 +23,10 @@ describe("Filter", () => { expect(outputIDs).toEqual(["id_1", "id_3", "id_4"]); }); it("Filters with 'this' context", () => { - const nestedData = testData.names.map((entry) => { - entry["nested"] = { first_name: entry.first_name }; - return entry; - }); + const nestedData = testData.names.map((entry) => ({ + ...entry, + nested: { first_name: entry.first_name }, + })); const testDf = new DataFrame(nestedData); const output = new filter(testDf, ["this.nested.first_name === 'Ada'"]).apply(); const outputIDs = output.column("first_name").values; diff --git a/packages/shared/src/models/dataPipe/operators/merge.spec.ts b/packages/shared/src/models/dataPipe/operators/merge.spec.ts index 6d0cfb9ac..ca65ebacf 100644 --- a/packages/shared/src/models/dataPipe/operators/merge.spec.ts +++ b/packages/shared/src/models/dataPipe/operators/merge.spec.ts @@ -1,9 +1,9 @@ import { DataFrame } from "danfojs"; -import { DataPipe } from "../pipe"; import testData from "../testData.spec"; +import { DataPipe } from "../pipe"; import merge from "./merge"; -(testData as any).merge_nationality = [ +const nationality_data = [ { id: "invalid_id", nationality: "German", @@ -19,9 +19,18 @@ import merge from "./merge"; }, ]; +const nested_data = [ + { + id: "id_1", + nested: { + value: "test", + }, + }, +]; + describe("Merge Operator", () => { const testDf = new DataFrame(testData.names); - const testPipe: DataPipe = new DataPipe([], testData); + const testPipe = new DataPipe([], { ...testData, nationality_data, nested_data }); it("Throws on missing list", () => { // throws on missing list @@ -31,7 +40,7 @@ describe("Merge Operator", () => { }); it("Merges multiple lists", () => { // merges data - additional nationality column appended for all entries and populated for available - const output = new merge(testDf, ["merge_nationality"], testPipe).apply(); + const output = new merge(testDf, ["nationality_data"], testPipe).apply(); expect(output.index).toEqual(["id_1", "id_2", "id_3", "id_4"]); // merges new nationality column const expectedNationalities = ["British", "French", undefined, undefined]; @@ -40,4 +49,13 @@ describe("Merge Operator", () => { const expectedNames = ["override", "Blaise", "Charles", "Daniel"]; expect(output.column("first_name").values).toEqual(expectedNames); }); + it("Merges multiple lists including list with nested data", () => { + // merges data - additional nationality column appended for all entries and populated for available + const output = new merge(testDf, ["nested_data"], testPipe).apply(); + expect(output.index).toEqual(["id_1", "id_2", "id_3", "id_4"]); + // merges nested column + // TODO - danfo stringifies nested data, should it be handled differently? + const expectedNested = ['{"value":"test"}', "undefined", "undefined", "undefined"]; + expect(output.column("nested").values).toEqual(expectedNested as any); + }); }); diff --git a/packages/shared/src/models/dataPipe/operators/merge.ts b/packages/shared/src/models/dataPipe/operators/merge.ts index 0be8d0922..c7864467c 100644 --- a/packages/shared/src/models/dataPipe/operators/merge.ts +++ b/packages/shared/src/models/dataPipe/operators/merge.ts @@ -22,6 +22,8 @@ class MergeOperator extends BaseOperator { } apply() { + if (this.df.shape[0] === 0) + console.error("Merge: No data in base dataframe - an input_source must be provided"); setIndexColumn(this.df, this.indexColumn); for (const dataList of this.args_list) { this.df = this.replaceUpdatedValues(dataList); @@ -49,20 +51,20 @@ class MergeOperator extends BaseOperator { /** Replace any values updated from the data in the original dataframe **/ private replaceUpdatedValues(data: any[]) { - const replacments = new DataFrame(data); - setIndexColumn(replacments, this.indexColumn); + const replacements = new DataFrame(data); + setIndexColumn(replacements, this.indexColumn); // remove any columns that does not exist in left - const droppedColumns = replacments.columns.filter( + const droppedColumns = replacements.columns.filter( (column) => column !== this.indexColumn && !this.df.columns.includes(column) ); - replacments.drop({ columns: droppedColumns, inplace: true }); + replacements.drop({ columns: droppedColumns, inplace: true }); // remove any rows that does not exist in left - const droppedIndexes = replacments.index.filter((i) => !this.df.index.includes(i)); - replacments.drop({ index: droppedIndexes, inplace: true }); + const droppedIndexes = replacements.index.filter((i) => !this.df.index.includes(i)); + replacements.drop({ index: droppedIndexes, inplace: true }); - // replace all values in left with values from replacments where defined - const replaceHashmap = arrayToHashmap(toJSON(replacments) as any, this.indexColumn); + // replace all values in left with values from replacements where defined + const replaceHashmap = arrayToHashmap(toJSON(replacements) as any, this.indexColumn); // handle replacement by looping over all rows and replacing values where override defined const replaceDf = this.df.apply((row: any[]) => {