Merge pull request #2342 from IDEMSInternational/fix/data-pipe-merge

fix: data pipe merge operator handles case of no input source

packages/shared/src/models/dataPipe/operators/filter.spec.ts

@@ -23,10 +23,10 @@ describe("Filter", () => {
     expect(outputIDs).toEqual(["id_1", "id_3", "id_4"]);
   });
   it("Filters with 'this' context", () => {
-    const nestedData = testData.names.map((entry) => {
-      entry["nested"] = { first_name: entry.first_name };
-      return entry;
-    });
+    const nestedData = testData.names.map((entry) => ({
+      ...entry,
+      nested: { first_name: entry.first_name },
+    }));
     const testDf = new DataFrame(nestedData);
     const output = new filter(testDf, ["this.nested.first_name === 'Ada'"]).apply();
     const outputIDs = output.column("first_name").values;

packages/shared/src/models/dataPipe/operators/merge.spec.ts

@@ -1,9 +1,9 @@
 import { DataFrame } from "danfojs";
-import { DataPipe } from "../pipe";
 import testData from "../testData.spec";
+import { DataPipe } from "../pipe";
 import merge from "./merge";
 
-(testData as any).merge_nationality = [
+const nationality_data = [
   {
     id: "invalid_id",
     nationality: "German",
@@ -19,9 +19,18 @@ import merge from "./merge";
   },
 ];
 
+const nested_data = [
+  {
+    id: "id_1",
+    nested: {
+      value: "test",
+    },
+  },
+];
+
 describe("Merge Operator", () => {
   const testDf = new DataFrame(testData.names);
-  const testPipe: DataPipe = new DataPipe([], testData);
+  const testPipe = new DataPipe([], { ...testData, nationality_data, nested_data });
 
   it("Throws on missing list", () => {
     // throws on missing list
@@ -31,7 +40,7 @@ describe("Merge Operator", () => {
   });
   it("Merges multiple lists", () => {
     // merges data - additional nationality column appended for all entries and populated for available
-    const output = new merge(testDf, ["merge_nationality"], testPipe).apply();
+    const output = new merge(testDf, ["nationality_data"], testPipe).apply();
     expect(output.index).toEqual(["id_1", "id_2", "id_3", "id_4"]);
     // merges new nationality column
     const expectedNationalities = ["British", "French", undefined, undefined];
@@ -40,4 +49,13 @@ describe("Merge Operator", () => {
     const expectedNames = ["override", "Blaise", "Charles", "Daniel"];
     expect(output.column("first_name").values).toEqual(expectedNames);
   });
+  it("Merges multiple lists including list with nested data", () => {
+    // merges data - additional nationality column appended for all entries and populated for available
+    const output = new merge(testDf, ["nested_data"], testPipe).apply();
+    expect(output.index).toEqual(["id_1", "id_2", "id_3", "id_4"]);
+    // merges nested column
+    // TODO - danfo stringifies nested data, should it be handled differently?
+    const expectedNested = ['{"value":"test"}', "undefined", "undefined", "undefined"];
+    expect(output.column("nested").values).toEqual(expectedNested as any);
+  });
 });

packages/shared/src/models/dataPipe/operators/merge.ts

@@ -22,6 +22,8 @@ class MergeOperator extends BaseOperator {
   }
 
   apply() {
+    if (this.df.shape[0] === 0)
+      console.error("Merge: No data in base dataframe - an input_source must be provided");
     setIndexColumn(this.df, this.indexColumn);
     for (const dataList of this.args_list) {
       this.df = this.replaceUpdatedValues(dataList);
@@ -49,20 +51,20 @@ class MergeOperator extends BaseOperator {
 
   /** Replace any values updated from the data in the original dataframe **/
   private replaceUpdatedValues(data: any[]) {
-    const replacments = new DataFrame(data);
-    setIndexColumn(replacments, this.indexColumn);
+    const replacements = new DataFrame(data);
+    setIndexColumn(replacements, this.indexColumn);
 
     // remove any columns that does not exist in left
-    const droppedColumns = replacments.columns.filter(
+    const droppedColumns = replacements.columns.filter(
       (column) => column !== this.indexColumn && !this.df.columns.includes(column)
     );
-    replacments.drop({ columns: droppedColumns, inplace: true });
+    replacements.drop({ columns: droppedColumns, inplace: true });
     // remove any rows that does not exist in left
-    const droppedIndexes = replacments.index.filter((i) => !this.df.index.includes(i));
-    replacments.drop({ index: droppedIndexes, inplace: true });
+    const droppedIndexes = replacements.index.filter((i) => !this.df.index.includes(i));
+    replacements.drop({ index: droppedIndexes, inplace: true });
 
-    // replace all values in left with values from replacments where defined
-    const replaceHashmap = arrayToHashmap(toJSON(replacments) as any, this.indexColumn);
+    // replace all values in left with values from replacements where defined
+    const replaceHashmap = arrayToHashmap(toJSON(replacements) as any, this.indexColumn);
 
     // handle replacement by looping over all rows and replacing values where override defined
     const replaceDf = this.df.apply((row: any[]) => {