Skip to content

Commit e1e5a43

Browse files
committed
changed method for caseNormalizer
1 parent 8933c66 commit e1e5a43

File tree

1 file changed

+3
-1
lines changed

1 file changed

+3
-1
lines changed

spark/core/src/main/java/zingg/spark/core/preprocess/casenormalize/SparkCaseNormalizer.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,12 @@ public SparkCaseNormalizer(IContext<SparkSession, Dataset<Row>, Row, Column, Dat
2929
protected ZFrame<Dataset<Row>, Row, Column> applyCaseNormalizer(ZFrame<Dataset<Row>, Row, Column> incomingDataFrame, List<String> relevantFields) {
3030
String[] incomingDFColumns = incomingDataFrame.columns();
3131
Column[] caseNormalizedValues = new Column[relevantFields.size()];
32+
String[] relevantFieldsArray = new String[relevantFields.size()];
3233
for (int idx = 0; idx < relevantFields.size(); idx++) {
3334
caseNormalizedValues[idx] = lower(incomingDataFrame.col(relevantFields.get(idx)));
35+
relevantFieldsArray[idx] = relevantFields.get(idx);
3436
}
3537

36-
return incomingDataFrame.withColumns(incomingDFColumns, caseNormalizedValues).select(incomingDFColumns);
38+
return incomingDataFrame.withColumns(relevantFieldsArray, caseNormalizedValues).select(incomingDFColumns);
3739
}
3840
}

0 commit comments

Comments
 (0)