Skip to content

Commit

Permalink
Merge pull request #896 from zinggAI/revert-895-main
Browse files Browse the repository at this point in the history
Revert "transpose the ZFrame"
  • Loading branch information
sonalgoyal authored Sep 17, 2024
2 parents 3fce6e3 + 1eb6be1 commit 5e299ba
Show file tree
Hide file tree
Showing 5 changed files with 6 additions and 101 deletions.
7 changes: 0 additions & 7 deletions common/client/src/main/java/zingg/common/client/ZFrame.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ public interface ZFrame<D, R, C> {

public static final String COL_COUNT = "count";
public static final String COL_VALUE = "VALUE";
public static String PIVOT_COLUMN = "field";
public static String VALUE_1 = "value1";
public static String VALUE_2 = "value2";
public static String ORDER = "order";

public ZFrame<D, R, C> cache();
public ZFrame<D, R, C> as(String s);
Expand Down Expand Up @@ -183,7 +179,4 @@ public interface ZFrame<D, R, C> {

public C gt(C column1, C column2);

public ZFrame<D, R, C> transpose(String pivotColumn);

public void showVertical();
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ public String getMsg2(double prediction, double score) {
public void displayRecords(ZFrame<D, R, C> records, String preMessage, String postMessage) {
//System.out.println();
System.out.println(preMessage);
// showHorizontal(records);
records.showVertical();
records.show(false);
System.out.println(postMessage);
System.out.println("\tWhat do you think? Your choices are: ");
System.out.println();
Expand Down Expand Up @@ -126,4 +125,6 @@ public ILabelDataViewHelper<S, D, R, C> getLabelDataViewHelper() throws Unsuppor
return this;
}



}

This file was deleted.

63 changes: 3 additions & 60 deletions spark/client/src/main/java/zingg/spark/client/SparkFrame.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
package zingg.spark.client;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.spark.internal.config.R;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
Expand All @@ -15,8 +14,6 @@
import zingg.common.client.FieldData;
import zingg.common.client.ZFrame;
import zingg.common.client.util.ColName;
import zingg.common.core.util.ListConverter;
import zingg.spark.client.util.ExtendedFunction;

//Dataset, Row, column
public class SparkFrame implements ZFrame<Dataset<Row>, Row, Column> {
Expand Down Expand Up @@ -461,7 +458,7 @@ public ZFrame<Dataset<Row>, Row, Column> groupByCount(String groupByCol1, String

@Override
public ZFrame<Dataset<Row>, Row, Column> intersect(ZFrame<Dataset<Row>, Row, Column> other) {
return new SparkFrame(df.intersect(other.df()));
return new SparkFrame(df.intersect(other.df()));
}

@Override
Expand All @@ -473,60 +470,6 @@ public Column substr(Column col, int startPos, int len) {
public Column gt(Column column1, Column column2) {
return column1.gt(column2);
}

@Override
public ZFrame<Dataset<Row>, Row, Column> transpose(String pivotColumn) {
ExtendedFunction extendedFunction = new ExtendedFunction();
List<String> columnsExceptPivot = new ArrayList<>(List.of(df.columns()));
columnsExceptPivot.remove(pivotColumn);
ListConverter<String> listConverter = new ListConverter<String>();
Dataset<Row> r = extendedFunction.TransposeDF(df, listConverter.convertListToSeq(columnsExceptPivot), pivotColumn);
return new SparkFrame(r);
}

/***
* Add auto incremental row like {1, 2, 3, 4, 5} to the dataframe
* @return ZFrame
*/

private ZFrame<Dataset<Row>, Row, Column> addAutoIncrementalRow() {
String[] columns = df.columns();
Dataset<Row> temporaryDF = df.limit(1);
List<String> monotonicIncreasing = new ArrayList<>();
for (int idx = 0; idx < columns.length; idx++) {
monotonicIncreasing.add(String.valueOf(idx));
}
Collections.sort(monotonicIncreasing);
for (int idx = 0; idx < columns.length; idx++) {
temporaryDF = temporaryDF.withColumn(columns[idx], functions.lit(monotonicIncreasing.get(idx)));
}
return new SparkFrame(df.union(temporaryDF));
}

@Override
public void showVertical() {
ZFrame<Dataset<Row>, Row, Column> headerIncludedFrame = getHeaderIncludedDataFrame(new SparkFrame(df));
ZFrame<Dataset<Row>, Row, Column> vertical = headerIncludedFrame.transpose(PIVOT_COLUMN);
vertical.sortAscending(ORDER).drop(ORDER).show(1000);
}

/***
* return new ZFrame with new Column added as PIVOT used for transposing the matrix
* @param records
* @return header included zFrame
*/
private ZFrame<Dataset<Row>, Row, Column> getHeaderIncludedDataFrame(ZFrame<Dataset<Row>, Row, Column> records) {
ZFrame<Dataset<Row>, Row, Column> orderedRowAdded = addAutoIncrementalRow();

ZFrame<Dataset<Row>, Row, Column> firstRecord = orderedRowAdded.limit(1);
ZFrame<Dataset<Row>, Row, Column> secondRecord = orderedRowAdded.except(firstRecord).limit(1);
ZFrame<Dataset<Row>, Row, Column> thirdRecord = orderedRowAdded.except(firstRecord.union(secondRecord));

//return new ZFrame with Field column added to be used as pivot
return firstRecord.withColumn(PIVOT_COLUMN, VALUE_1).
union(secondRecord.withColumn(PIVOT_COLUMN, VALUE_2)).
union(thirdRecord.withColumn(PIVOT_COLUMN, ORDER));
}


}

This file was deleted.

0 comments on commit 5e299ba

Please sign in to comment.