Skip to content

Commit

Permalink
relation: add support for selectAll() (#3032)
Browse files Browse the repository at this point in the history 
* relation: add relation::distinct() to handlers

* datacube: minor cleanup

* relation: add support for selectAll

* datacube: make sure of selectAll() syntax
  • Loading branch information
@akphi
akphi authored Aug 23, 2024
1 parent bbb53ea commit 89a39a6
Show file tree
Hide file tree
Showing 14 changed files with 95 additions and 25 deletions.
2 changes: 1 addition & 1 deletion legend-engine-config/legend-engine-repl/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ help # to see the list of commands
# first load a CSV file into local DuckDB
load data.csv local::DuckDuckConnection test1
# then show the data
#>{local::DuckDuckDatabase.test1}#->sort([])->from(local::DuckDuckRuntime)
#>{local::DuckDuckDatabase.test1}#->select()->from(local::DuckDuckRuntime)

# to show the result grid in DataCube
show
Expand Down
5 changes: 0 additions & 5 deletions ...-data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/DataCubeReplExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@ public String type()
return "relational";
}

private boolean canShowGrid()
{
return Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE);
}

public void initialize(Client client)
{
this.client = client;
Expand Down
2 changes: 1 addition & 1 deletion ...data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public boolean process(String line) throws Exception
statement.executeUpdate(DatabaseManager.fromString(databaseConnection.type.name()).relationalDatabaseSupport().load(tableName, tempFile.getTemporaryPathForFile()));
this.client.printInfo("Cached into table: '" + tableName + "'. Launching DataCube...");

String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->sort([])->from(" + DataCube.getLocalRuntimePath() + ")";
String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")";
String functionCode = "###Pure\n" +
"function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}";
PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode);
Expand Down
2 changes: 1 addition & 1 deletion ...ta-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeLoadCSV.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public boolean process(String line) throws Exception
this.client.printInfo("Loaded into table: '" + tableName + "'");
}

String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->sort([])->from(" + DataCube.getLocalRuntimePath() + ")";
String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")";
String functionCode = "###Pure\n" +
"function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}";
PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode);
Expand Down
2 changes: 1 addition & 1 deletion ...data-cube/src/main/java/org/finos/legend/engine/repl/dataCube/commands/DataCubeTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public boolean process(String line) throws Exception
}

String tableName = tokens[2];
String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->sort([])->from(" + DataCube.getLocalRuntimePath() + ")";
String functionBodyCode = "#>{" + DataCube.getLocalDatabasePath() + "." + tableName + "}#->select()->from(" + DataCube.getLocalRuntimePath() + ")";
String functionCode = "###Pure\n" +
"function " + REPL_RUN_FUNCTION_SIGNATURE + "\n{\n" + functionBodyCode + ";\n}";
PureModelContextData pureModelContextData = client.getModelState().parseWithTransient(functionCode);
Expand Down
14 changes: 11 additions & 3 deletions ...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1000,9 +1000,14 @@ public Handlers(PureModel pureModel)
register("meta::pure::functions::collection::getAllVersionsInRange_Class_1__Date_1__Date_1__T_MANY_", true, ps -> res(ps.get(0)._genericType()._typeArguments().getFirst(), "zeroMany"));
register("meta::pure::functions::collection::getAllForEachDate_Class_1__Date_MANY__T_MANY_", false, ps -> res(ps.get(0)._genericType()._typeArguments().getFirst(), "zeroMany"));

register(h("meta::pure::tds::distinct_TabularDataSet_1__TabularDataSet_1_", false, ps -> res("meta::pure::tds::TabularDataSet", "one"), ps -> "TabularDataSet".equals(ps.get(0)._genericType()._rawType()._name())),
h("meta::pure::functions::collection::distinct_T_MANY__T_MANY_", false, ps -> res(ps.get(0)._genericType(), "zeroMany"), ps -> true));

register(m(
m(h("meta::pure::functions::relation::distinct_Relation_1__ColSpecArray_1__Relation_1_", true, ps -> res(ps.get(0)._genericType(), "one"), ps -> true)),
m(
h("meta::pure::tds::distinct_TabularDataSet_1__TabularDataSet_1_", false, ps -> res("meta::pure::tds::TabularDataSet", "one"), ps -> "TabularDataSet".equals(ps.get(0)._genericType()._rawType()._name())),
h("meta::pure::functions::collection::distinct_T_MANY__T_MANY_", false, ps -> res(ps.get(0)._genericType(), "zeroMany"), ps -> true)
)
)
);

register(m(
m(h("meta::pure::functions::collection::isDistinct_T_MANY__Boolean_1_", false, ps -> res("Boolean", "one"), ps -> ps.size() == 1)),
Expand Down Expand Up @@ -1233,6 +1238,7 @@ private void registerTDS()
h("meta::pure::functions::relation::select_Relation_1__ColSpecArray_1__Relation_1_", true, ps -> getTypeAndMultiplicity(Lists.mutable.with((RelationType<?>) ps.get(1)._genericType()._typeArguments().getLast()._rawType()), pureModel), ps -> true)
)
);
register(h("meta::pure::functions::relation::select_Relation_1__Relation_1_", true, ps -> res(ps.get(0)._genericType(), "one"), ps -> true));

register(h("meta::pure::tds::renameColumns_TabularDataSet_1__Pair_MANY__TabularDataSet_1_", false, ps -> res("meta::pure::tds::TabularDataSet", "one"), ps -> true));

Expand Down Expand Up @@ -2764,13 +2770,15 @@ private Map<String, Dispatch> buildDispatch()
map.put("meta::pure::functions::relation::over_SortInfo_MANY___Window_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "SortInfo".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::over_String_MANY__SortInfo_MANY__Frame_$0_1$___Window_1_", (List<ValueSpecification> ps) -> ps.size() == 3 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "SortInfo".equals(ps.get(1)._genericType()._rawType()._name())) && matchZeroOne(ps.get(2)._multiplicity()) && Sets.immutable.with("Nil", "Frame", "_Range", "Rows").contains(ps.get(2)._genericType()._rawType()._name()));
map.put("meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "SortInfo".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::distinct_Relation_1__ColSpecArray_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpecArray".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::ascending_ColSpec_1__SortInfo_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "ColSpec".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::descending_ColSpec_1__SortInfo_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "ColSpec".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::join_Relation_1__Relation_1__JoinKind_1__Function_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 4 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(1)._genericType()._rawType()._name()) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "JoinKind".equals(ps.get(2)._genericType()._rawType()._name())) && isOne(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || check(funcType(ps.get(3)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 2 && isOne(nps.get(0)._multiplicity()) && isOne(nps.get(1)._multiplicity())))));
map.put("meta::pure::functions::relation::groupBy_Relation_1__ColSpecArray_1__AggColSpec_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpecArray".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggColSpec".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::groupBy_Relation_1__ColSpecArray_1__AggColSpecArray_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpecArray".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggColSpecArray".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::groupBy_Relation_1__ColSpec_1__AggColSpecArray_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpec".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggColSpecArray".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::groupBy_Relation_1__ColSpec_1__AggColSpec_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpec".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggColSpec".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::select_Relation_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()));
map.put("meta::pure::functions::relation::select_Relation_1__ColSpec_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpec".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::select_Relation_1__ColSpecArray_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "ColSpecArray".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::relation::project_C_MANY__FuncColSpecArray_1__Relation_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "FuncColSpecArray".equals(ps.get(1)._genericType()._rawType()._name())));
Expand Down
27 changes: 27 additions & 0 deletions .../src/main/resources/core_functions_relation/relation/functions/transformation/select.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ native function <<PCT.function>> meta::pure::functions::relation::select<T,Z>(r:

native function <<PCT.function>> meta::pure::functions::relation::select<T,Z>(r:Relation<T>[1], cols:ColSpec<Z⊆T>[1]):Relation<Z>[1];

native function <<PCT.function>> meta::pure::functions::relation::select<T>(r:Relation<T>[1]):Relation<T>[1];

function <<PCT.test>> meta::pure::functions::relation::tests::select::testMultiColsSelectShared<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
{
let expr = {
Expand Down Expand Up @@ -93,3 +95,28 @@ function <<PCT.test>> meta::pure::functions::relation::tests::select::testSingle
' e\n'+
'#', $res->toString());
}

function <<PCT.test>> meta::pure::functions::relation::tests::select::testSelectAll<T|m>(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1]
{
let expr = {
| #TDS
val, str, other
1, a, a
3, ewe, b
4, qw, c
5, wwe, d
6, weq, e
#->select();
};

let res = $f->eval($expr);

assertEquals( '#TDS\n'+
' val,str,other\n'+
' 1,a,a\n'+
' 3,ewe,b\n'+
' 4,qw,c\n'+
' 5,wwe,d\n'+
' 6,weq,e\n'+
'#', $res->toString());
}
1 change: 1 addition & 0 deletions ...end/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public List<Native> getExtraNatives()
new Distinct(),
new Select(),
new SelectArray(),
new SelectAll(),
new Offset(),
new RowNumber(),
new Rank(),
Expand Down
6 changes: 6 additions & 0 deletions .../pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,12 @@ public static <T> Relation<? extends Object> rename(Relation<? extends T> r, Col
return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(r).rename(old._name(), aNew._name()), ps);
}

public static <T> Relation<? extends Object> select(Relation<? extends T> r, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
return new TDSContainer((TestTDSCompiled) RelationNativeImplementation.getTDS(r).select(Sets.mutable.withAll(RelationNativeImplementation.getTDS(r).getColumnNames())), ps);
}

public static <T> Relation<? extends Object> select(Relation<? extends T> r, ColSpec<?> col, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
Expand Down
28 changes: 28 additions & 0 deletions ...inos/legend/pure/runtime/java/extension/external/relation/compiled/natives/SelectAll.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2024 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives;

import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.ColSpec;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Relation;
import org.finos.legend.pure.m3.execution.ExecutionSupport;
import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNativeFunctionGeneric;

public class SelectAll extends AbstractNativeFunctionGeneric
{
public SelectAll()
{
super("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.select", new Class[]{Relation.class, ExecutionSupport.class}, false, true, false, "select_Relation_1__Relation_1_");
}
}
1 change: 1 addition & 0 deletions ...re/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public RelationExtensionInterpreted()
Tuples.pair("groupBy_Relation_1__ColSpecArray_1__AggColSpecArray_1__Relation_1_", GroupBy::new),
Tuples.pair("project_C_MANY__FuncColSpecArray_1__Relation_1_", Project::new),
Tuples.pair("columns_Relation_1__Column_MANY_", Columns::new),
Tuples.pair("select_Relation_1__Relation_1_", Select::new),
Tuples.pair("select_Relation_1__ColSpec_1__Relation_1_", Select::new),
Tuples.pair("select_Relation_1__ColSpecArray_1__Relation_1_", Select::new),
Tuples.pair("rowNumber_Relation_1__T_1__Integer_1_", RowNumber::new),
Expand Down
23 changes: 15 additions & 8 deletions ...inos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Select.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,26 @@ public CoreInstance execute(ListIterable<? extends CoreInstance> params, Stack<M

TestTDS tds = getTDS(params, 0, processorSupport);

Object cols = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport);
ListIterable<String> ids;
if (cols instanceof ColSpec)
if (params.size() == 1)
{
ids = Lists.mutable.with(((ColSpec<?>) cols)._name());
}
else if (cols instanceof ColSpecArray)
{
ids = ((ColSpecArray<?>) cols)._names().collect(c -> (String) c).toList();
ids = tds.getColumnNames();
}
else
{
throw new RuntimeException("Not Possible");
Object cols = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport);
if (cols instanceof ColSpec)
{
ids = Lists.mutable.with(((ColSpec<?>) cols)._name());
}
else if (cols instanceof ColSpecArray)
{
ids = ((ColSpecArray<?>) cols)._names().collect(c -> (String) c).toList();
}
else
{
throw new RuntimeException("Not Possible");
}
}

return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(tds.select(ids.toSet()), returnGenericType, repository, processorSupport), false, processorSupport);
Expand Down
Loading

0 comments on commit 89a39a6

Please sign in to comment.