Skip to content

Commit

Permalink
Storing full query results
Browse files Browse the repository at this point in the history
  • Loading branch information
@hvarg
hvarg committed Feb 28, 2024
1 parent 66c7037 commit dd4984d
Show file tree
Hide file tree
Showing 11 changed files with 203 additions and 154 deletions.
104 changes: 58 additions & 46 deletions server/src/main/java/org/diskproject/server/db/DiskDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

Expand All @@ -20,6 +21,7 @@
import org.diskproject.shared.classes.hypothesis.GoalResult;
import org.diskproject.shared.classes.loi.DataQueryResult;
import org.diskproject.shared.classes.loi.DataQueryTemplate;
import org.diskproject.shared.classes.loi.LOICommon;
import org.diskproject.shared.classes.loi.LineOfInquiry;
import org.diskproject.shared.classes.loi.TriggeredLOI;
import org.diskproject.shared.classes.question.Question;
Expand Down Expand Up @@ -507,8 +509,10 @@ private KBObject _writeDataQueryTemplate (DataQueryTemplate dataQuery, KBObject
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_DATA_SOURCE), domainKB.getIndividual(dataQuery.getEndpoint().getId()));
if (dataQuery.getTemplate() != null)
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_QUERY_TEMPLATE), domainKB.createLiteral(dataQuery.getTemplate()));
if (dataQuery.getVariablesToShow() != null)
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES), domainKB.createLiteral(dataQuery.getVariablesToShow()));
if (dataQuery.getVariablesToShow() != null) {
String arrString = "[" + String.join(",",dataQuery.getVariablesToShow()) + "]";
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES), domainKB.createLiteral(arrString));
}
if (dataQuery.getFootnote() != null)
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_DESCRIPTION), domainKB.createLiteral(dataQuery.getFootnote()));
return dq;
Expand All @@ -521,8 +525,14 @@ private DataQueryTemplate loadDataQueryTemplate (KBObject objTemplate) {
if (domainKB.getComment(objTemplate) != null)
dataQuery.setDescription(domainKB.getComment(objTemplate));
KBObject objVars = domainKB.getPropertyValue(objTemplate, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES));
if (objVars != null)
dataQuery.setVariablesToShow(objVars.getValueAsString());
if (objVars != null) {
String raw = objVars.getValueAsString();
if (raw.startsWith("[") && raw.endsWith("]")) {
dataQuery.setVariablesToShow(Arrays.asList(raw.substring(1, raw.length()-1).split(",")));
} else {
System.out.println("Could not read table variables: " + raw);
}
}
KBObject objFootnotes = domainKB.getPropertyValue(objTemplate, DISKOnt.getProperty(DISK.HAS_TABLE_DESCRIPTION));
if (objFootnotes != null)
dataQuery.setFootnote(objFootnotes.getValueAsString());
Expand Down Expand Up @@ -583,7 +593,24 @@ public boolean writeLOI(LineOfInquiry loi) {
this.rdf.startWrite();

KBObject loiItem = writeCommonResource(loi, loiId, DISKOnt.getClass(DISK.LINE_OF_INQUIRY));
writeLOIExtras(loi, loiItem);
writeLOICommon(loi, loiItem);
if (loi.getDataQueryTemplate() != null) {
KBObject dqt = writeDataQueryTemplate(loi.getDataQueryTemplate());
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY), dqt);
}
List<WorkflowSeed> wf = loi.getWorkflowSeeds(), mwf = loi.getMetaWorkflowSeeds();
if (wf != null && wf.size() > 0) {
for (WorkflowSeed wfSeed: wf) {
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED),
writeWorkflowSeed(wfSeed, loi.getId()));
}
}
if (mwf != null && mwf.size() > 0) {
for (WorkflowSeed wfSeed: mwf) {
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED),
writeWorkflowSeed(wfSeed, loi.getId()));
}
}
this.rdf.save(domainKB);
this.rdf.end();

Expand All @@ -602,71 +629,56 @@ public LineOfInquiry loadLOI(String id) {
}

LineOfInquiry loi = new LineOfInquiry(loadCommonResource(loiItem));
loadLOIExtras(loi, loiItem);
loadLOICommon(loi, loiItem);
KBObject dataQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY));
if (dataQueryObj != null)
loi.setDataQueryTemplate(loadDataQueryTemplate(dataQueryObj));
List<KBObject> wfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED));
List<KBObject> mwfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED));
List<WorkflowSeed> wList = new ArrayList<WorkflowSeed>(), mList = new ArrayList<WorkflowSeed>();

if (wfSeeds != null && wfSeeds.size() > 0) {
for (KBObject t: wfSeeds) {
wList.add(loadWorkflowSeed(t));
}
}
if (mwfSeeds != null && mwfSeeds.size() > 0) {
for (KBObject t: mwfSeeds) {
mList.add(loadWorkflowSeed(t));
}
}
loi.setWorkflowSeeds(wList);
loi.setMetaWorkflowSeeds(mList);

this.rdf.end();
return loi;
}

private void writeLOIExtras (LineOfInquiry loi, KBObject loiItem) {
private void writeLOICommon (LOICommon loi, KBObject loiItem) {
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_UPDATE_CONDITION),
domainKB.createLiteral(loi.getUpdateCondition()));
if (loi.getGoalQuery() != null) {
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_GOAL_QUERY),
domainKB.createLiteral(loi.getGoalQuery()));
}
if (loi.getDataQueryTemplate() != null) {
KBObject dqt = writeDataQueryTemplate(loi.getDataQueryTemplate());
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY), dqt);
}
String questionId = loi.getQuestion().getId();
if (questionId != null)
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_QUESTION),
domainKB.createLiteral(questionId));
List<WorkflowSeed> wf = loi.getWorkflowSeeds(), mwf = loi.getMetaWorkflowSeeds();
if (wf != null && wf.size() > 0) {
for (WorkflowSeed wfSeed: wf) {
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED),
writeWorkflowSeed(wfSeed, loi.getId()));
}
}
if (mwf != null && mwf.size() > 0) {
for (WorkflowSeed wfSeed: mwf) {
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED),
writeWorkflowSeed(wfSeed, loi.getId()));
}
}

}

private void loadLOIExtras (LineOfInquiry loi, KBObject loiItem) {
private void loadLOICommon (LOICommon loi, KBObject loiItem) {
KBObject goalQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_GOAL_QUERY));
if (goalQueryObj != null)
loi.setGoalQuery(goalQueryObj.getValueAsString());
KBObject dataQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY));
if (dataQueryObj != null)
loi.setDataQueryTemplate(loadDataQueryTemplate(dataQueryObj));
KBObject questionobj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_QUESTION));
if (questionobj != null)
loi.setQuestion(new Question(questionobj.getValueAsString()));
KBObject updateCondObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_UPDATE_CONDITION));
if (updateCondObj != null)
loi.setUpdateCondition( Integer.parseInt(updateCondObj.getValueAsString()) );

List<KBObject> wfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED));
List<KBObject> mwfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED));
List<WorkflowSeed> wList = new ArrayList<WorkflowSeed>(), mList = new ArrayList<WorkflowSeed>();

if (wfSeeds != null && wfSeeds.size() > 0) {
for (KBObject t: wfSeeds) {
wList.add(loadWorkflowSeed(t));
}
}
if (mwfSeeds != null && mwfSeeds.size() > 0) {
for (KBObject t: mwfSeeds) {
mList.add(loadWorkflowSeed(t));
}
}
loi.setWorkflowSeeds(wList);
loi.setMetaWorkflowSeeds(mList);
}

public boolean deleteLOI(String id) {
Expand Down Expand Up @@ -1058,7 +1070,7 @@ public boolean writeTLOI(TriggeredLOI tloi) {

this.rdf.startWrite();
KBObject tloiItem = writeCommonResource(tloi, tloiId, DISKOnt.getClass(DISK.TRIGGERED_LINE_OF_INQUIRY));
writeLOIExtras(tloi, tloiItem);
writeLOICommon(tloi, tloiItem);
if (tloi.getParentLoi() != null) {
KBObject loiObj = domainKB.getResource(tloi.getParentLoi().getId());
domainKB.setPropertyValue(tloiItem, DISKOnt.getProperty(DISK.HAS_LINE_OF_INQUIRY), loiObj);
Expand Down Expand Up @@ -1103,7 +1115,7 @@ public TriggeredLOI loadTLOI(String id) {
KBObject obj = domainKB.getIndividual(tloiId);
if (obj != null && obj.getName() != null) {
TriggeredLOI tloi = new TriggeredLOI(loadCommonResource(obj));
loadLOIExtras(tloi, obj);
loadLOICommon(tloi, obj);

KBObject parentLOI = domainKB.getPropertyValue(obj, DISKOnt.getProperty(DISK.HAS_LINE_OF_INQUIRY));
if (parentLOI != null) // We do not load the LOI
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/diskproject/server/managers/MethodAdapterManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public String toString () {

/**
* Create records for each adapter into the RDF database.
* @param db Diskdb where to register the adapters.
* @param db DiskDB where to register the adapters.
*/
public void registerAdapters (DiskDB db) {
this.byId = new HashMap<String, MethodAdapter>();
Expand Down
57 changes: 42 additions & 15 deletions server/src/main/java/org/diskproject/server/quering/Match.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ public class Match {
public final Question question;
public final DataAdapter dataSource;
public boolean fullCSV, valid, ready;
public Set<String> seedVariables, arrayVariables; // These are the variables required from the seeds.
public Set<String> selectVariables, seedVariables, arrayVariables; // These are the variables required from the seeds.
public String csvURL, querySend;
public List<DataResult> queryResult;
public Set<String> queryVariables; //this ones dont have the ?
public Set<String> queryVariables; //this ones do not have the `?`

public Match (Goal goal, LineOfInquiry loi, Question question, DataAdapter dataSource) {
this.goal = goal;
Expand All @@ -58,7 +58,7 @@ public Match (Goal goal, LineOfInquiry loi, Question question, DataAdapter dataS
private boolean analyseWorkflows () {
boolean isCSV = false;
List<WorkflowSeed> allSeeds = Stream.concat(loi.getWorkflowSeeds().stream(), loi.getMetaWorkflowSeeds().stream()).collect(Collectors.toList());
Set<String> reqVariables = new HashSet<String>(), arrayVars = new HashSet<String>(), nonArrayVars = new HashSet<String>();
Set<String> reqVariables = new HashSet<String>(), arrayVars = new HashSet<String>(), selectVars = new HashSet<String>();
for (WorkflowSeed seed: allSeeds) {
Endpoint source = seed.getSource();
if (source == null || source.getId() == null || source.getId().equals("")) {
Expand All @@ -76,14 +76,14 @@ private boolean analyseWorkflows () {
} else {
//This should not happen.
System.out.println("Workflow seed configuration sets two variables for a single parameter/input");
nonArrayVars.add(raw);
return false;
}
} else {
raw = vb.getSingleBinding();
}
if (raw.startsWith("?")) {
reqVariables.add(raw);
selectVars.add(raw);
} else if (raw.equals(SPECIAL.CSV)) {
isCSV = true;
}
Expand All @@ -94,9 +94,16 @@ private boolean analyseWorkflows () {
System.out.println("Workflow seeds must require at least one variable from the data query ");
return false;
}

if (loi.getDataQueryTemplate() != null) {
for (String v: loi.getDataQueryTemplate().getVariablesToShow()) {
selectVars.add(v);
}
}
this.fullCSV = isCSV;
this.seedVariables = reqVariables;
this.arrayVariables = arrayVars;
this.selectVariables = selectVars;
return true;
}

Expand Down Expand Up @@ -179,7 +186,15 @@ public boolean createWorkflowInstances () {
}

public String getResultsAsCSV () {
return "";
String csv = String.join(",", queryVariables);
for (DataResult line: queryResult) {
List<String> arrLine = new ArrayList<String>();
for (String varName: queryVariables) {
arrLine.add(line.getValue(varName));
}
csv += "\n" + String.join(",", arrLine);
}
return csv;
}

public TriggeredLOI createTLOI () {
Expand All @@ -203,14 +218,15 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
if (queryResult == null || queryVariables == null || !ready || queryResult.size() == 0 || queryVariables.size() == 0) {
return null;
}
List<DataResult> filteredResults = filterQueryResults(selectVariables);
// One seed can create multiple instances. As the results are a table, we need to aggregate the results.
int runs = 0;
Map<String,Integer> ticks = new HashMap<String,Integer>();
for (String name: this.queryVariables) {
ticks.put(name, 0);
String lastValue = null;
boolean isArray = arrayVariables.contains("?" + name);
for (DataResult cell: this.queryResult) {
for (DataResult cell: filteredResults) {
String currentValue = cell.getValue(name);
if (currentValue != lastValue) {
int newMax = ticks.get(name) + 1;
Expand All @@ -223,15 +239,11 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
}
}

//for (String name: queryVariables) {
// System.out.println(name + (arrayVariables.contains("?" + name) ? " [array]" : " [single]") + " = " + ticks.get(name));
//}

//Separate the table depending of the runs.
List<List<DataResult>> independentResults = new ArrayList<List<DataResult>>();
List<DataResult> lastList = new ArrayList<DataResult>();
int count = 0, splitSize = queryResult.size()/runs;
for (DataResult cell: queryResult) {
int count = 0, splitSize = filteredResults.size()/runs;
for (DataResult cell: filteredResults) {
count += 1;
if (count >= splitSize) {
count = 0;
Expand All @@ -240,7 +252,6 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
}
lastList.add(cell);
}
//System.out.println(independentResults.size());

List<WorkflowInstantiation> inst = new ArrayList<WorkflowInstantiation>();
for (List<DataResult> resultsToBind: independentResults) {
Expand All @@ -267,7 +278,7 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
}
} else {
// Required variable is not on the query results.
// Should break the outer loop, FIXME;
// Should break the outer loop?
continue;
}
}
Expand All @@ -277,10 +288,26 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
}
}
current.setDataBindings(dataBindings);
//System.out.println(current);
inst.add(current);
}
return inst;
}

private List<DataResult> filterQueryResults (Set<String> allowedVariables) {
// If the query results includes variables not used on the workflow, these are removed if the contents are the same.
List<DataResult> list = new ArrayList<DataResult>();
String lastLine = "";
for (DataResult cell: queryResult) {
String currentLine = "";
for (String v: allowedVariables) {
String varName = v.startsWith("?") ? v.substring(1) : v;
currentLine += cell.getValue(varName) + ",";
}
if (!currentLine.equals(lastLine)) {
lastLine = currentLine;
list.add(cell);
}
}
return list;
}
}
2 changes: 1 addition & 1 deletion server/src/main/java/org/diskproject/server/repository/DiskRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,7 @@ public List<TriggeredLOI> queryGoal (String id) throws Exception, QueryParseExce

private TriggeredLOI uploadData (TriggeredLOI tloi) {
//Check and upload files.
DataAdapter dataAdapter = dataAdapters.getMethodAdapterByEndpoint(tloi.getDataQueryTemplate().getEndpoint());
DataAdapter dataAdapter = dataAdapters.getMethodAdapterByEndpoint(tloi.getQueryResults().getEndpoint());
List<WorkflowInstantiation> wf = new ArrayList<WorkflowInstantiation>(),
mwf = new ArrayList<WorkflowInstantiation>();
for (WorkflowInstantiation i: tloi.getWorkflows()) {
Expand Down
4 changes: 0 additions & 4 deletions server/src/main/java/org/diskproject/server/threads/MonitorThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,5 @@ private void updateWorkflowStatus() {
}
}
}
System.out.println("Update wf status:");
for (WorkflowInstantiation inst: list) {
System.out.println(inst.getStatus());
}
}
}
Loading

0 comments on commit dd4984d

Please sign in to comment.