Skip to content

Commit

Permalink
Merge branch 'hdt-2.1' into long-dict-id
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarioAriasGa
MarioAriasGa committed May 10, 2018
2 parents 9dc6b03 + 672fcce commit 6a76c03
Show file tree
Hide file tree
Showing 18 changed files with 169 additions and 134 deletions.
2 changes: 0 additions & 2 deletions hdt-fuseki/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
Expand All @@ -70,7 +69,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<!-- The configuration of the plugin -->
<configuration>
<!-- Specifies the configuration file of the assembly plugin -->
Expand Down
81 changes: 43 additions & 38 deletions hdt-fuseki/src/main/java/org/rdfhdt/hdt/fuseki/FusekiHDTCmd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.jena.atlas.lib.StrUtils;
import org.apache.jena.atlas.logging.LogCtl ;
import org.apache.jena.fuseki.Fuseki;
import org.apache.jena.fuseki.FusekiException;
import org.apache.jena.fuseki.mgt.ManagementServer;
import org.apache.jena.fuseki.server.FusekiConfig;
import org.apache.jena.fuseki.server.SPARQLServer;
Expand All @@ -45,6 +46,7 @@
import org.apache.jena.tdb.TDB;
import org.apache.jena.tdb.TDBFactory;
import org.apache.jena.tdb.sys.Names;
import org.apache.jena.tdb.transaction.TransactionManager;
import org.eclipse.jetty.server.Server;
import org.rdfhdt.hdt.hdt.HDT;
import org.rdfhdt.hdt.hdt.HDTManager;
Expand Down Expand Up @@ -141,29 +143,29 @@ static void setLogging() {
// pages/control/
// pages/query/ or /pages/sparql/

private static final ArgDecl argMgtPort = new ArgDecl(ArgDecl.HasValue, "mgtPort", "mgtport") ;
private static final ArgDecl argMem = new ArgDecl(ArgDecl.NoValue, "mem") ;
private static final ArgDecl argAllowUpdate = new ArgDecl(ArgDecl.NoValue, "update", "allowUpdate") ;
private static final ArgDecl argFile = new ArgDecl(ArgDecl.HasValue, "file") ;
private static final ArgDecl argMemTDB = new ArgDecl(ArgDecl.NoValue, "memtdb", "memTDB") ;
private static final ArgDecl argTDB = new ArgDecl(ArgDecl.HasValue, "loc", "location") ;
private static final ArgDecl argHDT = new ArgDecl(ArgDecl.HasValue, "hdt", "HDT") ;
private static final ArgDecl argPort = new ArgDecl(ArgDecl.HasValue, "port") ;
private static final ArgDecl argLocalhost = new ArgDecl(ArgDecl.NoValue, "localhost", "local") ;
private static final ArgDecl argTimeout = new ArgDecl(ArgDecl.HasValue, "timeout") ;
private static final ArgDecl argFusekiConfig = new ArgDecl(ArgDecl.HasValue, "config", "conf") ;
private static final ArgDecl argJettyConfig = new ArgDecl(ArgDecl.HasValue, "jetty-config") ;
private static final ArgDecl argGZip = new ArgDecl(ArgDecl.HasValue, "gzip") ;
private static final ArgDecl argUber = new ArgDecl(ArgDecl.NoValue, "uber", "über") ; // Use the überservlet (experimental)
private static final ArgDecl argBasicAuth = new ArgDecl(ArgDecl.HasValue, "basic-auth") ;
private static ArgDecl argMgtPort = new ArgDecl(ArgDecl.HasValue, "mgtPort", "mgtport") ;
private static ArgDecl argMem = new ArgDecl(ArgDecl.NoValue, "mem") ;
private static ArgDecl argAllowUpdate = new ArgDecl(ArgDecl.NoValue, "update", "allowUpdate") ;
private static ArgDecl argFile = new ArgDecl(ArgDecl.HasValue, "file") ;
private static ArgDecl argMemTDB = new ArgDecl(ArgDecl.NoValue, "memtdb", "memTDB") ;
private static ArgDecl argTDB = new ArgDecl(ArgDecl.HasValue, "loc", "location") ;
private static ArgDecl argHDT = new ArgDecl(ArgDecl.HasValue, "hdt", "HDT") ;
private static ArgDecl argPort = new ArgDecl(ArgDecl.HasValue, "port") ;
private static ArgDecl argLocalhost = new ArgDecl(ArgDecl.NoValue, "localhost", "local") ;
private static ArgDecl argTimeout = new ArgDecl(ArgDecl.HasValue, "timeout") ;
private static ArgDecl argFusekiConfig = new ArgDecl(ArgDecl.HasValue, "config", "conf") ;
private static ArgDecl argJettyConfig = new ArgDecl(ArgDecl.HasValue, "jetty-config") ;
private static ArgDecl argGZip = new ArgDecl(ArgDecl.HasValue, "gzip") ;
private static ArgDecl argUber = new ArgDecl(ArgDecl.NoValue, "uber", "über") ; // Use the überservlet (experimental)
private static ArgDecl argBasicAuth = new ArgDecl(ArgDecl.HasValue, "basic-auth") ;

private static final ArgDecl argGSP = new ArgDecl(ArgDecl.NoValue, "gsp") ; // GSP compliance mode
private static ArgDecl argGSP = new ArgDecl(ArgDecl.NoValue, "gsp") ; // GSP compliance mode

private static final ArgDecl argHome = new ArgDecl(ArgDecl.HasValue, "home") ;
private static final ArgDecl argPages = new ArgDecl(ArgDecl.HasValue, "pages") ;
private static ArgDecl argHome = new ArgDecl(ArgDecl.HasValue, "home") ;
private static ArgDecl argPages = new ArgDecl(ArgDecl.HasValue, "pages") ;

//private static ModLocation modLocation = new ModLocation() ;
private static final ModDatasetAssembler modDataset = new ModDatasetAssembler() ;
private static ModDatasetAssembler modDataset = new ModDatasetAssembler() ;

// fuseki [--mem|--desc assembler.ttl] [--port PORT] **** /datasetURI

Expand Down Expand Up @@ -195,9 +197,9 @@ public FusekiHDTCmd(String...argv)
{
super(argv) ;

// if ( false )
// // Consider ...
// TransactionManager.QueueBatchSize = TransactionManager.QueueBatchSize / 2 ;
if ( false )
// Consider ...
TransactionManager.QueueBatchSize = TransactionManager.QueueBatchSize / 2 ;

getUsage().startCategory("Fuseki") ;
addModule(modDataset) ;
Expand Down Expand Up @@ -226,7 +228,7 @@ public FusekiHDTCmd(String...argv)
super.modVersion.addClass(Fuseki.class) ;
}

static final String argUsage = "[--config=FILE] [--mem|--desc=AssemblerFile|--file=FILE] [--port PORT] /DatasetPathName" ;
static String argUsage = "[--config=FILE] [--mem|--desc=AssemblerFile|--file=FILE] [--port PORT] /DatasetPathName" ;

@Override
protected String getSummary()
Expand Down Expand Up @@ -270,14 +272,14 @@ protected void processModulesAndArgs()
if ( contains(argMem) )
{
log.info("Dataset: in-memory") ;
dsg = DatasetGraphFactory.createGeneral() ;
dsg = DatasetGraphFactory.create() ;
}
if ( contains(argFile) )
{
dsg = DatasetGraphFactory.createGeneral() ;
dsg = DatasetGraphFactory.create() ;
// replace by RiotLoader after ARQ refresh.
String filename = getValue(argFile) ;
log.info("Dataset: in-memory: load file: {}", filename);
log.info("Dataset: in-memory: load file: "+filename) ;
if ( ! FileOps.exists(filename) )
throw new CmdException("File not found: "+filename) ;

Expand Down Expand Up @@ -327,7 +329,7 @@ protected void processModulesAndArgs()
// Single Graph:
HDT hdt = HDTManager.mapIndexedHDT(hdtFile, null);
Graph graph = new HDTGraph(hdt);
dsg = DatasetGraphFactory.createOneGraph(graph);
dsg = DatasetGraphFactory.wrap(graph);

// Multiple Graphs:
// DatasetGraphMap datasetMap = new DatasetGraphMap(defaultGraph);
Expand Down Expand Up @@ -466,29 +468,29 @@ protected void exec()
}

homeDir = sort_out_dir(homeDir) ;
Fuseki.configLog.info("Home Directory: {}", FileOps.fullDirectoryPath(homeDir));
Fuseki.configLog.info("Home Directory: " + FileOps.fullDirectoryPath(homeDir));
if ( ! FileOps.exists(homeDir) )
Fuseki.configLog.warn("No such directory for Fuseki home: {}", homeDir);
Fuseki.configLog.warn("No such directory for Fuseki home: "+homeDir) ;

String staticContentDir = pagesDir ;
if ( staticContentDir == null )
staticContentDir = homeDir+Fuseki.PagesStatic ;

Fuseki.configLog.debug("Static Content Directory: {}", FileOps.fullDirectoryPath(staticContentDir));
Fuseki.configLog.debug("Static Content Directory: "+ FileOps.fullDirectoryPath(staticContentDir)) ;

if ( ! FileOps.exists(staticContentDir) ) {
Fuseki.configLog.warn("No such directory for static content: {}", FileOps.fullDirectoryPath(staticContentDir));
Fuseki.configLog.warn("No such directory for static content: " + FileOps.fullDirectoryPath(staticContentDir)) ;
Fuseki.configLog.warn("You may need to set the --pages or --home option to configure static content correctly");
}

if ( jettyConfigFile != null )
Fuseki.configLog.info("Jetty configuration: {}", jettyConfigFile);
Fuseki.configLog.info("Jetty configuration: "+jettyConfigFile) ;

ServerConfig serverConfig ;

if ( fusekiConfigFile != null )
{
Fuseki.configLog.info("Configuration file: {}", fusekiConfigFile);
Fuseki.configLog.info("Configuration file: "+fusekiConfigFile) ;
serverConfig = FusekiConfig.configure(fusekiConfigFile) ;
}
else
Expand Down Expand Up @@ -519,18 +521,21 @@ protected void exec()

if ( mgtPort > 0 )
{
Fuseki.configLog.info("Management services on port {}", mgtPort);
Fuseki.configLog.info("Management services on port "+mgtPort) ;
mgtServer = ManagementServer.createManagementServer(mgtPort) ;
try { mgtServer.start() ; }
catch (java.net.BindException ex)
{
serverLog.error("SPARQLServer: Failed to start management server: {}", ex.getMessage()); System.exit(1) ; }
{ serverLog.error("SPARQLServer: Failed to start management server: " + ex.getMessage()) ; System.exit(1) ; }
catch (Exception ex)
{
serverLog.error("SPARQLServer: Failed to start management server: {}", ex.getMessage(), ex); System.exit(1) ; }
{ serverLog.error("SPARQLServer: Failed to start management server: " + ex.getMessage(), ex) ; System.exit(1) ; }
}

try {
server.start() ;
} catch (FusekiException ex) {
serverLog.warn("Failed to start the server.", ex) ;
}

try { server.getServer().join() ; } catch (Exception ex) {}

if ( mgtServer != null )
Expand Down
9 changes: 5 additions & 4 deletions hdt-java-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,7 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>jena-arq</artifactId>
</dependency>

<dependency>
<groupId>org.rdfhdt</groupId>
<artifactId>hdt-api</artifactId>
Expand All @@ -47,6 +44,10 @@
<groupId>org.apache.jena</groupId>
<artifactId>jena-base</artifactId>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>jena-arq</artifactId>
</dependency>
<dependency>
<groupId>pl.edu.icm</groupId>
<artifactId>JLargeArrays</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion hdt-java-core/src/main/java/org/rdfhdt/hdt/util/string/UnicodeEscape.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public static void escapeString(String label, Appendable appendable)

char prev = label.charAt(i-1);
if(curr=='@' && prev=='"') {
last = i-2;
last = i-1;
break;
}
if(curr=='^' && prev=='^') {
Expand Down
17 changes: 14 additions & 3 deletions hdt-java-core/src/test/java/org/rdfhdt/hdt/hdt/writer/TripleWritterHDTTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package org.rdfhdt.hdt.hdt.writer;

import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;

import java.io.File;
import java.io.IOException;

import org.junit.Before;
import org.junit.Test;
import org.rdfhdt.hdt.enums.RDFNotation;
import org.rdfhdt.hdt.exceptions.ParserException;
import org.rdfhdt.hdt.hdt.HDT;
import org.rdfhdt.hdt.hdt.HDTManager;
import org.rdfhdt.hdt.options.HDTSpecification;
import org.rdfhdt.hdt.rdf.RDFParserCallback;
import org.rdfhdt.hdt.rdf.RDFParserFactory;
Expand All @@ -22,7 +25,10 @@ public void setUp() throws Exception {
@Test
public void test() {
try {
final TripleWriterHDT wr = new TripleWriterHDT("http://example.org", new HDTSpecification(), "out.hdt", false);
File file = new File("out.hdt");
file.deleteOnExit();

final TripleWriterHDT wr = new TripleWriterHDT("http://example.org", new HDTSpecification(), file.toString(), false);

RDFParserCallback pars = RDFParserFactory.getParserCallback(RDFNotation.NTRIPLES);
pars.doParse("data/test.nt", "http://example.org", RDFNotation.NTRIPLES, new RDFParserCallback.RDFCallback() {
Expand All @@ -36,8 +42,13 @@ public void processTriple(TripleString triple, long pos) {
}
}
});

wr.close();

HDT hdt = HDTManager.loadHDT(file.toString());
assertEquals("Successfully loaded HDT with same number of triples", 10, hdt.getTriples().getNumberOfElements());

// Delete temp file
file.delete();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
Expand Down
4 changes: 0 additions & 4 deletions hdt-jena/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,6 @@
<version>2.1-SNAPSHOT</version>
</parent>

<properties>
<jdk.version>${jena-jdk.version}</jdk.version>
</properties>

<dependencies>
<dependency>
<groupId>org.rdfhdt</groupId>
Expand Down
75 changes: 40 additions & 35 deletions hdt-jena/src/test/java/JenaHDTTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,56 +66,60 @@ public static void iterateObjects(Model model, String label) throws IOException
}

public static void findObjects(Model model, String label) throws IOException {
BufferedReader reader = new BufferedReader(new FileReader(new File("objects2.txt")));
PrintWriter out = new PrintWriter(DummyOutputStream.getInstance());

StopWatch st = new StopWatch();

String line;
while((line = reader.readLine())!=null) {
StmtIterator iter = model.listStatements(null, null, line);
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
// Resource subject = stmt.getSubject(); // get the subject
// Property predicate = stmt.getPredicate(); // get the predicate
// RDFNode object = stmt.getObject(); // get the object

out.println(stmt);
}
try(BufferedReader reader = new BufferedReader(new FileReader(new File("objects2.txt")))) {
PrintWriter out = new PrintWriter(DummyOutputStream.getInstance());

StopWatch st = new StopWatch();

String line;
while((line = reader.readLine())!=null) {
StmtIterator iter = model.listStatements(null, null, line);
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
// Resource subject = stmt.getSubject(); // get the subject
// Property predicate = stmt.getPredicate(); // get the predicate
// RDFNode object = stmt.getObject(); // get the object

out.println(stmt);
}
}
System.out.println(label+st.stopAndShow());
}
System.out.println(label+st.stopAndShow());
}

public static void findSubjects(Model model, String label) throws IOException {
BufferedReader reader = new BufferedReader(new FileReader(new File("subjects2.txt")));
PrintWriter out = new PrintWriter(DummyOutputStream.getInstance());

StopWatch st = new StopWatch();

String line;
while((line = reader.readLine())!=null) {
StmtIterator iter = model.listStatements(model.getResource(line), null, (RDFNode)null);
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
// Resource subject = stmt.getSubject(); // get the subject
// Property predicate = stmt.getPredicate(); // get the predicate
// RDFNode object = stmt.getObject(); // get the object

out.println(stmt);
}
try(BufferedReader reader = new BufferedReader(new FileReader(new File("subjects2.txt")))){
PrintWriter out = new PrintWriter(DummyOutputStream.getInstance());

StopWatch st = new StopWatch();

String line;
while((line = reader.readLine())!=null) {
StmtIterator iter = model.listStatements(model.getResource(line), null, (RDFNode)null);
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
// Resource subject = stmt.getSubject(); // get the subject
// Property predicate = stmt.getPredicate(); // get the predicate
// RDFNode object = stmt.getObject(); // get the object

out.println(stmt);
}
}
System.out.println(label+st.stopAndShow());
}
System.out.println(label+st.stopAndShow());
}

public static void iterateTriples(Model model, String label) {
@SuppressWarnings("unused")
PrintWriter out = new PrintWriter(DummyOutputStream.getInstance());

// list the statements in the Model
StopWatch st = new StopWatch();

StmtIterator iter = model.listStatements();//(Resource)null, model.getProperty("http://www.geonames.org/ontology#alternateName"), (RDFNode)null);
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
@SuppressWarnings("unused")
Statement stmt = iter.nextStatement(); // get next statement
// Resource subject = stmt.getSubject(); // get the subject
// Property predicate = stmt.getPredicate(); // get the predicate
// RDFNode object = stmt.getObject(); // get the object
Expand Down Expand Up @@ -177,6 +181,7 @@ public static void measureTriples(Model model) {
System.out.println("Total time: "+st.stopAndGet());
}

@SuppressWarnings("unused")
public static void main(String[] args) throws Exception {

StopWatch st = new StopWatch();
Expand Down
2 changes: 1 addition & 1 deletion hdt-jena/src/test/java/org/rdfhdt/hdtjena/solver/OptimizedCountTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class OptimizedCountTest {
public void setUp() throws Exception {
HDT hdt = HDTManager.generateHDT(new DummyIteratorTripleString(), "http://www.rdfhdt.org", new HDTSpecification(), null);
graph =new HDTGraph(hdt, true);
dataset = DatasetGraphFactory.createOneGraph(graph);
dataset = DatasetGraphFactory.wrap(graph);
}

private Plan getPlan(String sparql) {
Expand Down
4 changes: 2 additions & 2 deletions hdt-jena/testing/ARQ/Construct/results-construct-quad-syntax-3.ttl
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@prefix : <http://example.org/ns#> .
:s :p :o
@prefix : <http://example.org/ns#> .
:s :p :o .
Loading

0 comments on commit 6a76c03

Please sign in to comment.