diff --git a/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/CassandraStorageLayerFactory.java b/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/CassandraStorageLayerFactory.java index 4ea56fa..fe0e520 100644 --- a/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/CassandraStorageLayerFactory.java +++ b/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/CassandraStorageLayerFactory.java @@ -20,7 +20,7 @@ import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; import org.gazzax.labs.jena.nosql.fwk.dictionary.node.TransientNodeDictionary; import org.gazzax.labs.jena.nosql.fwk.ds.MapDAO; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.factory.ClientShutdownHook; import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; @@ -38,6 +38,7 @@ import com.datastax.driver.core.policies.Policies; import com.datastax.driver.core.policies.ReconnectionPolicy; import com.datastax.driver.core.policies.RetryPolicy; +import com.hp.hpl.jena.graph.Node; /** * Concrete factory for creating Cassandra-backed domain and data access objects. @@ -75,12 +76,19 @@ public MapDAO getMapDAO( } @Override - public TripleIndexDAO getTripleIndexDAO() { + public GraphDAO getGraphDAO(final Node name) { + return new CassandraTripleIndexDAO(session, dictionary); + } + + @Override + public GraphDAO getGraphDAO() { return new CassandraTripleIndexDAO(session, dictionary); } @Override public void accept(final Configuration> configuration) { + deletionBatchSize = configuration.getParameter("delete-batch-size", Integer.valueOf(1000)); + final String hosts = configuration.getParameter("cassandra-contact-points", "localhost"); final Cluster.Builder builder = Cluster.builder() diff --git a/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/dao/CassandraTripleIndexDAO.java b/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/dao/CassandraTripleIndexDAO.java index 46f47c5..7680385 100644 --- a/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/dao/CassandraTripleIndexDAO.java +++ b/jena-nosql-binding/jena-nosql-binding-cassandra/src/main/java/org/gazzax/labs/jena/nosql/cassandra/dao/CassandraTripleIndexDAO.java @@ -11,7 +11,7 @@ import org.gazzax.labs.jena.nosql.fwk.StorageLayerException; import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import com.datastax.driver.core.BatchStatement; import com.datastax.driver.core.BoundStatement; @@ -22,7 +22,7 @@ import com.google.common.collect.AbstractIterator; /** - * Cassandra 2x (CQL-based) implementation of {@link TripleIndexDAO}. + * Cassandra 2x (CQL-based) implementation of {@link GraphDAO}. * * This class has been derived from CumulusRDF code, with many thanks to CumulusRDF team for allowing this. * @@ -30,7 +30,7 @@ * @author Andrea Gazzarini * @since 1.0 */ -public class CassandraTripleIndexDAO implements TripleIndexDAO { +public class CassandraTripleIndexDAO implements GraphDAO { protected static final byte[] EMPTY_VAL = new byte[0]; protected static final String SELECT_SPOC_FROM = "SELECT s, p, o, c FROM "; diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/SolrStorageLayerFactory.java b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/SolrStorageLayerFactory.java index e0765a4..3f82606 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/SolrStorageLayerFactory.java +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/SolrStorageLayerFactory.java @@ -7,10 +7,10 @@ import org.gazzax.labs.jena.nosql.fwk.configuration.Configuration; import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; import org.gazzax.labs.jena.nosql.fwk.ds.MapDAO; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.factory.ClientShutdownHook; import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; -import org.gazzax.labs.jena.nosql.solr.dao.SolrTripleIndexDAO; +import org.gazzax.labs.jena.nosql.solr.dao.SolrGraphDAO; import org.gazzax.labs.jena.nosql.solr.graph.SolrGraph; import com.hp.hpl.jena.graph.Graph; @@ -30,11 +30,14 @@ public class SolrStorageLayerFactory extends StorageLayerFactory { @Override public void accept(final Configuration> configuration) { + deletionBatchSize = configuration.getParameter("delete-batch-size", Integer.valueOf(1000)); + final String address = configuration.getParameter("solr-address", "http://127.0.0.1:8080/solr/store"); try { solr = (SolrServer) Class.forName(configuration.getParameter("solr-server-class", HttpSolrServer.class.getName())) .getConstructor(String.class) .newInstance(address); + } catch (final Exception exception) { throw new IllegalArgumentException(exception); } @@ -51,19 +54,24 @@ public MapDAO getMapDAO( @Override public Graph getGraph() { - return new SolrGraph(this); + return new SolrGraph(this, deletionBatchSize); } @Override public Graph getGraph(Node graphNode) { - return new SolrGraph(graphNode, this); + return new SolrGraph(graphNode, this, deletionBatchSize); } @Override - public TripleIndexDAO getTripleIndexDAO() { - return new SolrTripleIndexDAO(solr); + public GraphDAO getGraphDAO(final Node name) { + return new SolrGraphDAO(solr, name); } - + + @Override + public GraphDAO getGraphDAO() { + return new SolrGraphDAO(solr); + } + @Override public TopLevelDictionary getDictionary() { return dictionary; diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrDeepPagingIterator.java b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrDeepPagingIterator.java index 85a7599..82db90c 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrDeepPagingIterator.java +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrDeepPagingIterator.java @@ -117,7 +117,7 @@ public Triple next() { * @param solr the SOLR facade. * @param query the query that will be submitted. */ - public SolrDeepPagingIterator(final SolrServer solr, final SolrQuery query) { + SolrDeepPagingIterator(final SolrServer solr, final SolrQuery query) { this.solr = solr; this.query = query; this.sentCursorMark = CursorMarkParams.CURSOR_MARK_START; diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrTripleIndexDAO.java b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrGraphDAO.java similarity index 71% rename from jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrTripleIndexDAO.java rename to jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrGraphDAO.java index 2fe33b6..06fb8fe 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrTripleIndexDAO.java +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/dao/SolrGraphDAO.java @@ -14,7 +14,7 @@ import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrInputDocument; import org.gazzax.labs.jena.nosql.fwk.StorageLayerException; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.log.Log; import org.gazzax.labs.jena.nosql.fwk.log.MessageCatalog; import org.gazzax.labs.jena.nosql.solr.Field; @@ -24,18 +24,36 @@ import com.hp.hpl.jena.graph.Triple; import com.hp.hpl.jena.graph.TripleMatch; -public class SolrTripleIndexDAO implements TripleIndexDAO { - protected final Log logger = new Log(LoggerFactory.getLogger(SolrTripleIndexDAO.class)); +/** + * {@link GraphDAO} implementation for Apache SOLR. + * + * @see http://lucene.apache.org/solr + * @author Andrea Gazzarini + * @since 1.0 + */ +public class SolrGraphDAO implements GraphDAO { + protected final Log logger = new Log(LoggerFactory.getLogger(SolrGraphDAO.class)); private final SolrServer solr; + private final Node name; /** - * Builds a new {@link TripleIndexDAO} with the given SOLR client. + * Builds a new {@link GraphDAO} with the given SOLR client. * * @param solr the SOLR client. */ - public SolrTripleIndexDAO(final SolrServer solr) { + public SolrGraphDAO(final SolrServer solr) { + this(solr, null); + } + + /** + * Builds a new {@link GraphDAO} with the given SOLR client. + * + * @param solr the SOLR client. + */ + public SolrGraphDAO(final SolrServer solr, final Node name) { this.solr = solr; + this.name = name; } @Override @@ -45,6 +63,9 @@ public void insertTriple(final Triple triple) throws StorageLayerException { document.setField(Field.P, asNtURI(triple.getPredicate())); document.setField(Field.O, asNt(triple.getObject())); + // TODO : with a state pattern I could avoid this conditional logic. + document.setField(Field.C, name != null ? asNtURI(name) : null); + try { solr.add(document); } catch (final Exception exception) { @@ -61,21 +82,6 @@ public void deleteTriple(final Triple triple) throws StorageLayerException { } } - /** - * Builds a delete query starting from a given triple. - * - * @param triple the triple. - * @return a delete query starting from a given triple. - */ - private String deleteQuery(final Triple triple) { - - return new StringBuilder() - .append(Field.S).append(":\"").append(ClientUtils.escapeQueryChars(asNt(triple.getSubject()))).append("\" AND ") - .append(Field.P).append(":\"").append(ClientUtils.escapeQueryChars(asNt(triple.getPredicate()))).append("\" AND ") - .append(Field.O).append(":\"").append(ClientUtils.escapeQueryChars(asNt(triple.getObject()))).append("\"") - .toString(); - } - // TODO: To be optimized...with this implementation wildcard queries are not supported // so if I need to delete 5 triples then 5 commands should be issued. @Override @@ -100,17 +106,14 @@ public List deleteTriples( @Override public void executePendingMutations() throws StorageLayerException { - try { - solr.commit(); - } catch (final Exception exception) { - throw new StorageLayerException(exception); - } + // Do nothing here... } + // TODO: delete without name deletes all?? @Override public void clear() { try { - solr.deleteByQuery("*:*"); + solr.deleteByQuery(name != null ? "*:*" : Field.C + ":\"" + ClientUtils.escapeQueryChars(asNtURI(name)) + "\""); } catch (final Exception exception) { logger.error(MessageCatalog._00170_UNABLE_TO_CLEAR, exception); } @@ -126,15 +129,19 @@ public Iterator query(final TripleMatch query) throws StorageLayerExcept final Node o = query.getMatchObject(); if (s != null) { - q.addFilterQuery(newFilterQuery(Field.S, ClientUtils.escapeQueryChars(asNt(s)))); + q.addFilterQuery(newFilterQuery(Field.S, asNt(s))); } if (p != null) { - q.addFilterQuery(newFilterQuery(Field.P, ClientUtils.escapeQueryChars(asNtURI(p)))); + q.addFilterQuery(newFilterQuery(Field.P, asNtURI(p))); } if (o != null) { - q.addFilterQuery(newFilterQuery(Field.O, ClientUtils.escapeQueryChars(asNt(o)))); + q.addFilterQuery(newFilterQuery(Field.O, asNt(o))); + } + + if (name != null) { + q.addFilterQuery(newFilterQuery(Field.C, asNtURI(name))); } return new SolrDeepPagingIterator(solr, q); @@ -156,4 +163,23 @@ String newFilterQuery(final String fieldName, final String value) { .toString(); } + /** + * Builds a delete query starting from a given triple. + * + * @param triple the triple. + * @return a delete query starting from a given triple. + */ + String deleteQuery(final Triple triple) { + + final StringBuilder builder = new StringBuilder() + .append(Field.S).append(":\"").append(ClientUtils.escapeQueryChars(asNt(triple.getSubject()))).append("\" AND ") + .append(Field.P).append(":\"").append(ClientUtils.escapeQueryChars(asNtURI(triple.getPredicate()))).append("\" AND ") + .append(Field.O).append(":\"").append(ClientUtils.escapeQueryChars(asNt(triple.getObject()))).append("\""); + + if (name != null) { + builder.append(" AND ").append(Field.C).append(":\"").append(ClientUtils.escapeQueryChars(asNtURI(name))).append("\""); + } + + return builder.toString(); + } } \ No newline at end of file diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/graph/SolrGraph.java b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/graph/SolrGraph.java index 56c67fc..150d0c4 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/graph/SolrGraph.java +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/main/java/org/gazzax/labs/jena/nosql/solr/graph/SolrGraph.java @@ -4,7 +4,7 @@ import java.util.Iterator; import org.gazzax.labs.jena.nosql.fwk.StorageLayerException; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; import org.gazzax.labs.jena.nosql.fwk.log.Log; import org.gazzax.labs.jena.nosql.fwk.log.MessageCatalog; @@ -29,38 +29,40 @@ */ public class SolrGraph extends GraphBase { private final static Log LOGGER = new Log(LoggerFactory.getLogger(SolrGraph.class)); - private final static ExtendedIterator EMPTY_TRIPLES_ITERATOR = WrappedIterator.createNoRemove(new ArrayList(0).iterator()); + + private final int deletionBatchSize; - private final TripleIndexDAO dao; - private final Node name; - + private final GraphDAO dao; + /** * Builds a new unnamed graph with the given factory. * * @param factory the storage layer factory. + * @param deletionBatchSize the batch size in case of massive deletions. */ - public SolrGraph(final StorageLayerFactory factory) { - this(null, factory); + public SolrGraph(final StorageLayerFactory factory, final int deletionBatchSize) { + this(null, factory, deletionBatchSize); } - + /** * Builds a new named graph with the given data. * * @param name the graph name. * @param factory the storage layer factory. + * @param deletionBatchSize the batch size in case of massive deletions. */ - @SuppressWarnings("unchecked") - public SolrGraph(final Node name, final StorageLayerFactory factory) { - this.name = name; - this.dao = factory.getTripleIndexDAO(); + @SuppressWarnings("unchecked") + public SolrGraph(final Node name, final StorageLayerFactory factory, final int deletionBatchSize) { + this.deletionBatchSize = deletionBatchSize; + this.dao = name != null ? factory.getGraphDAO(name) : factory.getGraphDAO(); } @Override public void performAdd(final Triple triple) { try { dao.insertTriple(triple); - dao.executePendingMutations(); +// dao.executePendingMutations(); } catch (final StorageLayerException exception) { final String message = MessageFactory.createMessage(MessageCatalog._00101_UNABLE_TO_ADD_TRIPLE, triple); LOGGER.error(message, exception); @@ -73,13 +75,10 @@ public void performDelete(final Triple triple) { try { if (triple.isConcrete()) { dao.deleteTriple(triple); - } else if (triple.getSubject().isConcrete() && - triple.getPredicate().isConcrete() && - triple.getObject().isConcrete()){ + } else if ( !triple.getSubject().isConcrete() && !triple.getPredicate().isConcrete() && !triple.getObject().isConcrete()){ clear(); } else { - // TODO: batch size must be configurable - dao.deleteTriples(query(triple), 1000); + dao.deleteTriples(query(triple), deletionBatchSize); } } catch (final StorageLayerException exception) { final String message = MessageFactory.createMessage(MessageCatalog._00100_UNABLE_TO_DELETE_TRIPLE, triple); @@ -89,16 +88,14 @@ public void performDelete(final Triple triple) { } @Override - public void clear() - { + public void clear() { dao.clear(); getEventManager().notifyEvent(this, GraphEvents.removeAll ) ; } @Override - protected ExtendedIterator graphBaseFind(final TripleMatch pattern) { - try - { + public ExtendedIterator graphBaseFind(final TripleMatch pattern) { + try { return WrappedIterator.createNoRemove(query(pattern)); } catch (StorageLayerException exception) { LOGGER.error(MessageCatalog._00010_DATA_ACCESS_LAYER_FAILURE, exception); diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/main/resources/jena-nosql-default.yaml b/jena-nosql-binding/jena-nosql-binding-solr/src/main/resources/jena-nosql-default.yaml index 2daff6d..2b2e52e 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/main/resources/jena-nosql-default.yaml +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/main/resources/jena-nosql-default.yaml @@ -3,6 +3,6 @@ ## JENA-NOSQL default configuration ## ########################################### -# SorlServer implementation solr-server-class: "org.apache.solr.client.solrj.impl.HttpSolrServer" solr-address: "http://127.0.0.1:8080/solr/store" +delete-batch-size: 1000 diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/schema.xml b/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/schema.xml index f2bf4ea..bddd4a6 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/schema.xml +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/schema.xml @@ -6,11 +6,9 @@ - id - - + diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/solrconfig.xml b/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/solrconfig.xml index 82893e2..7fe17ae 100644 --- a/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/solrconfig.xml +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/solr-home/plain-store/conf/solrconfig.xml @@ -1,4 +1,12 @@ + LUCENE_48 ${solr.data.dir}/${solr.core.name}/data @@ -9,45 +17,29 @@ ${solr.lock.type:native} - - ${solr.ulog.dir:} - 15000 false - - 1000 - 1024 - - - - - - + + + true - 20 - 200 - false 2 - + - + explicit 10 *:* @@ -61,14 +53,9 @@ - - - - + + - *:* @@ -77,7 +64,6 @@ all - true diff --git a/jena-nosql-binding/jena-nosql-binding-solr/src/test/java/RemoveMe.java b/jena-nosql-binding/jena-nosql-binding-solr/src/test/java/RemoveMe.java new file mode 100644 index 0000000..4b12532 --- /dev/null +++ b/jena-nosql-binding/jena-nosql-binding-solr/src/test/java/RemoveMe.java @@ -0,0 +1,30 @@ +import static org.gazzax.labs.jena.nosql.fwk.TestUtility.DUMMY_BASE_URI; + +import java.io.File; + +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; + +import com.hp.hpl.jena.query.Dataset; +import com.hp.hpl.jena.query.DatasetFactory; +import com.hp.hpl.jena.rdf.model.Model; + + +public class RemoveMe { + public static void main(String[] args) { + StorageLayerFactory factory = StorageLayerFactory.getFactory(); + Dataset dataset = DatasetFactory.create(factory.getDatasetGraph()); + + RDFDataMgr.read( + factory.getDatasetGraph(), + new File("/work/data/jena-nosql/triples_gridpedia.nt").toURI().toString(), + DUMMY_BASE_URI, + Lang.NTRIPLES); +// final Model model = dataset.getDefaultModel().read(new File("/work/data/jena-nosql/triples_gridpedia.nt").toURI().toString(), DUMMY_BASE_URI, "N-TRIPLE"); + +// System.out.println(model.size()); + System.out.println("RemoveMe.main()"); + factory.getClientShutdownHook().close(); + } +} diff --git a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/TripleIndexDAO.java b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/GraphDAO.java similarity index 94% rename from jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/TripleIndexDAO.java rename to jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/GraphDAO.java index 42c4454..36e6d7e 100644 --- a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/TripleIndexDAO.java +++ b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/ds/GraphDAO.java @@ -6,7 +6,7 @@ import org.gazzax.labs.jena.nosql.fwk.StorageLayerException; /** - * Data Access Object that encapsulates the interaction with a triple index. + * Data Access Object that encapsulates the persistence of a graph. * Each concrete storage implementation must define here how to perform basic triple operations. * * @author Andrea Gazzarini @@ -14,7 +14,7 @@ * @param how this DAO represents a triple. * @param

how this DAO represents a triple pattern. */ -public interface TripleIndexDAO { +public interface GraphDAO { /** * Inserts a triple. * diff --git a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/factory/StorageLayerFactory.java b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/factory/StorageLayerFactory.java index 4f045e9..d371a5d 100644 --- a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/factory/StorageLayerFactory.java +++ b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/factory/StorageLayerFactory.java @@ -9,7 +9,7 @@ import org.gazzax.labs.jena.nosql.fwk.configuration.DefaultConfigurator; import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; import org.gazzax.labs.jena.nosql.fwk.ds.MapDAO; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.graph.NoSqlDatasetGraph; import org.gazzax.labs.jena.nosql.fwk.graph.NoSqlGraph; @@ -42,6 +42,8 @@ public abstract class StorageLayerFactory implements Configurable { default_factory = iterator.hasNext() ? iterator.next() : null; } + protected int deletionBatchSize; + /** * Returns the {@link MapDAO}. * A {@link MapDAO} instance is required in order to manage the persistent logic of a BIndex. @@ -61,20 +63,32 @@ public abstract MapDAO getMapDAO( String name); /** - * Returns the Data Access Object for interacting with the triple index. + * Returns the Data Access Object for interacting with a (named) graph. + * If the given name is null, then the dao is supposed to be associated with an unnamed graph. + * In this case, please consider the appropriate method, which should have the same contract but with a meaningful interface. * + * @param name the name of the graph that will be associated with the resulting dao (null in case of unnamed graph). * @return the Data Access Object for interacting with the triple index. */ @SuppressWarnings("rawtypes") - public abstract TripleIndexDAO getTripleIndexDAO(); - + public abstract GraphDAO getGraphDAO(Node name); + + /** + * Returns the Data Access Object for interacting with an unnamed graph. + * + * @param name the name of the graph that will be associated with the resulting dao. + * @return the Data Access Object for interacting with the triple index. + */ + @SuppressWarnings("rawtypes") + public abstract GraphDAO getGraphDAO(); + /** * Returns an unnamed {@link Graph} specific implementation associated with the underlying kind of storage. * * @return an unnamed {@link Graph} specific implementation associated with the underlying kind of storage. */ public Graph getGraph() { - return new NoSqlGraph(this); + return new NoSqlGraph(this, deletionBatchSize); } /** @@ -83,7 +97,7 @@ public Graph getGraph() { * @return a named {@link Graph} specific implementation associated with the underlying kind of storage. */ public Graph getGraph(Node graphNode) { - return new NoSqlGraph(graphNode, this); + return new NoSqlGraph(graphNode, this, deletionBatchSize); } /** diff --git a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlDatasetGraph.java b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlDatasetGraph.java index 718b97a..cf98466 100644 --- a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlDatasetGraph.java +++ b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlDatasetGraph.java @@ -19,7 +19,6 @@ public class NoSqlDatasetGraph extends DatasetGraphCaching { private final StorageLayerFactory factory; - /** * Builds a new Dataset graph with the given factory. * diff --git a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlGraph.java b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlGraph.java index bc13a60..2f0d9b0 100644 --- a/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlGraph.java +++ b/jena-nosql-framework/src/main/java/org/gazzax/labs/jena/nosql/fwk/graph/NoSqlGraph.java @@ -5,7 +5,7 @@ import org.gazzax.labs.jena.nosql.fwk.StorageLayerException; import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; import org.gazzax.labs.jena.nosql.fwk.log.Log; import org.gazzax.labs.jena.nosql.fwk.log.MessageCatalog; @@ -34,30 +34,35 @@ public class NoSqlGraph extends GraphBase { private final static Iterator EMPTY_IDS_ITERATOR = new ArrayList(0).iterator(); private final static ExtendedIterator EMPTY_TRIPLES_ITERATOR = WrappedIterator.createNoRemove(new ArrayList(0).iterator()); - private final TripleIndexDAO dao; + private final int deletionBatchSize; + + private final GraphDAO dao; private final TopLevelDictionary dictionary; private final Node name; /** * Builds a new unnamed graph with the given factory. * + * @param deletionBatchSize the batch size in case of massive deletions. * @param factory the storage layer factory. */ - public NoSqlGraph(final StorageLayerFactory factory) { - this(null, factory); + public NoSqlGraph(final StorageLayerFactory factory, final int deletionBatchSize) { + this(null, factory, deletionBatchSize); } /** * Builds a new named graph with the given data. * + * @param deletionBatchSize the batch size in case of massive deletions. * @param name the graph name. * @param factory the storage layer factory. */ @SuppressWarnings("unchecked") - public NoSqlGraph(final Node name, final StorageLayerFactory factory) { + public NoSqlGraph(final Node name, final StorageLayerFactory factory, final int deletionBatchSize) { this.name = name; - this.dao = factory.getTripleIndexDAO(); + this.dao = name != null ? factory.getGraphDAO(name) : factory.getGraphDAO(); this.dictionary = factory.getDictionary(); + this.deletionBatchSize = deletionBatchSize; } @Override @@ -93,13 +98,10 @@ public void performDelete(final Triple triple) { if (triple.isConcrete()) { dao.deleteTriple(identifiers); - } else if (triple.getSubject().isConcrete() && - triple.getPredicate().isConcrete() && - triple.getObject().isConcrete()){ + } else if (!triple.getSubject().isConcrete() && !triple.getPredicate().isConcrete() && !triple.getObject().isConcrete()) { clear(); } else { - // TODO: batch size must be configurable - dao.deleteTriples(query(identifiers), 1000); + dao.deleteTriples(query(identifiers), deletionBatchSize); } } catch (final StorageLayerException exception) { final String message = MessageFactory.createMessage(MessageCatalog._00100_UNABLE_TO_DELETE_TRIPLE, triple); diff --git a/jena-nosql-framework/src/test/java/org/gazzax/labs/jena/nosql/fwk/TestUtility.java b/jena-nosql-framework/src/test/java/org/gazzax/labs/jena/nosql/fwk/TestUtility.java index 3e53c85..6ccd590 100644 --- a/jena-nosql-framework/src/test/java/org/gazzax/labs/jena/nosql/fwk/TestUtility.java +++ b/jena-nosql-framework/src/test/java/org/gazzax/labs/jena/nosql/fwk/TestUtility.java @@ -12,7 +12,7 @@ import org.gazzax.labs.jena.nosql.fwk.configuration.Configuration; import org.gazzax.labs.jena.nosql.fwk.dictionary.TopLevelDictionary; import org.gazzax.labs.jena.nosql.fwk.ds.MapDAO; -import org.gazzax.labs.jena.nosql.fwk.ds.TripleIndexDAO; +import org.gazzax.labs.jena.nosql.fwk.ds.GraphDAO; import org.gazzax.labs.jena.nosql.fwk.factory.ClientShutdownHook; import org.gazzax.labs.jena.nosql.fwk.factory.StorageLayerFactory; @@ -42,8 +42,15 @@ public void accept(final Configuration> configuration) { } @Override - public TripleIndexDAO getTripleIndexDAO() { - return mock(TripleIndexDAO.class); + @SuppressWarnings("rawtypes") + public GraphDAO getGraphDAO(final Node name) { + return mock(GraphDAO.class); + } + + @Override + @SuppressWarnings("rawtypes") + public GraphDAO getGraphDAO() { + return mock(GraphDAO.class); } @Override diff --git a/jena-nosql-integration-tests/src/test/java/org/gazzax/labs/jena/nosql/fwk/SparqlIntegrationTestCase.java b/jena-nosql-integration-tests/src/test/java/org/gazzax/labs/jena/nosql/fwk/SparqlIntegrationTestCase.java index fbab3fe..30f968c 100644 --- a/jena-nosql-integration-tests/src/test/java/org/gazzax/labs/jena/nosql/fwk/SparqlIntegrationTestCase.java +++ b/jena-nosql-integration-tests/src/test/java/org/gazzax/labs/jena/nosql/fwk/SparqlIntegrationTestCase.java @@ -86,7 +86,7 @@ protected int howManyExamples() { */ @After public void tearDown() { - factory.getTripleIndexDAO().clear(); + factory.getGraphDAO().clear(); dataset.close(); factory.getClientShutdownHook().close(); } @@ -119,7 +119,7 @@ protected String readFile(final String filename) throws IOException { * @param datafileName the name of the datafile. */ protected void load(final String datafileName) { - final Model model = dataset.getDefaultModel().read(new File(EXAMPLES_DIR + File.separator + chapter() + File.separator, datafileName).toURI().toString(), DUMMY_BASE_URI, "TTL"); + final Model model = dataset.getDefaultModel().read(new File("/work/data/jena-nosql/triples_gridpedia.nt").toURI().toString(), DUMMY_BASE_URI, "TTL"); assertFalse(model.isEmpty()); } } diff --git a/jena-nosql-integration-tests/src/test/resources/log4j.xml b/jena-nosql-integration-tests/src/test/resources/log4j.xml index 8f64c20..b5ff7cd 100644 --- a/jena-nosql-integration-tests/src/test/resources/log4j.xml +++ b/jena-nosql-integration-tests/src/test/resources/log4j.xml @@ -10,7 +10,7 @@ - +