From 2e075d526f4115d39c21fad9539cd7cebd2617f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Thu, 24 Oct 2024 15:07:52 +0200 Subject: [PATCH] GH-5148 cleanup naming and docs --- .../nativerdf/NativeStatementIterator.java | 4 +-- .../rdf4j/sail/nativerdf/NativeStore.java | 8 +++--- .../rdf4j/sail/nativerdf/TripleStore.java | 10 +++---- .../rdf4j/sail/nativerdf/ValueStore.java | 26 ++++++++++--------- .../sail/nativerdf/datastore/DataFile.java | 6 ++--- .../sail/nativerdf/model/CorruptIRI.java | 6 +++-- .../nativerdf/model/CorruptIRIOrBNode.java | 6 +++-- .../sail/nativerdf/model/CorruptLiteral.java | 6 +++-- .../nativerdf/model/CorruptUnknownValue.java | 6 +++-- .../sail/nativerdf/model/CorruptValue.java | 14 +++++----- .../NativeSailStoreCorruptionTest.java | 6 ++--- .../documentation/programming/repository.md | 2 +- 12 files changed, 57 insertions(+), 43 deletions(-) diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStatementIterator.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStatementIterator.java index eee0d088ac..29b803e6cb 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStatementIterator.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStatementIterator.java @@ -10,7 +10,7 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.nativerdf; -import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA; +import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES; import java.io.IOException; @@ -89,7 +89,7 @@ public Statement getNextElement() throws SailException { if (contextID != 0) { context = valueStore.getResource(contextID); } - if (SOFT_FAIL_ON_CORRUPT_DATA) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { if (subj == null) { subj = new CorruptIRIOrBNode(valueStore.getRevision(), subjID, null); } diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java index 36149498fe..e156083b8e 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java @@ -66,11 +66,13 @@ public class NativeStore extends AbstractNotifyingSail implements FederatedServi /** * Do not throw an exception when corrupt data is detected. Instead, try to return as much data as possible. * - * Variable can be set through the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData. + * Variable can be set through the system property + * org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes. */ @InternalUseOnly - public static boolean SOFT_FAIL_ON_CORRUPT_DATA = "true" - .equalsIgnoreCase(System.getProperty("org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData"));; + public static boolean SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES = "true" + .equalsIgnoreCase( + System.getProperty("org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes"));; private static final Cleaner REMOVE_STORES_USED_FOR_MEMORY_OVERFLOW = Cleaner.create(); diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/TripleStore.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/TripleStore.java index a546ee02da..3c060af663 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/TripleStore.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/TripleStore.java @@ -295,12 +295,12 @@ private void initIndexes(Set indexSpecs) throws IOException { try { indexes.add(new TripleIndex(fieldSeq, false)); } catch (Exception e) { - if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA) { + if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { invalidIndexes.add(fieldSeq); logger.warn("Ignoring index because it failed to initialize index '{}'", fieldSeq, e); } else { logger.error( - "Failed to initialize index '{}', consider setting org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true.", + "Failed to initialize index '{}', consider setting org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true.", fieldSeq, e); throw e; } @@ -309,7 +309,7 @@ private void initIndexes(Set indexSpecs) throws IOException { } - if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA) { + if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { indexSpecs.removeAll(invalidIndexes); } @@ -319,12 +319,12 @@ private void initIndexes(Set indexSpecs) throws IOException { checkIfIndexesAreEmptyOrNot(nonEmptyIndexes, emptyIndexes); if (!emptyIndexes.isEmpty() && !nonEmptyIndexes.isEmpty()) { - if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA) { + if (NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { indexes.removeAll(emptyIndexes); } else { for (TripleIndex index : emptyIndexes) { throw new IOException("Index '" + new String(index.getFieldSeq()) - + "' is unexpectedly empty while other indexes are not. Consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true. Index file: " + + "' is unexpectedly empty while other indexes are not. Consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true. Index file: " + index.getBTree().getFile().getAbsolutePath()); } } diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/ValueStore.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/ValueStore.java index 8ed2f97948..6a8f1260b7 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/ValueStore.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/ValueStore.java @@ -10,7 +10,7 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.nativerdf; -import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA; +import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES; import java.io.File; import java.io.IOException; @@ -223,11 +223,11 @@ public T getResource(int id) throws IOExcepti NativeValue resultValue = getValue(id); if (resultValue != null && !(resultValue instanceof Resource)) { - if (SOFT_FAIL_ON_CORRUPT_DATA && resultValue instanceof CorruptValue) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES && resultValue instanceof CorruptValue) { return (T) new CorruptIRIOrBNode(revision, id, ((CorruptValue) resultValue).getData()); } logger.warn( - "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); } return (T) resultValue; @@ -245,14 +245,14 @@ public T getIRI(int id) throws IOException { NativeValue resultValue = getValue(id); if (resultValue != null && !(resultValue instanceof IRI)) { - if (SOFT_FAIL_ON_CORRUPT_DATA && resultValue instanceof CorruptValue) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES && resultValue instanceof CorruptValue) { if (resultValue instanceof CorruptIRI) { return (T) resultValue; } return (T) new CorruptIRI(revision, id, null, ((CorruptValue) resultValue).getData()); } logger.warn( - "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); } return (T) resultValue; @@ -586,12 +586,12 @@ private boolean isNamespaceData(byte[] data) { private NativeValue data2value(int id, byte[] data) throws IOException { if (data.length == 0) { - if (SOFT_FAIL_ON_CORRUPT_DATA) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { logger.error("Soft fail on corrupt data: Empty data array for value with id {}", id); return new CorruptUnknownValue(revision, id, data); } throw new SailException("Empty data array for value with id " + id - + " consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + + " consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); } switch (data[0]) { case URI_VALUE: @@ -601,12 +601,12 @@ private NativeValue data2value(int id, byte[] data) throws IOException { case LITERAL_VALUE: return data2literal(id, data); default: - if (SOFT_FAIL_ON_CORRUPT_DATA) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { logger.error("Soft fail on corrupt data: Invalid type {} for value with id {}", data[0], id); return new CorruptUnknownValue(revision, id, data); } throw new SailException("Invalid type " + data[0] + " for value with id " + id - + " consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + + " consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); } } @@ -621,11 +621,12 @@ private T data2uri(int id, byte[] data) throws IOE return (T) new NativeIRI(revision, namespace, localName, id); } catch (Throwable e) { - if (SOFT_FAIL_ON_CORRUPT_DATA && (e instanceof Exception || e instanceof AssertionError)) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES + && (e instanceof Exception || e instanceof AssertionError)) { return (T) new CorruptIRI(revision, id, namespace, data); } logger.error( - "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + "Possible corrupt data consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); throw e; } @@ -663,7 +664,8 @@ private T data2literal(int id, byte[] data) th return (T) new NativeLiteral(revision, label, CoreDatatype.XSD.STRING, id); } } catch (Throwable e) { - if (SOFT_FAIL_ON_CORRUPT_DATA && (e instanceof Exception || e instanceof AssertionError)) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES + && (e instanceof Exception || e instanceof AssertionError)) { return (T) new CorruptLiteral(revision, id, data); } throw e; diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/datastore/DataFile.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/datastore/DataFile.java index 10d98ab3b4..ab22e31056 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/datastore/DataFile.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/datastore/DataFile.java @@ -10,7 +10,7 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.nativerdf.datastore; -import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA; +import static org.eclipse.rdf4j.sail.nativerdf.NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES; import java.io.Closeable; import java.io.File; @@ -205,7 +205,7 @@ public byte[] getData(long offset) throws IOException { // If the data length is larger than 750MB, we are likely reading the wrong data. Probably data corruption. if (dataLength > 128 * 1024 * 1024) { - if (SOFT_FAIL_ON_CORRUPT_DATA) { + if (SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES) { logger.error( "Data length is {}MB which is larger than 750MB. This is likely data corruption. Truncating length to 32 MB.", dataLength / ((1024 * 1024))); @@ -244,7 +244,7 @@ public byte[] getData(long offset) throws IOException { } catch (OutOfMemoryError e) { if (dataLength > 128 * 1024 * 1024) { logger.error( - "Trying to read large amounts of data may be a sign of data corruption. Consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData to true"); + "Trying to read large amounts of data may be a sign of data corruption. Consider setting the system property org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes to true"); } throw e; } diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRI.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRI.java index 819e13cae5..71816d29e4 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRI.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRI.java @@ -15,13 +15,15 @@ import org.apache.commons.codec.binary.Hex; import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.sail.nativerdf.NativeStore; import org.eclipse.rdf4j.sail.nativerdf.ValueStoreRevision; import com.google.common.net.UrlEscapers; /** - * CorruptIRI is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled (see - * ValueStore#softFailOnCorruptData). + * CorruptIRI is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled + * + * @see NativeStore#SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES . * * @author Håvard M. Ottestad */ diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRIOrBNode.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRIOrBNode.java index 740530a21d..83cdb9e665 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRIOrBNode.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptIRIOrBNode.java @@ -16,13 +16,15 @@ import org.apache.commons.codec.binary.Hex; import org.eclipse.rdf4j.model.BNode; import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.sail.nativerdf.NativeStore; import org.eclipse.rdf4j.sail.nativerdf.ValueStoreRevision; import com.google.common.net.UrlEscapers; /** - * CorruptIRIOrBNode is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled (see - * ValueStore#softFailOnCorruptData). + * CorruptIRIOrBNode is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled + * + * @see NativeStore#SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES . * * @author Håvard M. Ottestad */ diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptLiteral.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptLiteral.java index 0a8bef5552..eb6b2587c2 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptLiteral.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptLiteral.java @@ -22,11 +22,13 @@ import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.base.CoreDatatype; import org.eclipse.rdf4j.model.util.Values; +import org.eclipse.rdf4j.sail.nativerdf.NativeStore; import org.eclipse.rdf4j.sail.nativerdf.ValueStoreRevision; /** - * CorruptLiteral is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled (see - * ValueStore#softFailOnCorruptData). + * CorruptLiteral is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled + * + * @see NativeStore#SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES . * * @author Håvard M. Ottestad */ diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptUnknownValue.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptUnknownValue.java index 9af3748108..ea200b55fa 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptUnknownValue.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptUnknownValue.java @@ -22,11 +22,13 @@ import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.base.CoreDatatype; import org.eclipse.rdf4j.model.vocabulary.XSD; +import org.eclipse.rdf4j.sail.nativerdf.NativeStore; import org.eclipse.rdf4j.sail.nativerdf.ValueStoreRevision; /** - * CorruptUnknownValue is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled (see - * ValueStore#softFailOnCorruptData). Since a type is needed + * CorruptUnknownValue is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled + * + * @see NativeStore#SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES . * * @author Håvard M. Ottestad */ diff --git a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptValue.java b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptValue.java index 53a3576e44..94028b5c57 100644 --- a/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptValue.java +++ b/core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/model/CorruptValue.java @@ -11,15 +11,17 @@ package org.eclipse.rdf4j.sail.nativerdf.model; +import org.eclipse.rdf4j.sail.nativerdf.NativeStore; import org.eclipse.rdf4j.sail.nativerdf.ValueStoreRevision; /** - * CorruptValue is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled (see - * ValueStore#softFailOnCorruptData). - *

- * There is no method isCorruptValue() as it would exist for a "regular" implementation of NativeValue. Since - * CorruptValue is only to be used in exceptional situations, the recommended way of checking for it is using - * "instanceof". + * CorruptValue is used when a NativeValue cannot be read from the ValueStore and if soft failure is enabled + * + * @see NativeStore#SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES . + *

+ * There is no method isCorruptValue() as it would exist for a "regular" implementation of NativeValue. Since + * CorruptValue is only to be used in exceptional situations, the recommended way of checking for it is using + * "instanceof". * * @author Hannes Ebner */ diff --git a/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStoreCorruptionTest.java b/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStoreCorruptionTest.java index 262383429c..12119ceb50 100644 --- a/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStoreCorruptionTest.java +++ b/core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStoreCorruptionTest.java @@ -97,7 +97,7 @@ public void before() throws IOException { backupFile(dataDir, "triples-spoc.alloc"); backupFile(dataDir, "triples-spoc.dat"); - NativeStore.SOFT_FAIL_ON_CORRUPT_DATA = true; + NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES = true; } @@ -316,7 +316,7 @@ public void testCorruptValuesPoscDataFile() throws IOException { long fileSize = nativeStoreFile.length(); for (long i = 4; i < fileSize; i++) { - NativeStore.SOFT_FAIL_ON_CORRUPT_DATA = true; + NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES = true; restoreFile(dataDir, file); overwriteByteInFile(nativeStoreFile, i, 0x0); repo.init(); @@ -391,7 +391,7 @@ private List getStatements() { @AfterEach public void after() throws IOException { - NativeStore.SOFT_FAIL_ON_CORRUPT_DATA = false; + NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES = false; repo.shutDown(); } } diff --git a/site/content/documentation/programming/repository.md b/site/content/documentation/programming/repository.md index 7aab3115d0..691a839c74 100644 --- a/site/content/documentation/programming/repository.md +++ b/site/content/documentation/programming/repository.md @@ -98,7 +98,7 @@ import org.eclipse.rdf4j.sail.nativerdf.NativeStore; Repository repo = new SailRepository(new NativeStore()); ``` -In the unlikely event of corruption the system property `org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptData` can be set to `true` to +In the unlikely event of corruption the system property `org.eclipse.rdf4j.sail.nativerdf.softFailOnCorruptDataAndRepairIndexes` can be set to `true` to allow the NativeStore to output CorruptValue/CorruptIRI/CorruptIRIOrBNode/CorruptLiteral objects. Take a backup of all data before setting this property as it allows the NativeStore to delete corrupt indexes in an attempt to recreate them. Consider this feature experimental and use with caution.