Skip to content

Commit

Permalink
Merge branch 'main' into issue_895
Browse files Browse the repository at this point in the history
  • Loading branch information
@jordanpadams
jordanpadams committed Oct 16, 2024
2 parents 8d5b86b + 3ce43db commit 1fe093a
Show file tree
Hide file tree
Showing 85 changed files with 6,817 additions and 6,712 deletions.
10 changes: 9 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,22 @@
# Changelog

## [«unknown»](https://github.com/NASA-PDS/validate/tree/«unknown») (2024-08-08)
## [«unknown»](https://github.com/NASA-PDS/validate/tree/«unknown») (2024-10-14)

[Full Changelog](https://github.com/NASA-PDS/validate/compare/v3.5.2...«unknown»)

**Defects:**

- The PDF verification / VeraPDF component of Validate seems to error on Windows paths [\#1008](https://github.com/NASA-PDS/validate/issues/1008) [[s.medium](https://github.com/NASA-PDS/validate/labels/s.medium)]
- OutOfMemoryError when NASA validate v3.5.2 is executed through a library for a batch of products [\#979](https://github.com/NASA-PDS/validate/issues/979) [[s.medium](https://github.com/NASA-PDS/validate/labels/s.medium)]
- Slow performance with all content and product validation turned off [\#969](https://github.com/NASA-PDS/validate/issues/969) [[s.medium](https://github.com/NASA-PDS/validate/labels/s.medium)]
- `--disable-context-mismatch-warnings` only works for rule `pds4.label` [\#950](https://github.com/NASA-PDS/validate/issues/950) [[s.high](https://github.com/NASA-PDS/validate/labels/s.high)]
- Validate does not show correct filename for PDF/A failures when validating a directory [\#936](https://github.com/NASA-PDS/validate/issues/936) [[s.medium](https://github.com/NASA-PDS/validate/labels/s.medium)]
- Configuration file parser does not reject incorrect options [\#923](https://github.com/NASA-PDS/validate/issues/923) [[s.medium](https://github.com/NASA-PDS/validate/labels/s.medium)]

**Other closed issues:**

- Re-organize test cases per build and tag of validate [\#1006](https://github.com/NASA-PDS/validate/issues/1006)

## [v3.5.2](https://github.com/NASA-PDS/validate/tree/v3.5.2) (2024-06-12)

[Full Changelog](https://github.com/NASA-PDS/validate/compare/v3.5.1...v3.5.2)
Expand Down
40 changes: 20 additions & 20 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.3.0</version>
<version>3.4.2</version>
<configuration>
<archive>
<manifest>
Expand All @@ -93,7 +93,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>3.2.0</version>
<version>3.2.1</version>
<executions>
<execution>
<phase>validate</phase>
Expand Down Expand Up @@ -135,7 +135,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<version>4.0.0-M15</version>
<version>4.0.0-M13</version>
<configuration>
<skip>false</skip> <!-- This is `true` in parent pom -->
<skipDeploy>false</skipDeploy> <!-- This is `true` in parent pom -->
Expand Down Expand Up @@ -174,13 +174,13 @@
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-xjc</artifactId>
<version>2.3.4</version>
<version>4.0.5</version>
</dependency>

<dependency>
<groupId>org.verapdf</groupId>
<artifactId>validation-model</artifactId>
<version>1.24.2</version>
<version>1.26.1</version>
</dependency>
<dependency>
<groupId>org.verapdf</groupId>
Expand All @@ -191,12 +191,12 @@
<dependency>
<groupId>net.sf.saxon</groupId>
<artifactId>Saxon-HE</artifactId>
<version>12.4</version>
<version>12.5</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.7.0</version>
<version>1.9.0</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand All @@ -214,7 +214,7 @@
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.3.3</version>
<version>1.3.4</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -242,7 +242,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.16.1</version>
<version>2.17.0</version>
</dependency>
<!-- NOTE FOR DEVELOPER: To test via command-line and see all logger messages
uncomment this, and comment out the NOP dependency to see both developer
Expand All @@ -261,7 +261,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-nop</artifactId>
<version>2.0.13</version>
<version>2.0.16</version>
</dependency>
<dependency>
<groupId>com.jamesmurty.utils</groupId>
Expand All @@ -276,17 +276,17 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.14.0</version>
<version>3.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.2</version>
<version>1.27.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>33.2.0-jre</version>
<version>33.3.0-jre</version>
</dependency>
<dependency>
<groupId>xerces</groupId>
Expand All @@ -301,7 +301,7 @@
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.23.1</version>
<version>2.24.1</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
Expand All @@ -311,25 +311,25 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.2</version>
<version>5.11.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java</artifactId>
<version>6.9.1</version>
<version>7.18.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit</artifactId>
<version>6.9.1</version>
<version>7.18.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<version>5.10.2</version>
<version>5.11.2</version>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -340,7 +340,7 @@
<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>pds4-jparser</artifactId>
<version>2.9.0-SNAPSHOT</version>
<version>2.9.0</version>
<scope>compile</scope>
<exclusions>
<exclusion>
Expand All @@ -366,7 +366,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.17.1</version>
<version>2.18.0</version>
</dependency>
<dependency>
<groupId>org.mp4parser</groupId>
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/gov/nasa/pds/tools/util/PDFUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,11 @@ public synchronized boolean validateFileStandardConformity(String baseDir, Strin
// directory.
String parent = parentURL.getFile();

// tired of tricks to make it work everywhere so explicitly handling windoze
if (System.getProperty("os.name").toLowerCase().startsWith("window")) {
parent = new File(parentURL.toURI()).getAbsolutePath();
}

// Build the full pathname of the PDF file.
String pdfRef = parent + File.separator + pdfBase;
LOG.debug("validateFileStandardConformity:parent,pdfBase,pdfRef [{}],[{}],[{}]", parent,
Expand Down
65 changes: 18 additions & 47 deletions src/main/java/gov/nasa/pds/tools/util/ReferentialIntegrityUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ public class ReferentialIntegrityUtil {
private static ArrayList<URL> urlsParsedCumulative = new ArrayList<>(0);
private static ArrayList<String> logicalIdentifiersCumulative = new ArrayList<>(0);
private static ArrayList<String> lidOrLidVidReferencesCumulative = new ArrayList<>(0);
private static HashSet<String> lidAndFilenameCombo = new HashSet<String>();
private static HashMap<String, HashSetReferenceInfo> contextReferencesCumulative =
new HashMap<>(0); // Collect all references defined in "Context_Area" tag from all labels.
private static HashMap<String, HashSet> bundleOrCollectionReferenceMap = new HashMap<String, HashSet>(); // Collect
private static HashMap<String, HashSet<String>> bundleOrCollectionReferenceMap = new HashMap<String, HashSet<String>>(); // Collect
// all
// references
// defined
Expand Down Expand Up @@ -170,6 +171,7 @@ public static void reset() {
// and it will be difficult to figure out why. The code may work when validate
// runs from the command line
// but not in regression test.
ReferentialIntegrityUtil.lidAndFilenameCombo.clear();
ReferentialIntegrityUtil.logicalIdentifiersCumulative.clear();
ReferentialIntegrityUtil.lidOrLidVidReferencesCumulative.clear();
ReferentialIntegrityUtil.contextReferencesCumulative.clear();
Expand Down Expand Up @@ -466,35 +468,6 @@ public static void reportLidOrLidvidReferenceToNonExistLogicalReferences() {
}
}

private static boolean hasReferenceIDAndFilenameComboAdded(String singleLidorLidVidReference,
URL filename) {
boolean referenceIDAndFilenameComboAddedFlag = false;
// Build the combo of reference and filename together from input parameters.
// Remove the use of the slash '/' to avoid confusion. We are merely looking at
// the combination of the lid_reference (or lidvid_reference) plus filename as
// strings for comparison.
String referenceIDAndFilenameComboValue = singleLidorLidVidReference + filename.toString();
for (int ii = 0; ii < ReferentialIntegrityUtil.lidOrLidVidReferencesCumulative.size(); ii++) {
// Build the combo of reference and filename together from each value in
// ReferentialIntegrityUtil.lidOrLidVidReferencesCumulative.get and
// ReferentialIntegrityUtil.lidOrLidVidReferencesCumulativeFileNames.get.
// Remove the use of the slash '/' to avoid confusion. We are merely looking at
// the combination of the lid_reference (or lidvid_reference) plus filename as
// strings for comparison.
String singleComboValue = ReferentialIntegrityUtil.lidOrLidVidReferencesCumulative.get(ii)
+ ReferentialIntegrityUtil.lidOrLidVidReferencesCumulativeFileNames.get(ii);
LOG.debug(
"hasReferenceIDAndFilenameComboAdded:referenceIDAndFilenameComboValue,singleComboValue {},{}",
referenceIDAndFilenameComboValue, singleComboValue);
if (referenceIDAndFilenameComboValue.equals(singleComboValue)) {
// If there is a compare, we have found our answer and will break out of loop.
referenceIDAndFilenameComboAddedFlag = true;
break;
}
}
return (referenceIDAndFilenameComboAddedFlag);
}

private static boolean isIdentiferMatchingBundleBaseID(String singleLogicalIdentifier) {
// Given a logical identifier, check if it contains the bundle base identifier.
// If the bundle base identifier is urn:nasa:pds:kaguya_grs_spectra
Expand Down Expand Up @@ -757,10 +730,10 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun

boolean labelIsCollectionFlag = false;
boolean labelIsBundleFlag = false;
List<Target> children = new ArrayList<>();
String parentId = null;

try {
List<Target> children = new ArrayList<>();
if (getContext().getCrawler() != null) {
children = getContext().getCrawler().crawl(crawlTarget, true, getContext().getFileFilters()); // Get also the directories.
} else {
Expand Down Expand Up @@ -790,15 +763,15 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
// local_identifier and lid_reference or lidvid_reference tags.
url = child.getUrl();

if (url.toString().endsWith("." + getContext().getLabelExtension()) && TargetExaminer.isTargetALabel(url)) {
// Check this URL has been parsed before. If yes, skip this file.
if (ReferentialIntegrityUtil.urlsParsedCumulative.contains(url)) {
LOG.info("SKIPPING_URL_TRUE:referenceType,url {},{}",
ReferentialIntegrityUtil.getReferenceType(), url);
continue;
}

// Check this URL has been parsed before. If yes, skip this file.
if (ReferentialIntegrityUtil.urlsParsedCumulative.contains(url)) {
LOG.info("SKIPPING_URL_TRUE:referenceType,url {},{}",
ReferentialIntegrityUtil.getReferenceType(), url);
continue;
}
LOG.info("SKIPPING_URL_FALSE:referenceType,url {},{}",
if (url.toString().endsWith("." + getContext().getLabelExtension()) && TargetExaminer.isTargetALabel(url)) {
LOG.info("SKIPPING_URL_FALSE:referenceType,url {},{}",
ReferentialIntegrityUtil.getReferenceType(), url);
labelIsCollectionFlag = false;
labelIsBundleFlag = false;
Expand All @@ -813,18 +786,15 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
if (TargetExaminer.isTargetCollectionType (child.getUrl())) {
labelIsCollectionFlag = true;
}

xml = db.parse(url.openStream());
domSource = new DOMSource(xml);

// Note that the function getLidVidReferences() collects all references in the
// Reference_List group in Internal_Reference tags.
// so the lidOrLidVidReferencesCumulative will be a cumulative collection of all
// references collected in lidOrLidVidReferences for each label.

ArrayList<String> lidOrLidVidReferences = LabelUtil.getLidVidReferences(domSource, url);
ArrayList<String> logicalIdentifiers = LabelUtil.getLogicalIdentifiers(domSource, url);

LOG.debug("additionalReferentialIntegrityChecks:url,lidOrLidVidReferences {},{}", url,
lidOrLidVidReferences.size());
LOG.debug("additionalReferentialIntegrityChecks:url,logicalIdentifiers {},{}", url,
Expand All @@ -848,6 +818,7 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
}

if ((lidOrLidVidReferences != null) && !lidOrLidVidReferences.isEmpty()) {
String urlstr = url.toString();
for (int ii = 0; ii < lidOrLidVidReferences.size(); ii++) {
LOG.debug(
"additionalReferentialIntegrityChecks:ii,url,lidOrLidVidReferences.get(ii) {},{},[{}]",
Expand All @@ -859,7 +830,7 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
// Note that because the reference id can be the same, the combination of the id
// plus the file name will make it unique.
if (!ReferentialIntegrityUtil
.hasReferenceIDAndFilenameComboAdded(lidOrLidVidReferences.get(ii), url)) {
.lidAndFilenameCombo.contains(lidOrLidVidReferences.get(ii) + urlstr)) {

ReferentialIntegrityUtil.lidOrLidVidReferencesCumulative
.add(lidOrLidVidReferences.get(ii));
Expand All @@ -874,7 +845,7 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
// be
// referred
// to.

ReferentialIntegrityUtil.lidAndFilenameCombo.add(lidOrLidVidReferences.get(ii) + urlstr);
LOG.debug("additionalReferentialIntegrityChecks:ADDING_REFERENCE {}",
lidOrLidVidReferences.get(ii), lidOrLidVidReferencesCumulative.size());
}
Expand Down Expand Up @@ -938,7 +909,7 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
ReferentialIntegrityUtil.referenceType, crawlTarget, bundleOrCollectionReferenceMap,
bundleOrCollectionReferenceMap.size());
}

/**
* Reports an error to the validation listener.
*
Expand All @@ -949,7 +920,7 @@ public static void additionalReferentialIntegrityChecks(URL crawlTarget, URL bun
*/
protected static void reportError(ProblemDefinition defn, URL targetUrl, int lineNumber,
int columnNumber) {
ValidationProblem problem = new ValidationProblem(defn, new ValidationTarget(targetUrl),
ValidationProblem problem = new ValidationProblem(defn, ValidationTarget.build(targetUrl),
lineNumber, columnNumber, defn.getMessage());
problemListener.addProblem(problem);
}
Expand All @@ -965,7 +936,7 @@ protected static void reportError(ProblemDefinition defn, URL targetUrl, int lin
*/
protected static void reportError(ProblemDefinition defn, URL target, int lineNumber,
int columnNumber, String message) {
ValidationProblem problem = new ValidationProblem(defn, new ValidationTarget(target),
ValidationProblem problem = new ValidationProblem(defn, ValidationTarget.build(target),
lineNumber, columnNumber, message);
problemListener.addProblem(problem);
}
Expand Down
Loading

0 comments on commit 1fe093a

Please sign in to comment.