New --complete-descriptions flag to warn for data not described by metadata in label

src/main/java/gov/nasa/pds/tools/label/LocationValidator.java

@@ -433,6 +433,9 @@ public void setRule(String ruleName) {
   public void setEveryN(int value) {
     ruleContext.setEveryN(value);
   }
+  public void setCompleteDescriptions(boolean b) {
+    ruleContext.setCompleteDescriptions(b);
+  }
   public void setSpotCheckData(int value) {
     ruleContext.setSpotCheckData(value);
   }

src/main/java/gov/nasa/pds/tools/validate/ProblemType.java

@@ -174,6 +174,8 @@ public enum ProblemType {
 
   BAD_SCHEMATYPENS("warning.label.bad_schematypens"),
 
+  DATA_NOT_DESCRIBED("warning.data.not_described"),
+
   MISSING_SCHEMATYPENS("warning.label.missing_schematypens"),
 
   SCHEMATRON_WARNING("warning.label.schematron"),

src/main/java/gov/nasa/pds/tools/validate/rule/RuleContext.java

@@ -122,6 +122,12 @@ public class RuleContext extends ContextBase {
    * The key used to indicate how many lines or records to skip during content validation.
    */
   public static final String EVERY_N_KEY = "validate.every-n";
+
+  /**
+   * The key used to indicate enable/disable of every bit account for in file area
+   */
+  public static final String COMPLETE_DESCRIPTIONS = "validate.complete-descriptions";
+
   /**
    * Property to specify directory for PDF error directory.
    */
@@ -404,14 +410,18 @@ public void setCheckData(boolean flag) {
   public int getEveryN() {
 	return getContextValue(EVERY_N_KEY, Integer.class) == null ? 1 : getContextValue(EVERY_N_KEY, Integer.class);
   }
-
   public void setEveryN(int value) {
 	putContextValue(EVERY_N_KEY, value);
   }
+  public boolean getCompleteDescriptions() {
+    return getContextValue(COMPLETE_DESCRIPTIONS, Boolean.class) == null ? false : getContextValue(COMPLETE_DESCRIPTIONS, boolean.class);
+  }
+  public void setCompleteDescriptions(boolean b) {
+    putContextValue(COMPLETE_DESCRIPTIONS, b);
+  }
   public String getPDFErrorDir() {
     return getContextValue(PDF_ERROR_DIR, String.class) == null ? "" : getContextValue(PDF_ERROR_DIR, String.class);
   }
-
   public void setPDFErrorDir(String dir) {
     putContextValue(PDF_ERROR_DIR, dir);
   }

src/main/java/gov/nasa/pds/tools/validate/rule/pds4/DataDefinitionAndContentValidationRule.java

@@ -1,5 +1,6 @@
 package gov.nasa.pds.tools.validate.rule.pds4;
 
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
@@ -79,6 +80,15 @@ public void validate() throws MalformedURLException, URISyntaxException {
         // e.g. File_Area_Observational + File_Area_Observational_Supplemental
 
         if (!obj.getDataFile().equals(previousDataFile)) {
+          if (previousDataFile != null) {
+            long filesize = new File(previousDataFile.getPath()).length();
+            if (this.getContext().getCompleteDescriptions() && minimumExpectedOffset < filesize) {
+              getListener().addProblem(new ValidationProblem(
+                  new ProblemDefinition(ExceptionType.WARNING, ProblemType.DATA_NOT_DESCRIBED,
+                      "Data not described at the end of the file: " + (filesize - minimumExpectedOffset) + " bytes"),
+                  getTarget(), objectCounter, -1));
+            }
+          }
           tableCounter = 0;
           arrayCounter = 0;
           headerCounter = 0;
@@ -109,9 +119,16 @@ public void validate() throws MalformedURLException, URISyntaxException {
           // anything right now)
           headerCounter++;
         }
-
       }
-
+      if (previousDataFile != null) {
+        long filesize = new File(previousDataFile.getPath()).length();
+        if (this.getContext().getCompleteDescriptions() && minimumExpectedOffset < filesize) {
+          getListener().addProblem(new ValidationProblem(
+              new ProblemDefinition(ExceptionType.WARNING, ProblemType.DATA_NOT_DESCRIBED,
+                  "Data not described at the end of the file: " + (filesize - minimumExpectedOffset) + " bytes"),
+              getTarget(), objectCounter, -1));
+        }
+      }
       LOG.debug("arrays validated: {}", arrayCounter);
       LOG.debug("tables validated: {}", tableCounter);
       LOG.debug("headers validated: {}", headerCounter);

src/main/java/gov/nasa/pds/validate/ValidateLauncher.java

@@ -207,6 +207,9 @@ public class ValidateLauncher {
   /** The validation rule name to use. */
   private String validationRule;
 
+  /** flag to enable/disable (true/false) that every bit is account for within a file area */
+  private boolean completeDescriptions;
+
   /** Flag to enable/disable data content validation. */
   private boolean contentValidationFlag;
 
@@ -274,6 +277,7 @@ public ValidateLauncher() throws TransformerConfigurationException {
     contextReferenceCheck = true;
     skipProductValidation = false;
     maxErrors = MAX_ERRORS;
+    completeDescriptions = false;
     everyN = 1;
     spotCheckData = -1;
     allowUnlabeledFiles = false;
@@ -333,6 +337,7 @@ public void query(CommandLine line) throws Exception {
       throw new InvalidOptionException(
         "Could not parse value '" + line.getOptionValue("everyN", "1") + "': " + a.getMessage());
     }
+    setCompleteDescriptions(line.hasOption("complete-descriptions"));
     setPDFErrorDir(line.getOptionValue("pdf-error-dir", ""));
     File dir = new File(pdfErrorDir);
     if (!this.pdfErrorDir.isEmpty() && !dir.isDirectory()) {
@@ -767,6 +772,9 @@ public void query(File configuration) throws ConfigurationException {
       if (config.containsKey(ConfigKey.EVERY_N)) {
           setEveryN(config.getInt(ConfigKey.EVERY_N));
       }
+      if (config.containsKey(ConfigKey.COMPLETE_DESCRIPTIONS)) {
+        setCompleteDescriptions(true);
+      }
       if (config.containsKey(ConfigKey.PDF_ERROR_DIR)) {
         setPDFErrorDir(config.getString(ConfigKey.PDF_ERROR_DIR));
       }
@@ -1083,6 +1091,10 @@ public void setEveryN(int value) {
     this.everyN = value;
   }
 
+  public void setCompleteDescriptions (boolean b) {
+    this.completeDescriptions = b;
+  }
+
   public void setPDFErrorDir(String dir) {
     this.pdfErrorDir = dir;
   }
@@ -1321,6 +1333,9 @@ public void setupReport() throws IOException {
     if (everyN != 1) {
       report.addParameter("   Data Every N                  " + everyN);
     }
+    if (this.completeDescriptions) {
+      report.addParameter("   Complete Descriptions         true");
+    }
     if (!pdfErrorDir.isEmpty()) {
       report.addParameter("   PDF Error Directory           " + pdfErrorDir);
     }
@@ -1401,6 +1416,7 @@ public boolean doValidation(Map<URL, String> checksumManifest) throws Exception
         validator.setCheckData(contentValidationFlag);
         validator.setSpotCheckData(spotCheckData);
         validator.setEveryN(everyN);
+        validator.setCompleteDescriptions(this.completeDescriptions);
         validator.setPDFErrorDir(pdfErrorDir);
         validator.setAllowUnlabeledFiles(allowUnlabeledFiles);
         validator.setValidateContext(validateContext);

src/main/java/gov/nasa/pds/validate/commandline/options/ConfigKey.java

@@ -116,6 +116,10 @@ public class ConfigKey {
    */
   public static final String EVERY_N = "validate.everyN";
 
+  /**
+   * Property to enable/disable (true/false) full bit checking of tables and arrays
+   */
+  public static final String COMPLETE_DESCRIPTIONS = "validate.completeDescriptions";
   /**
    * Property to specify directory for PDF error directory.
    */

src/main/java/gov/nasa/pds/validate/commandline/options/Flag.java

@@ -47,6 +47,8 @@ public enum Flag {
   CATALOG("C", "catalog", "catalog files", String.class, true,
       "Specify catalog files to use during validation."),
 
+  COMPLETE_DESCRIPTIONS(null, "complete-descriptions",
+      "File Areas are fully described in the sense that every bit is described within the file."),
   /**
    * Flag to specify a configuration file to configure the tool behavior.
    */

src/main/java/gov/nasa/pds/validate/commandline/options/FlagOptions.java

@@ -49,6 +49,7 @@ public class FlagOptions {
     options.addOption(new ToolsOption(Flag.BASE_PATH));
     options.addOption(new ToolsOption(Flag.CATALOG));
     options.addOption(new ToolsOption(Flag.CHECKSUM_MANIFEST));
+    options.addOption(new ToolsOption(Flag.COMPLETE_DESCRIPTIONS));
     options.addOption(new ToolsOption(Flag.CONFIG));
     options.addOption(new ToolsOption(Flag.MAX_ERRORS));
     options.addOption(new ToolsOption(Flag.EXTENSION));

src/test/resources/features/validate.feature

@@ -63,6 +63,10 @@ Scenario Outline: Execute validate command for tests below.
 # Validate#561
 |"NASA-PDS/validate#597 Success Filenanme Does Not Contain Bundle/Collection" | "github561" | 0 | "0 errors expected" | "totalWarnings" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github561_1.json -s json -R pds4.collection --label-extension lblx --skip-context-validation -t {resourceDir}/github561" | "report_github561_1.json" |
 
+# Validate#535
+|"NASA-PDS/validate#535 Warning when excess table content" | "github535" | 1 | "1 warnings expected" | "DATA_NOT_DESCRIBED" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github535_1.json -s json -t {resourceDir}/github535/uvis_euv_2008_003_solar_time_series_ingress.xml --complete-descriptions" | "report_github535_1.json" |
+|"NASA-PDS/validate#535 Warning when excess table content" | "github535" | 1 | "1 errors expected" | "TABLE_DEFINITION_PROBLEM" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github535_2.json -s json -t {resourceDir}/github535/uvis_euv_2016_288_solar_time_series_egress.xml --complete-descriptions" | "report_github535_2.json" |
+
 # Validate#531
 |"NASA-PDS/validate#531 Success Binary Field Group Lengths" | "github531" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github531_1.json -s json -t {resourceDir}/github531/success/b.xml" | "report_github531_1.json" |
 |"NASA-PDS/validate#531 Fail Binary Field Group Lengths" | "github531" | 1 | "1 errors expected" | "TABLE_DEFINITION_PROBLEM" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github531_2.json -s json -t {resourceDir}/github531/fail/b.xml" | "report_github531_2.json" |