427: spaces in directory names (#746)

* try making a test first * try it this way * try it this way * try quotes - this is becomming untestable via standard tools * shrink the data size * try url encoding it * update cucumber to change %20 back to space * code changes to not do url.getPath() directly --------- Co-authored-by: Al Niessner <[email protected]>
NASA-PDS · Nov 1, 2023 · 88d2a9e · 88d2a9e
1 parent a849607
commit 88d2a9e
Show file tree

Hide file tree

Showing 16 changed files with 412 additions and 37 deletions.
diff --git a/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java b/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java
@@ -19,7 +19,6 @@
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -48,6 +47,7 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Attr;
@@ -396,8 +396,7 @@ private void checkSchemaSchematronVersions(ProblemHandler handler, URL url) {
         for (String schema : specifiedSchema) {
           try {
             URL schemaURL = new URL(schema);
-            String schemaName =
-                Paths.get(schemaURL.getPath()).getFileName().toString().toLowerCase();
+            String schemaName = schemaURL.getFile().toLowerCase();
             if (schemaName.endsWith(".xsd")) {
               schemaName = schemaName.substring(0, schemaName.length() - 4);
               schemas.add(schemaName);
@@ -412,8 +411,7 @@ private void checkSchemaSchematronVersions(ProblemHandler handler, URL url) {
               String modelHREF = part.substring(part.indexOf('"') + 1, part.lastIndexOf('"'));
               try {
                 URL schematronURL = new URL(modelHREF);
-                String schematronName =
-                    Paths.get(schematronURL.getPath()).getFileName().toString().toLowerCase();
+                String schematronName = schematronURL.getFile().toLowerCase();
                 if (schematronName.endsWith(".sch")) {
                   schematronName = schematronName.substring(0, schematronName.length() - 4);
                   schematrons.add(schematronName);

diff --git a/src/main/java/gov/nasa/pds/tools/util/DocumentUtil.java b/src/main/java/gov/nasa/pds/tools/util/DocumentUtil.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -174,11 +175,11 @@ public String readFile(URL fileUrl) {
     //
     // Using alternative method to get the parent.
     String parent = "";
-    if (fileUrl.getPath().lastIndexOf("/") < 0) {
-      LOG.error("The path does not contain a file separator {}", fileUrl.getPath());
+    if (FileUtils.toFile(fileUrl).getPath().lastIndexOf("/") < 0) {
+      LOG.error("The path does not contain a file separator {}", FileUtils.toFile(fileUrl).getPath());
       return (null);
     }
-    parent = fileUrl.getPath().substring(0, fileUrl.getPath().lastIndexOf("/"));
+    parent = FileUtils.toFile(fileUrl).getPath().substring(0, FileUtils.toFile(fileUrl).getPath().lastIndexOf("/"));
     LOG.debug("readFile:fileUrl,parent,FilenameUtils.getName(fileUrl) {},{},{}", fileUrl, parent,
         FilenameUtils.getName(fileUrl.toString()));
 

diff --git a/src/main/java/gov/nasa/pds/tools/util/FileSizesUtil.java b/src/main/java/gov/nasa/pds/tools/util/FileSizesUtil.java
@@ -6,6 +6,7 @@
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLDecoder;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,10 +34,10 @@ public static long getExternalFilesize(URL url) throws Exception {
     if (url.getProtocol().equals("file")) {
       long fileSize = -1;
       LOG.debug("getExternalFilesize:url,fileSize {},{}", url, fileSize);
-      LOG.debug("getExternalFilesize:afor:path [{}]", url.getPath());
+      LOG.debug("getExternalFilesize:afor:path [{}]", FileUtils.toFile(url).getPath());
       // For some strange reason, the File class does not handle well with the "%20"
       // in the name. It needs to be an actual space.
-      String actualPath = URLDecoder.decode(url.getPath(), "UTF-8"); // Use the URLDecoder.decode()
+      String actualPath = URLDecoder.decode(FileUtils.toFile(url).getPath(), "UTF-8"); // Use the URLDecoder.decode()
                                                                      // to convert
                                                                      // from %20 to " ".
       LOG.debug("getExternalFilesize:after:path [{}]", actualPath);

diff --git a/src/main/java/gov/nasa/pds/tools/util/PDFUtil.java b/src/main/java/gov/nasa/pds/tools/util/PDFUtil.java
@@ -62,8 +62,6 @@ private synchronized void writeErrorToFile(String baseDir, String pdfFullName, V
     FileWriter myWriter = null;
     try {
       myWriter = new FileWriter(this.externalErrorFilename);
-      // myWriter.write("Error messages for PDF file " + uri.getPath() + " using
-      // flavor " + this.parserFlavor + "\n") ;
       myWriter.write("PDF_FILE " + pdfFullName + " PARSER_FLAVOR " + this.parserFlavor + "\n");
       String headerMessage = "getRuleId, getStatus, getMessage, getLocation";
       myWriter.write(headerMessage + "\n");

diff --git a/src/main/java/gov/nasa/pds/tools/util/Utility.java b/src/main/java/gov/nasa/pds/tools/util/Utility.java
@@ -313,7 +313,7 @@ public static TargetType getTargetType(URL source, URL label) {
     }
     if (label != null 
         && -1 < extIndex
-        && label.getPath().toLowerCase().endsWith(source.getPath().substring(extIndex).toLowerCase())) {
+        && FileUtils.toFile(label).getPath().toLowerCase().endsWith(FileUtils.toFile(source).getPath().substring(extIndex).toLowerCase())) {
       if (TargetExaminer.isTargetBundleType (source, true)) {
         return TargetType.BUNDLE;
       }

diff --git a/src/main/java/gov/nasa/pds/tools/util/VersionInfo.java b/src/main/java/gov/nasa/pds/tools/util/VersionInfo.java
@@ -30,6 +30,7 @@
 import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
+import org.apache.commons.io.FileUtils;
 
 /**
  * This class provides the means to retrieve underlying supported versions of standards.
@@ -224,7 +225,7 @@ public static String[] getResourceListing(String path) throws URISyntaxException
 
     if (dirURL.getProtocol().equals("jar")) {
       /* A JAR path */
-      String jarPath = dirURL.getPath().substring(5, dirURL.getPath().indexOf("!")); // strip out
+      String jarPath = FileUtils.toFile(dirURL).getPath().substring(5, FileUtils.toFile(dirURL).getPath().indexOf("!")); // strip out
                                                                                      // only the JAR
                                                                                      // file
       JarFile jar = new JarFile(URLDecoder.decode(jarPath, "UTF-8"));

diff --git a/src/main/java/gov/nasa/pds/tools/validate/AggregateManager.java b/src/main/java/gov/nasa/pds/tools/validate/AggregateManager.java
@@ -20,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import gov.nasa.pds.tools.label.ExceptionType;
@@ -600,7 +601,7 @@ public static ArrayList<Target> findOtherBundleFiles(URL url, String labelExtens
 
     try {
       // Get the parent directory of url and crawl for files that starts with 'bundle'
-      String dirName = (new File(url.getPath())).getParent();
+      String dirName = FileUtils.toFile(url).getParent();
       LOG.debug("findOtherBundleFiles:dirName {}", dirName);
       Crawler crawler = CrawlerFactory.newInstance(new File(dirName).toURI().toURL());
 
@@ -696,7 +697,7 @@ public static ArrayList<Target> findOtherCollectionFiles(List<Target> targetList
       for (Target crawlTarget : targetList) {
         urlCrawl = crawlTarget.getUrl();
         // Get the parent directory of url and crawl for files that starts with 'bundle'
-        String dirName = (new File(urlCrawl.getPath())).getParent();
+        String dirName = FileUtils.toFile(urlCrawl).getParent();
         LOG.debug("findOtherCollectionFiles:dirName,crawledDirectory.size {},{}", dirName,
             crawledDirectory.size());
 
@@ -828,7 +829,7 @@ public static void makeException(URL url, String location, String labelFileExten
     String parentToLocation = null;
     try {
       // The new location (as a directory) will be applicable later on.
-      parentToLocation = (new File(url.getPath())).getParent() + File.separator;
+      parentToLocation = FileUtils.toFile(url).getParent() + File.separator;
       m_location = "file:" + parentToLocation;
       LOG.debug("m_location {}", m_location);
     } catch (Exception e) {

diff --git a/src/main/java/gov/nasa/pds/tools/validate/TargetExaminer.java b/src/main/java/gov/nasa/pds/tools/validate/TargetExaminer.java
@@ -13,11 +13,11 @@
 // $Id$
 package gov.nasa.pds.tools.validate;
 
-import java.io.File;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 import javax.xml.transform.sax.SAXSource;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.InputSource;
@@ -56,16 +56,16 @@ public static boolean isTargetBundleType(URL url, boolean ignoreErrors) {
     boolean targetIsBundleFlag = false;
 
     // Do a sanity check if the file or directory exist.
-    if (!(new File(url.getPath()).exists())) {
-      LOG.error("Provided url does not exist: {}", url);
+    if (!FileUtils.toFile(url).exists()) {
+      LOG.error("Provided file does not exist: {}", FileUtils.toFile(url));
       return (targetIsBundleFlag);
     }
 
-    if ((new File(url.getPath()).isFile()) && (TargetExaminer.tagMatches(url, PRODUCT_NAME_TAG, ignoreErrors))) {
+    if (FileUtils.toFile(url).isFile() && TargetExaminer.tagMatches(url, PRODUCT_NAME_TAG, ignoreErrors)) {
       targetIsBundleFlag = true;
     }
     LOG.debug("isTargetBundleType:url,(new File(url.getPath()).isFile() {},{}", url,
-        (new File(url.getPath()).isFile()));
+        FileUtils.toFile(url).isFile());
     LOG.debug("isTargetBundleType:url,PRODUCT_NAME_TAG,targetIsBundleFlag {},{},{}", url,
         PRODUCT_NAME_TAG, targetIsBundleFlag);
 
@@ -85,16 +85,16 @@ public static boolean isTargetCollectionType(URL url, boolean ignoreErrors) {
     boolean targetIsCollectionFlag = false;
 
     // Do a sanity check if the file or directory exist.
-    if (!(new File(url.getPath()).exists())) {
-      LOG.error("Provided url does not exist: {}", url);
+    if (!FileUtils.toFile(url).exists()) {
+      LOG.error("Provided file does not exist: {}", FileUtils.toFile(url));
       return (targetIsCollectionFlag);
     }
 
-    if ((new File(url.getPath()).isFile()) && (TargetExaminer.tagMatches(url, PRODUCT_NAME_TAG, ignoreErrors))) {
+    if (FileUtils.toFile(url).isFile() && TargetExaminer.tagMatches(url, PRODUCT_NAME_TAG, ignoreErrors)) {
       targetIsCollectionFlag = true;
     }
     LOG.debug("isTargetCollectionType:url,(new File(url.getPath()).isFile() {},{}", url,
-        (new File(url.getPath()).isFile()));
+        FileUtils.toFile(url).isFile());
     LOG.debug("isTargetCollectionType:url,PRODUCT_NAME_TAG,targetIsCollectionFlag {},{},{}", url,
         PRODUCT_NAME_TAG, targetIsCollectionFlag);
 

diff --git a/src/main/java/gov/nasa/pds/tools/validate/content/AudioVideo.java b/src/main/java/gov/nasa/pds/tools/validate/content/AudioVideo.java
@@ -7,6 +7,7 @@
 import gov.nasa.pds.tools.validate.ValidationProblem;
 import gov.nasa.pds.tools.validate.ValidationTarget;
 import java.net.URL;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.mp4parser.IsoFile;
 import org.mp4parser.boxes.iso14496.part12.MovieBox;
@@ -24,7 +25,7 @@ public AudioVideo (ProblemListener listener, ValidationTarget target, URL urlRef
   public void checkMetadata (boolean audio, boolean video) {
     try {
       boolean a = false,v = false;
-      IsoFile content = new IsoFile(this.urlRef.getPath());
+      IsoFile content = new IsoFile(FileUtils.toFile(this.urlRef).getPath());
       MovieBox movie = content.getMovieBox();
       if (movie == null) {
         this.listener.addProblem(new ValidationProblem(

diff --git a/src/main/java/gov/nasa/pds/tools/validate/rule/pds4/FindUnreferencedFiles.java b/src/main/java/gov/nasa/pds/tools/validate/rule/pds4/FindUnreferencedFiles.java
@@ -21,6 +21,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.HashSet;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import gov.nasa.pds.tools.validate.SkippedItems;
@@ -69,7 +70,7 @@ private HashSet<String> buildReferencedFromSkipped() {
   }
 
   private String extractPath(URL url) {
-    Path fullpath = Paths.get(url.getPath());
+    Path fullpath = FileUtils.toFile(url).toPath();
     return fullpath.getParent().toString();
   }
 

diff --git a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java
@@ -69,6 +69,7 @@
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
@@ -1559,8 +1560,8 @@ private void printWarningCollocatedData(ArrayList<URL> alternateReferentialPaths
       try {
         LOG.debug(
             "printWarningCollocatedData:url.getPath(),(new File(url.getPath()).exists() {},{}",
-            url.getPath(), (new File(url.getPath()).exists()));
-        if (new File(url.getPath()).exists()) {
+            url.getPath(), FileUtils.toFile(url).exists());
+        if (FileUtils.toFile(url).exists()) {
           ValidationProblem p1 = new ValidationProblem(
               new ProblemDefinition(ExceptionType.WARNING, ProblemType.GENERAL_INFO,
                   "This data should be collocated with the other bundle data"),

diff --git a/src/main/java/gov/nasa/pds/web/ui/utils/DataSetValidator.java b/src/main/java/gov/nasa/pds/web/ui/utils/DataSetValidator.java
@@ -7,7 +7,6 @@
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
@@ -970,11 +969,7 @@ protected static synchronized void initMasterDictionary() {
     if (masterDictionary == null) {
       File dataDictionaryFile;
       URL dictionaryURL = DataSetValidator.class.getResource("/masterdd.full"); //$NON-NLS-1$
-      try {
-        dataDictionaryFile = new File(dictionaryURL.toURI());
-      } catch (URISyntaxException e) {
-        dataDictionaryFile = new File(dictionaryURL.getPath());
-      }
+      dataDictionaryFile = org.apache.commons.io.FileUtils.toFile(dictionaryURL);
 
       try {
         masterDictionary = DictionaryParser.parse(dataDictionaryFile);

diff --git a/src/test/java/cucumber/StepDefs.java b/src/test/java/cucumber/StepDefs.java
@@ -176,7 +176,7 @@ private String[] resolveArgumentStrings() {
       // Replace every occurence of "{resourceDir}" with actual value.
       resolvedToken = temp.replace("{reportDir}", this.reportDir);
       resolvedToken = resolvedToken.replace("{resourceDir}", this.resourceDir);
-
+      resolvedToken = resolvedToken.replace("%20", " ");
       args[argIndex++] = resolvedToken;
     }
 

diff --git a/src/test/resources/features/developer.feature b/src/test/resources/features/developer.feature
@@ -449,6 +449,8 @@ Scenario Outline: Execute validate command for tests below.
 
  |"NASA-PDS/validate#429 VALID" | "github429" | 0 | "0 warning messages expected" | "totalWarnings" | "src/test/resources" | "target/test" | "-R pds4.label -r {reportDir}/report_github429_label_valid.json -s json -t {resourceDir}/github429/EPPS_EDR_SIS.xml" | "report_github429_label_valid.json" |
 
+# Validate#427
+|"NASA-PDS/validate#427 Success with spaces in directory" | "github427" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github427.json -s json --skip-context-validation  -t {resourceDir}/github427/dir%20with%20spaces/rs_20160518_014000_udsc64_l3_e_v10.xml" | "report_github427.json" |
 
 # https://github.com/nasa-pds/validate/issues/424 Validate does not allow SIP tab file to have lines of differing lengths  
 # The checking should NOT be done based on the file extension but based on the table type.  The Table_Delimited can have variable length records.

diff --git a/src/test/resources/github427/dir with spaces/rs_20160518_014000_udsc64_l3_e_v10.tab b/src/test/resources/github427/dir with spaces/rs_20160518_014000_udsc64_l3_e_v10.tab
@@ -0,0 +1,2 @@
+00001  2016-05-18T02:24:21.556  516810329.741615      15307.329725      15297.461237    8410932000.000000      6092.292  -0.9999E+01     90245.596477  -9999999.999999          1.0010332740      1033.274000  -9999999.999999  -0.9999E+01    -99999999.999999      6098.587  -0.9999E+01      22.87      -6.25
+