Changes in unit-tests for URI-encoding and validation of manifest fil…

…enames. Signed-off-by: Indrek Jentson <[email protected]>
open-eid · Sep 1, 2017 · c72ea65 · c72ea65
1 parent 14aa2f6
commit c72ea65
Show file tree

Hide file tree

Showing 10 changed files with 127 additions and 102 deletions.
diff --git a/RELEASE-NOTES.txt b/RELEASE-NOTES.txt
@@ -1,5 +1,19 @@
 DigiDoc4J Java library release notes
 ------------------------------------
+Release 1.0.7 Beta.2
+--------------------
+Summary of the major changes since 1.0.7 Beta.1
+----------------------------------------
+* Changed TSL URL for Test mode
+* Fixed problem with URI encoding for libdigidocpp
+* Fixed problem with newline symbol in signature policy ID value
+
+Release 1.0.7 Beta.1
+--------------------
+Summary of the major changes since 1.0.6
+----------------------------------------
+* Started to use DSS version 5.0
+
 Release 1.0.6
 --------------------
 There are no major changes since 1.0.6 RC.1

diff --git a/digidoc4j.iml b/digidoc4j.iml
@@ -57,28 +57,28 @@
     <orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.5.0" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.json:json:20160810" level="project" />
     <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-common-validation-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-detailed-report-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-diagnostic-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-document:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-model:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-policy-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-reports:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-service:5.0.d4j.1" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-common-validation-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-detailed-report-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-diagnostic-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-document:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-model:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-policy-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-reports:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-service:5.0.d4j.3" level="project" />
     <orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.22" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-simple-report-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-spi:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-token:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-tsl-jaxb:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-tsl-validation:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-xades:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:validation-policy:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-common:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-cades:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-cades:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-xades:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils-apache-commons:5.0.d4j.1" level="project" />
-    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils-google-guava:5.0.d4j.1" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-simple-report-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-spi:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-token:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-tsl-jaxb:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-tsl-validation:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-xades:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:validation-policy:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-common:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-cades:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-cades:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-asic-xades:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils-apache-commons:5.0.d4j.3" level="project" />
+    <orderEntry type="library" name="Maven: org.digidoc4j.dss:dss-utils-google-guava:5.0.d4j.3" level="project" />
   </component>
 </module>
diff --git a/pom.xml b/pom.xml
@@ -92,7 +92,7 @@
     <bouncycastle.version>1.54</bouncycastle.version>
     <junit.version>4.11</junit.version>
     <dss.groupId>org.digidoc4j.dss</dss.groupId>
-    <dss.version>5.0.d4j.1</dss.version>
+    <dss.version>5.0.d4j.3</dss.version>
     <dss.util.build>${project.basedir}/build/</dss.util.build>
     <dss.util.lib>${project.basedir}/utility-package-lib</dss.util.lib>
     <dss.zip.lib>${project.basedir}/zip-package-lib</dss.zip.lib>

diff --git a/src/org/digidoc4j/impl/bdoc/manifest/ManifestParser.java b/src/org/digidoc4j/impl/bdoc/manifest/ManifestParser.java
@@ -41,7 +41,7 @@ public boolean containsManifestFile() {
   }
 
   public Map<String, ManifestEntry> getManifestFileItems() {
-    if(!containsManifestFile()) {
+    if (!containsManifestFile()) {
       return Collections.emptyMap();
     }
     entries = new HashMap<>();
@@ -76,7 +76,7 @@ private void addFileEntry(Node firstChild) {
   }
 
   private void validateNotDuplicateFile(String filePath) {
-    if(entries.containsKey(filePath)) {
+    if (entries.containsKey(filePath)) {
       DuplicateDataFileException digiDoc4JException = new DuplicateDataFileException("duplicate entry in manifest file: " + filePath);
       logger.error(digiDoc4JException.getMessage());
       throw digiDoc4JException;

diff --git a/src/org/digidoc4j/impl/bdoc/manifest/ManifestValidator.java b/src/org/digidoc4j/impl/bdoc/manifest/ManifestValidator.java
@@ -10,10 +10,8 @@
 
 package org.digidoc4j.impl.bdoc.manifest;
 
-import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -31,15 +29,14 @@
 
 import eu.europa.esig.dss.DSSDocument;
 import eu.europa.esig.dss.DomUtils;
-import eu.europa.esig.dss.xades.DSSXMLUtils;
 
 /**
  * For validating meta data within the manifest file and signature files.
  */
 public class ManifestValidator {
-  private static final Logger logger = LoggerFactory.getLogger(ManifestValidator.class);
   public static final String MANIFEST_PATH = "META-INF/manifest.xml";
   public static final String MIMETYPE_PATH = "mimetype";
+  private static final Logger logger = LoggerFactory.getLogger(ManifestValidator.class);
   private List<DSSDocument> detachedContents;
   private ManifestParser manifestParser;
   private Collection<Signature> signatures;
@@ -50,6 +47,69 @@ public ManifestValidator(ManifestParser manifestParser, List<DSSDocument> detach
     this.signatures = signatures;
   }
 
+  @SuppressWarnings("unchecked")
+  static List<String> validateEntries(Map<String, ManifestEntry> manifestEntries, Set<ManifestEntry> signatureEntries,
+                                      String signatureId) {
+    logger.debug("");
+    ArrayList<String> errorMessages = new ArrayList<>();
+
+    if (signatureEntries.size() == 0)
+      return errorMessages;
+
+    Set<ManifestEntry> one = new HashSet(manifestEntries.values());
+    Set<ManifestEntry> onePrim = new HashSet(manifestEntries.values());
+    Set<ManifestEntry> two = new HashSet(signatureEntries);
+    Set<ManifestEntry> twoPrim = new HashSet();
+    for (ManifestEntry manifestEntry : signatureEntries) {
+      String mimeType = manifestEntry.getMimeType();
+      String alterName = manifestEntry.getFileName().replaceAll("\\+", " ");
+      twoPrim.add(new ManifestEntry(alterName, mimeType));
+    }
+
+    one.removeAll(signatureEntries);
+    onePrim.removeAll(twoPrim);
+    two.removeAll(manifestEntries.values());
+    twoPrim.removeAll(manifestEntries.values());
+
+    if (one.size() > 0 && onePrim.size() > 0) {
+      for (ManifestEntry manifestEntry : one) {
+
+        String fileName = manifestEntry.getFileName();
+        ManifestEntry signatureEntry = signatureEntryForFile(fileName, signatureEntries);
+        if (signatureEntry != null) {
+          errorMessages.add("Manifest file has an entry for file " + fileName + " with mimetype " +
+              manifestEntry.getMimeType() + " but the signature file for signature " + signatureId +
+              " indicates the mimetype is " + signatureEntry.getMimeType());
+          two.remove(signatureEntry);
+        } else {
+          errorMessages.add("Manifest file has an entry for file " + fileName + " with mimetype "
+              + manifestEntry.getMimeType() + " but the signature file for signature " + signatureId +
+              " does not have an entry for this file");
+        }
+      }
+    }
+
+    if (two.size() > 0 && twoPrim.size() > 0) {
+      for (ManifestEntry manifestEntry : two) {
+        errorMessages.add("The signature file for signature " + signatureId + " has an entry for file "
+            + manifestEntry.getFileName() + " with mimetype " + manifestEntry.getMimeType()
+            + " but the manifest file does not have an entry for this file");
+      }
+    }
+
+    return errorMessages;
+  }
+
+  private static ManifestEntry signatureEntryForFile(String fileName, Set<ManifestEntry> signatureEntries) {
+    logger.debug("File name: " + fileName);
+    for (ManifestEntry signatureEntry : signatureEntries) {
+      if (fileName.equals(signatureEntry.getFileName())) {
+        return signatureEntry;
+      }
+    }
+    return null;
+  }
+
   /**
    * Validate the container.
    *
@@ -74,7 +134,8 @@ public List<String> validateDocument() {
 
     errorMessages.addAll(validateFilesInContainer(signatureEntries));
 
-    logger.info("Validation of meta data within the manifest file and signature files error count: " + errorMessages.size());
+    logger.info("Validation of meta data within the manifest file and signature files error count: "
+        + errorMessages.size());
     return errorMessages;
   }
 
@@ -87,8 +148,9 @@ private List<String> validateFilesInContainer(Set<ManifestEntry> signatureEntrie
 
     Set<String> signatureEntriesFileNames = getFileNamesFromManifestEntrySet(signatureEntries);
     List<String> filesInContainer = getFilesInContainer();
-    for(String fileInContainer: filesInContainer) {
-      if(!signatureEntriesFileNames.contains(fileInContainer)) {
+    for (String fileInContainer : filesInContainer) {
+      String alterName = fileInContainer.replaceAll("\\ ", "+");
+      if (!signatureEntriesFileNames.contains(fileInContainer) && !signatureEntriesFileNames.contains(alterName)) {
         logger.error("Container contains unsigned data file '" + fileInContainer + "'");
         errorMessages.add("Container contains a file named " + fileInContainer + " which is not found in the signature file");
       }
@@ -106,55 +168,6 @@ private Set<String> getFileNamesFromManifestEntrySet(Set<ManifestEntry> signatur
     return signatureEntriesFileNames;
   }
 
-  @SuppressWarnings("unchecked")
-  static List<String> validateEntries(Map<String, ManifestEntry> manifestEntries, Set<ManifestEntry> signatureEntries,
-                                      String signatureId) {
-    logger.debug("");
-    ArrayList<String> errorMessages = new ArrayList<>();
-
-    if (signatureEntries.size() == 0)
-      return errorMessages;
-
-    Set<ManifestEntry> one = new HashSet(manifestEntries.values());
-    Set<ManifestEntry> two = new HashSet(signatureEntries);
-    one.removeAll(signatureEntries);
-    two.removeAll(manifestEntries.values());
-
-    for (ManifestEntry manifestEntry : one) {
-
-      String fileName = manifestEntry.getFileName();
-      ManifestEntry signatureEntry = signatureEntryForFile(fileName, signatureEntries);
-      if (signatureEntry != null) {
-        errorMessages.add("Manifest file has an entry for file " + fileName + " with mimetype " +
-            manifestEntry.getMimeType() + " but the signature file for signature " + signatureId +
-            " indicates the mimetype is " + signatureEntry.getMimeType());
-        two.remove(signatureEntry);
-      } else {
-        errorMessages.add("Manifest file has an entry for file " + fileName + " with mimetype "
-            + manifestEntry.getMimeType() + " but the signature file for signature " + signatureId +
-            " does not have an entry for this file");
-      }
-    }
-
-    for (ManifestEntry manifestEntry : two) {
-      errorMessages.add("The signature file for signature " + signatureId + " has an entry for file "
-          + manifestEntry.getFileName() + " with mimetype " + manifestEntry.getMimeType()
-          + " but the manifest file does not have an entry for this file");
-    }
-
-    return errorMessages;
-  }
-
-  private static ManifestEntry signatureEntryForFile(String fileName, Set<ManifestEntry> signatureEntries) {
-    logger.debug("File name: " + fileName);
-    for (ManifestEntry signatureEntry : signatureEntries) {
-      if (fileName.equals(signatureEntry.getFileName())) {
-        return signatureEntry;
-      }
-    }
-    return null;
-  }
-
   private Set<ManifestEntry> getSignatureEntries(BDocSignature signature) {
     Set<ManifestEntry> signatureEntries = new HashSet<>();
     List<Reference> references = signature.getOrigin().getReferences();
@@ -173,25 +186,25 @@ private Set<ManifestEntry> getSignatureEntries(BDocSignature signature) {
         }
 
         // TODO: mimeTypeString == null ? node == null?
-         String uri = getFileURI(reference);
-         signatureEntries.add(new ManifestEntry(uri, mimeTypeString));
+        String uri = getFileURI(reference);
+        signatureEntries.add(new ManifestEntry(uri, mimeTypeString));
       }
     }
 
     return signatureEntries;
   }
 
-    private String getFileURI(Reference reference) {
-        String uri = reference.getURI();
-        try {
-          uri = new URI(uri).getPath();
-        } catch (URISyntaxException e) {
-          logger.warn("Does not parse as an URI, therefore assuming it's not encoded: '" + uri + "'");
-        }
-
-        return uri;
+  private String getFileURI(Reference reference) {
+    String uri = reference.getURI();
+    try {
+      uri = new URI(uri).getPath();
+    } catch (URISyntaxException e) {
+      logger.warn("Does not parse as an URI, therefore assuming it's not encoded: '" + uri + "'");
     }
 
+    return uri;
+  }
+
   private List<String> getFilesInContainer() {
     List<String> fileEntries = new ArrayList<>();
 
@@ -209,7 +222,7 @@ private List<String> getFilesInContainer() {
 
   private List<String> getSignatureFileNames() {
     List<String> signatureFileNames = new ArrayList<>();
-    for (Signature signature :signatures) {
+    for (Signature signature : signatures) {
       String signatureFileName = "META-INF/signature" + signature.getId().toLowerCase() + ".xml";
 
       if (signatureFileNames.contains(signatureFileName)) {

diff --git a/test/org/digidoc4j/impl/bdoc/UriEncodingTest.java b/test/org/digidoc4j/impl/bdoc/UriEncodingTest.java
@@ -35,7 +35,7 @@ public class UriEncodingTest extends DigiDoc4JTestHelper {
     // DetachedSignatureBuilder.createReference(...) uses UTF-8 from dss5.0
     public void signatureReferencesUseUriEncodingButManifestUsesPlainUtf8() throws InterruptedException {
         String fileName = "dds_JÜRIÖÖ € žŠ päev.txt";
-        String expectedEncoding = "dds_J%C3%9CRI%C3%96%C3%96+%E2%82%AC+%C5%BE%C5%A0+p%C3%A4ev.txt";
+        String expectedEncoding = "dds_J%C3%9CRI%C3%96%C3%96%20%E2%82%AC%20%C5%BE%C5%A0%20p%C3%A4ev.txt";
         signAndAssertEncoding(fileName, expectedEncoding);
         // TODO: Also write an assertion to verify that the manifest file does NOT use URI encoding
     }
@@ -44,7 +44,7 @@ public void signatureReferencesUseUriEncodingButManifestUsesPlainUtf8() throws I
     // DetachedSignatureBuilder.createReference(...) uses UTF-8 from dss5.0
     public void encodeDataFileWithSpecialCharacters() throws Exception {
         String fileName = "et10i_0123456789!#$%&'()+,-. ;=@[]_`}~ et_EE";
-        String expectedEncoding = "et10i_0123456789%21%23%24%25%26%27%28%29%2B%2C-.+%3B%3D%40%5B%5D_%60%7D%7E+et_EE";
+        String expectedEncoding = "et10i_0123456789%21%23%24%25%26%27%28%29%2B%2C-.%20%3B%3D%40%5B%5D_%60%7D~%20et_EE";
         signAndAssertEncoding(fileName, expectedEncoding);
     }
 
@@ -59,7 +59,7 @@ public void validatePartialEncoding_shouldBeValid() throws Exception {
     }
 
     @Test
-    @Ignore("https://www.pivotaltracker.com/story/show/125469911")
+    //@Ignore("https://www.pivotaltracker.com/story/show/125469911")
     public void validateContainer_withWhitespaceEncodedAsPlus_shouldBeValid() throws Exception {
         Container container = ContainerBuilder.
             aContainer().

diff --git a/test/org/digidoc4j/impl/bdoc/ValidationTests.java b/test/org/digidoc4j/impl/bdoc/ValidationTests.java
@@ -296,11 +296,10 @@ public void invalidNoncePolicyOid() {
   @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
   @Test
   public void noNoncePolicy() {
-    Container container = ContainerOpener.open("testFiles/invalid-containers/23608_bdoc21-no-nonce-policy.bdoc", PROD_CONFIGURATION);
+    Container container = ContainerOpener.open("testFiles/valid-containers/23608_bdoc21-no-nonce-policy.bdoc", PROD_CONFIGURATION);
     ValidationResult result = container.validate();
     List<DigiDoc4JException> errors = result.getErrors();
-    assertEquals(1, errors.size());
-    assertEquals("The signature policy is not available!", errors.get(0).toString());
+    assertEquals(0, errors.size());
   }
 
   @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@@ -392,11 +391,10 @@ public void invalidNonce() {
   @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
   @Test
   public void noPolicyURI() {
-    Container container = ContainerOpener.open("testFiles/invalid-containers/SP-06_bdoc21-no-uri.bdoc", PROD_CONFIGURATION);
+    Container container = ContainerOpener.open("testFiles/valid-containers/SP-06_bdoc21-no-uri.bdoc", PROD_CONFIGURATION);
     ValidationResult result = container.validate();
     List<DigiDoc4JException> errors = result.getErrors();
-    assertEquals(1, errors.size());
-    assertEquals("The signature policy is not available!", errors.get(0).toString());
+    assertEquals(0, errors.size());
   }
 
   @SuppressWarnings("ThrowableResultOfMethodCallIgnored")

diff --git a/...tainers/23608_bdoc21-no-nonce-policy.bdoc → ...tainers/23608_bdoc21-no-nonce-policy.bdoc b/...tainers/23608_bdoc21-no-nonce-policy.bdoc → ...tainers/23608_bdoc21-no-nonce-policy.bdoc
diff --git a/...valid-containers/SP-06_bdoc21-no-uri.bdoc → ...valid-containers/SP-06_bdoc21-no-uri.bdoc b/...valid-containers/SP-06_bdoc21-no-uri.bdoc → ...valid-containers/SP-06_bdoc21-no-uri.bdoc
diff --git a/testFiles/valid-containers/testing.asice b/testFiles/valid-containers/testing.asice