Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

various fixes #125

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,10 @@ private static void checkManifestPayload(final Path manifestFile, final Charset
String path = parsePath(line);

path = checkForManifestCreatedWithMD5SumTools(path, warnings, warningsToIgnore);
paths.add(path.toLowerCase());

checkForDifferentCase(path, paths, manifestFile, warnings, warningsToIgnore);
paths.add(path.toLowerCase());

if(encoding.name().startsWith("UTF")){
checkNormalization(path, manifestFile.getParent(), warnings, warningsToIgnore);
}
Expand Down Expand Up @@ -256,7 +257,7 @@ static void checkAlgorthm(final String algorithm, final Set<BagitWarning> warnin
warnings.add(BagitWarning.WEAK_CHECKSUM_ALGORITHM);
}

else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA-512".equals(upperCaseAlg)){
else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA512".equals(upperCaseAlg)){
logger.warn(messages.getString("non_standard_algorithm_warning"), algorithm);
warnings.add(BagitWarning.NON_STANDARD_ALGORITHM);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package gov.loc.repository.bagit.exceptions;

import java.nio.file.Path;

import org.slf4j.helpers.MessageFormatter;

/**
* The payload directory is a required file. This class represents the error if it is not found.
*/
public class MissingPayloadDirectoryException extends Exception {
private static final long serialVersionUID = 1L;

public MissingPayloadDirectoryException(final String message){
super(message);
public MissingPayloadDirectoryException(final String message, final Path path){
super(MessageFormatter.format(message, path).getMessage());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
public enum StandardSupportedAlgorithms implements SupportedAlgorithm{
MD5("MD5"),
SHA1("SHA-1"),

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this "SHA-1" now different than the others? Now the others don't have this dash anymore, but this one still has it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. I will update to include sha1 as well as tests on very basic valid test bags

SHA224("SHA-224"),
SHA256("SHA-256"),
SHA512("SHA-512");
SHA224("SHA224"),
SHA256("SHA256"),
SHA512("SHA512");

private final String messageDigestName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public final class BagVerifier implements AutoCloseable{
private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class);
private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle");

private final PayloadVerifier manifestVerifier;
private final ManifestVerifier manifestVerifier;
private final ExecutorService executor;

/**
Expand Down Expand Up @@ -74,7 +74,7 @@ public BagVerifier(final ExecutorService executor){
* @param executor the thread pool to use when doing work
*/
public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){
manifestVerifier = new PayloadVerifier(nameMapping, executor);
manifestVerifier = new ManifestVerifier(nameMapping, executor);
this.executor = executor;
}

Expand Down Expand Up @@ -209,14 +209,14 @@ public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws

MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion());

manifestVerifier.verifyPayload(bag, ignoreHiddenFiles);
manifestVerifier.verifyManifests(bag, ignoreHiddenFiles);
}

public ExecutorService getExecutor() {
return executor;
}

public PayloadVerifier getManifestVerifier() {
public ManifestVerifier getManifestVerifier() {
return manifestVerifier;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public static void checkPayloadDirectoryExists(final Bag bag) throws MissingPayl
final Path dataDir = PathUtils.getDataDir(bag);

if(!Files.exists(dataDir)){
throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error"));
throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error"), dataDir);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
/**
* Responsible for all things related to the manifest during verification.
*/
public class PayloadVerifier implements AutoCloseable{
private static final Logger logger = LoggerFactory.getLogger(PayloadVerifier.class);
public class ManifestVerifier implements AutoCloseable{
private static final Logger logger = LoggerFactory.getLogger(ManifestVerifier.class);
private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle");

private transient final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping;
Expand All @@ -42,7 +42,7 @@ public class PayloadVerifier implements AutoCloseable{
* Create a PayloadVerifier using a cached thread pool and the
* {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping
*/
public PayloadVerifier(){
public ManifestVerifier(){
this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newCachedThreadPool());
}

Expand All @@ -51,7 +51,7 @@ public PayloadVerifier(){
*
* @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm
*/
public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) {
public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) {
this(nameMapping, Executors.newCachedThreadPool());
}

Expand All @@ -61,7 +61,7 @@ public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameM
*
* @param executor the thread pool to use when doing work
*/
public PayloadVerifier(final ExecutorService executor) {
public ManifestVerifier(final ExecutorService executor) {
this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), executor);
}

Expand All @@ -71,7 +71,7 @@ public PayloadVerifier(final ExecutorService executor) {
* @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm
* @param executor the thread pool to use when doing work
*/
public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) {
public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) {
this.nameMapping = nameMapping;
this.executor = executor;
}
Expand All @@ -83,19 +83,20 @@ public void close() throws SecurityException{
}

/**
* Verify that all the files in the payload directory are listed in the manifest and
* all files listed in the manifests exist.
* Verify that all the files in the payload directory are listed in the payload manifest and
* all files listed in all manifests exist.
*
* @param bag the bag to check to check
* @param ignoreHiddenFiles to ignore hidden files unless they are specifically listed in a manifest
*
* @throws IOException if there is a problem reading a file
* @throws MaliciousPathException the path in the manifest was specifically crafted to cause harm
* @throws UnsupportedAlgorithmException if the algorithm used for the manifest is unsupported
* @throws InvalidBagitFileFormatException if any of the manifests don't conform to the bagit specification
* @throws FileNotInPayloadDirectoryException if a file is listed in a manifest but doesn't exist in the payload directory
* @throws InterruptedException if a thread is interrupted while doing work
*/
public void verifyPayload(final Bag bag, final boolean ignoreHiddenFiles)
public void verifyManifests(final Bag bag, final boolean ignoreHiddenFiles)
throws IOException, MaliciousPathException, UnsupportedAlgorithmException,
InvalidBagitFileFormatException, FileNotInPayloadDirectoryException, InterruptedException {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTar
assertUtilityClassWellDefined(BagLinter.class);
}

@Test
public void testConformantBag() throws Exception{
Path goodBag = Paths.get("src", "test", "resources", "bags", "v1_0", "bag");
Set<BagitWarning> warnings = BagLinter.lintBag(goodBag);
Assertions.assertTrue(warnings.size() == 0);
}

@Test
public void testLintBag() throws Exception{
Set<BagitWarning> expectedWarnings = new HashSet<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,23 @@ public class PayloadVerifierTest {
private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI());
private BagReader reader = new BagReader();

private PayloadVerifier sut;
private ManifestVerifier sut;

@BeforeEach
public void setup(){
sut = new PayloadVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping());
sut = new ManifestVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping());
}

@Test
public void testOtherConstructors() throws Exception {
rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI());
Bag bag = reader.read(rootDir);

sut = new PayloadVerifier();
sut.verifyPayload(bag, true);
sut = new ManifestVerifier();
sut.verifyManifests(bag, true);

sut = new PayloadVerifier(Executors.newCachedThreadPool());
sut.verifyPayload(bag, true);
sut = new ManifestVerifier(Executors.newCachedThreadPool());
sut.verifyManifests(bag, true);
}

@Test
Expand All @@ -45,7 +45,7 @@ public void testErrorWhenManifestListFileThatDoesntExist() throws Exception{
Bag bag = reader.read(rootDir);

Assertions.assertThrows(FileNotInPayloadDirectoryException.class,
() -> { sut.verifyPayload(bag, true); });
() -> { sut.verifyManifests(bag, true); });
}

@Test
Expand All @@ -54,22 +54,22 @@ public void testErrorWhenFileIsntInManifest() throws Exception{
Bag bag = reader.read(rootDir);

Assertions.assertThrows(FileNotInManifestException.class,
() -> { sut.verifyPayload(bag, true); });
() -> { sut.verifyManifests(bag, true); });
}

@Test
public void testBagWithTagFilesInPayloadIsValid() throws Exception{
rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI());
Bag bag = reader.read(rootDir);

sut.verifyPayload(bag, true);
sut.verifyManifests(bag, true);
}

@Test
public void testNotALlFilesListedInAllManifestsThrowsException() throws Exception{
Path bagDir = Paths.get(new File("src/test/resources/notAllFilesListedInAllManifestsBag").toURI());
Bag bag = reader.read(bagDir);
Assertions.assertThrows(FileNotInManifestException.class,
() -> { sut.verifyPayload(bag, true); });
() -> { sut.verifyManifests(bag, true); });
}
}
3 changes: 3 additions & 0 deletions src/test/resources/bags/v1_0/bag/bag-info.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Bag-Software-Agent: bagit.py v1.7.0 <https://github.com/LibraryOfCongress/bagit-python>
Bagging-Date: 2018-07-22
Payload-Oxum: 6.1
2 changes: 2 additions & 0 deletions src/test/resources/bags/v1_0/bag/bagit.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
BagIt-Version: 1.0
Tag-File-Character-Encoding: UTF-8
1 change: 1 addition & 0 deletions src/test/resources/bags/v1_0/bag/data/foo.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hello
1 change: 1 addition & 0 deletions src/test/resources/bags/v1_0/bag/manifest-sha512.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e7c22b994c59d9cf2b48e549b1e24666636045930d3da7c1acb299d1c3b7f931f94aae41edda2c2b207a36e10f8bcb8d45223e54878f5b316e7ce3b6bc019629 data/foo.txt
3 changes: 3 additions & 0 deletions src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
f5bdbc7f273dd8b95d30b77b3d6f727d999ecbe9be06a7656388e7a3a46d963881563d779a2a99265b0c2785de2a7b72ac05fa7bc5b66b471d3f4e80fe9bb370 bag-info.txt
1d73ae108d4109b61f56698a5e19ee1f8947bdf8940bbce6adbe5e0940c2363caace6a547b4f1b3ec6a4fd2b7fa845e9cb9d28823bc72c59971718bb26f2fbd8 bagit.txt
35d40e38f5e2eb7261a1cc0c0ccf9c6c50d2e07e39a9460ec4f99046048f939d82647c68d8f744c653c7a37c05cc9d71153c8028786dfeba4322a3be5fe81af0 manifest-sha512.txt