Skip to content

Commit

Permalink
Fixed unit tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@janvanmansum
janvanmansum committed Oct 25, 2024
1 parent d4a50d6 commit 550fb34
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 66 deletions.
42 changes: 31 additions & 11 deletions src/main/java/nl/knaw/dans/dvcli/action/BatchProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;

import java.util.List;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;

/**
* Processes a batch of labeled items by applying an action to each item. The labels are used for reporting. Typically, the label is the ID of the item. After each action, the processor waits for a
Expand All @@ -31,12 +33,17 @@
*/
@Builder
@Slf4j
public class BatchProcessor<I, R> {
public class BatchProcessor<I, R> {
/**
* The labeled items to process.
*/
@NonNull
private final List<Pair<String, I>> labeledItems;
private final Stream<Pair<String, I>> labeledItems;

/**
* The number of items to process. If the labeled items are a collection, this number is the size of the collection. Otherwise, it is null.
*/
private final Long numberOfItems;

/**
* The action to apply to each item.
Expand All @@ -56,15 +63,28 @@ public class BatchProcessor<I, R> {
@Builder.Default
private final long delay = 1000;

public static class BatchProcessorBuilder<I, R> {
public BatchProcessorBuilder<I, R> labeledItems(Collection<Pair<String, I>> items) {
this.labeledItems = items.stream();
this.numberOfItems = (long) items.size();
return this;
}
}

public void process() {
log.info("Starting batch processing");
int i = 0;
for (var labeledItem : labeledItems) {
delayIfNeeded(i);
log.info("Processing item {} of {}", ++i, labeledItems.size());
callAction(labeledItem.getFirst(), labeledItem.getSecond());
log.info("Starting batch processing of " + (numberOfItems == null ? "?" : numberOfItems + " items"));
AtomicInteger i = new AtomicInteger(0);
try {
labeledItems.forEach(labeledItem -> {
int index = i.incrementAndGet();
delayIfNeeded(index);
log.info("Processing item {} of {}: {}", index, numberOfItems == null ? "?" : numberOfItems, labeledItem.getFirst());
callAction(labeledItem.getFirst(), labeledItem.getSecond());
});
} finally {
labeledItems.close();
}
log.info("Finished batch processing of {} items", labeledItems.size());
log.info("Finished batch processing of " + (numberOfItems == null ? "?" : numberOfItems + " items"));
}

private void callAction(String label, I item) {
Expand All @@ -78,7 +98,7 @@ private void callAction(String label, I item) {
}

private void delayIfNeeded(int i) {
if (delay > 0 && i > 0) {
if (delay > 0 && i > 1) {
log.debug("Sleeping for {} ms", delay);
try {
Thread.sleep(delay);
Expand Down
26 changes: 11 additions & 15 deletions src/main/java/nl/knaw/dans/dvcli/command/dataset/DatasetValidateFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
*/
package nl.knaw.dans.dvcli.command.dataset;

import nl.knaw.dans.dvcli.action.BatchProcessor;
import nl.knaw.dans.dvcli.action.ConsoleReport;
import nl.knaw.dans.dvcli.action.Pair;
import nl.knaw.dans.dvcli.action.SingleIdOrIdsFile;
import nl.knaw.dans.dvcli.action.ThrowingFunction;
import nl.knaw.dans.dvcli.command.AbstractCmd;
import nl.knaw.dans.lib.dataverse.AdminApi;
import nl.knaw.dans.lib.dataverse.DataverseException;
Expand All @@ -30,35 +30,31 @@

@Command(name = "validate-files",
mixinStandardHelpOptions = true,
description = "Make sure that all files are correctly stored in object storage.")
description = "Validate the fixity checksums of the files in a dataset.")
public class DatasetValidateFiles extends AbstractCmd {
@ParentCommand
private DatasetCmd datasetCmd;

protected List<Pair<String, IdParam>> getIds() throws IOException {
protected List<Pair<String, String>> getIds() throws IOException {
List<String> pids = new SingleIdOrIdsFile(datasetCmd.getTargets(), SingleIdOrIdsFile.DEFAULT_TARGET_PLACEHOLDER).getPids().toList();
return pids.stream().map(p -> new Pair<>(p, new IdParam(datasetCmd.getDataverseClient().admin(), p))).toList();
// The label is the same as the id. Since the BatchProcessor expects labeled items, we create a list of pairs with the same id as label.
return pids.stream().map(p -> new Pair<>(p, p)).toList();
}

protected record IdParam(AdminApi admin, String id) {
}

private static class ValidateFilesAction implements ThrowingFunction<IdParam, String, Exception> {
@Override
public String apply(IdParam idParam) throws IOException, DataverseException {
var r = idParam.admin().validateDatasetFiles(idParam.id);
return r.getBodyAsString();
}
}

@Override
public void doCall() throws IOException, DataverseException {
datasetCmd.<IdParam> paramsBatchProcessorBuilder()
// Not using the helper method on datasetCmd because we need to call the admin endpoint and not the dataset endpoint.
BatchProcessor.<String, String> builder()
.labeledItems(getIds())
.action(new ValidateFilesAction())
.action(pid -> {
var r = datasetCmd.getDataverseClient().admin().validateDatasetFiles(pid);
return r.getBodyAsString();
})
.report(new ConsoleReport<>())
.build()
.process();
}

}
44 changes: 22 additions & 22 deletions src/test/java/nl/knaw/dans/dvcli/action/BatchProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,23 +59,23 @@ BatchProcessor.<DatasetApi, String> builder()
a: OK. b: FAILED: Exception type = RuntimeException, message = test
c: OK.""" + " "); // java text block trims trailing spaces
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: a
ok
DEBUG Sleeping for 1 ms
INFO Processing item 2 of 3
INFO Processing item 2 of 3: b
DEBUG Sleeping for 1 ms
INFO Processing item 3 of 3
INFO Processing item 3 of 3: c
ok
INFO Finished batch processing of 3 items
""");
assertThat(messagesOf(logged))
.containsExactly("INFO Starting batch processing",
"INFO Processing item 1 of 3",
.containsExactly("INFO Starting batch processing of 3 items",
"INFO Processing item 1 of 3: a",
"DEBUG Sleeping for 1 ms",
"INFO Processing item 2 of 3",
"INFO Processing item 2 of 3: b",
"DEBUG Sleeping for 1 ms",
"INFO Processing item 3 of 3",
"INFO Processing item 3 of 3: c",
"INFO Finished batch processing of 3 items");
}

Expand All @@ -95,14 +95,14 @@ BatchProcessor.<DatasetApi, String> builder()
assertThat(stderr.toString())
.isEqualTo("a: OK. b: OK. c: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: a
ok
DEBUG Sleeping for 1000 ms
INFO Processing item 2 of 3
INFO Processing item 2 of 3: b
ok
DEBUG Sleeping for 1000 ms
INFO Processing item 3 of 3
INFO Processing item 3 of 3: c
ok
INFO Finished batch processing of 3 items
""");
Expand All @@ -124,12 +124,12 @@ BatchProcessor.<DatasetApi, String> builder()

assertThat(stderr.toString()).isEqualTo("A: OK. B: OK. C: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: A
ok
INFO Processing item 2 of 3
INFO Processing item 2 of 3: B
ok
INFO Processing item 3 of 3
INFO Processing item 3 of 3: C
ok
INFO Finished batch processing of 3 items
""");
Expand All @@ -150,12 +150,12 @@ BatchProcessor.<DatasetApi, String> builder()

assertThat(stderr.toString()).isEqualTo("X: OK. Y: OK. Z: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: X
ok
INFO Processing item 2 of 3
INFO Processing item 2 of 3: Y
ok
INFO Processing item 3 of 3
INFO Processing item 3 of 3: Z
ok
INFO Finished batch processing of 3 items
""");
Expand All @@ -172,11 +172,11 @@ BatchProcessor.<DatasetApi, String> builder()

assertThat(stderr.toString()).isEqualTo("");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Starting batch processing of 0 items
INFO Finished batch processing of 0 items
""");
assertThat(messagesOf(logged)).containsExactly(
"INFO Starting batch processing",
"INFO Starting batch processing of 0 items",
"INFO Finished batch processing of 0 items");
}

Expand Down
11 changes: 6 additions & 5 deletions src/test/java/nl/knaw/dans/dvcli/command/AbstractSubcommandContainerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package nl.knaw.dans.dvcli.command;

import lombok.extern.slf4j.Slf4j;
import nl.knaw.dans.dvcli.AbstractCapturingTest;
import nl.knaw.dans.dvcli.action.Pair;
import nl.knaw.dans.dvcli.command.collection.CollectionCmd;
Expand All @@ -32,8 +33,8 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;

@Slf4j
public class AbstractSubcommandContainerTest extends AbstractCapturingTest {
private static final Logger log = LoggerFactory.getLogger(AbstractSubcommandContainerTest.class);

private static class TestCmd extends AbstractSubcommandContainer<Object> {

Expand Down Expand Up @@ -265,14 +266,14 @@ public void batchProcessor_does_not_throw() {

assertThat(stderr.toString()).isEqualTo("1: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 1
INFO Starting batch processing of 1 items
INFO Processing item 1 of 1: 1
value of 1
INFO Finished batch processing of 1 items
""");
assertThat(logged.list.stream().map(Object::toString).toList()).containsExactly(
"[INFO] Starting batch processing",
"[INFO] Processing item 1 of 1",
"[INFO] Starting batch processing of 1 items",
"[INFO] Processing item 1 of 1: 1",
"[INFO] Finished batch processing of 1 items"
);
}
Expand Down
18 changes: 9 additions & 9 deletions src/test/java/nl/knaw/dans/dvcli/command/CollectionCreateDatasetTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ public void doCall_continues_on_unknownHost() throws Exception {
cmd.doCall();

assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: A
DEBUG buildUri: https://does.not.exist.dans.knaw.nl/api/dataverses/A/datasets
INFO Processing item 2 of 3
INFO Processing item 2 of 3: B
DEBUG buildUri: https://does.not.exist.dans.knaw.nl/api/dataverses/B/datasets
INFO Processing item 3 of 3
INFO Processing item 3 of 3: C
DEBUG buildUri: https://does.not.exist.dans.knaw.nl/api/dataverses/C/datasets
INFO Finished batch processing of 3 items
""");
Expand Down Expand Up @@ -103,8 +103,8 @@ public void doCall_is_happy() throws Exception {

assertThat(stderr.toString()).isEqualTo("A: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 1
INFO Starting batch processing of 1 items
INFO Processing item 1 of 1: A
mock response
INFO Finished batch processing of 1 items
""");
Expand Down Expand Up @@ -138,9 +138,9 @@ public void doCall_with_dir_as_json_file_fails() throws Exception {
B: FAILED: Exception type = IOException, message = Is a directory
""");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 2
INFO Processing item 2 of 2
INFO Starting batch processing of 2 items
INFO Processing item 1 of 2: A
INFO Processing item 2 of 2: B
INFO Finished batch processing of 2 items
""");
}
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/nl/knaw/dans/dvcli/command/TruncateNotificationsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,16 @@ public void doCall_with_several_users_to_truncate_notifications_works() throws E

assertThat(stderr.toString()).isEqualTo("1: OK. 2: OK. 3: OK. ");
assertThat(stdout.toString()).isEqualTo("""
INFO Starting batch processing
INFO Processing item 1 of 3
INFO Starting batch processing of 3 items
INFO Processing item 1 of 3: 1
INFO Deleting notifications for user with id 1
Deleted 3 record(s) for user with id 1
DEBUG Sleeping for 10 ms
INFO Processing item 2 of 3
INFO Processing item 2 of 3: 2
INFO Deleting notifications for user with id 2
Deleted 2 record(s) for user with id 2
DEBUG Sleeping for 10 ms
INFO Processing item 3 of 3
INFO Processing item 3 of 3: 3
INFO Deleting notifications for user with id 3
Deleted 1 record(s) for user with id 3
INFO Finished batch processing of 3 items
Expand Down

0 comments on commit 550fb34

Please sign in to comment.