From 155959aa4ce71d228186ddff276c37a719e2fbe1 Mon Sep 17 00:00:00 2001 From: Jan van Mansum Date: Thu, 20 Jun 2024 17:14:52 +0200 Subject: [PATCH] BatchProcessor works for delete-draft --- .../java/nl/knaw/dans/dvcli/ItemProvider.java | 8 --- .../nl/knaw/dans/dvcli/action/Action.java | 19 ------ .../knaw/dans/dvcli/action/ActionSource.java | 6 -- .../dans/dvcli/action/BatchProcessor.java | 68 ++++++++++++++++--- .../knaw/dans/dvcli/action/ConsoleReport.java | 35 ++++++++++ .../java/nl/knaw/dans/dvcli/action/Item.java | 4 -- .../java/nl/knaw/dans/dvcli/action/Pair.java | 24 +++++++ .../nl/knaw/dans/dvcli/action/Report.java | 43 +++++++++++- .../action/SingleDatasetOrDatasetsFile.java | 46 +++++++++++++ .../dans/dvcli/action/SingleIdOrIdsFile.java | 36 ++++++++++ .../dans/dvcli/action/ThrowingFunction.java | 21 ++++++ .../knaw/dans/dvcli/command/DatasetCmd.java | 26 +++---- .../knaw/dans/dvcli/command/DeleteDraft.java | 19 ++++-- 13 files changed, 282 insertions(+), 73 deletions(-) delete mode 100644 src/main/java/nl/knaw/dans/dvcli/ItemProvider.java delete mode 100644 src/main/java/nl/knaw/dans/dvcli/action/Action.java delete mode 100644 src/main/java/nl/knaw/dans/dvcli/action/ActionSource.java create mode 100644 src/main/java/nl/knaw/dans/dvcli/action/ConsoleReport.java delete mode 100644 src/main/java/nl/knaw/dans/dvcli/action/Item.java create mode 100644 src/main/java/nl/knaw/dans/dvcli/action/Pair.java create mode 100644 src/main/java/nl/knaw/dans/dvcli/action/SingleDatasetOrDatasetsFile.java create mode 100644 src/main/java/nl/knaw/dans/dvcli/action/SingleIdOrIdsFile.java create mode 100644 src/main/java/nl/knaw/dans/dvcli/action/ThrowingFunction.java diff --git a/src/main/java/nl/knaw/dans/dvcli/ItemProvider.java b/src/main/java/nl/knaw/dans/dvcli/ItemProvider.java deleted file mode 100644 index 7fe41c1..0000000 --- a/src/main/java/nl/knaw/dans/dvcli/ItemProvider.java +++ /dev/null @@ -1,8 +0,0 @@ -package nl.knaw.dans.dvcli; - -public class ItemProvider { - - - - -} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/Action.java b/src/main/java/nl/knaw/dans/dvcli/action/Action.java deleted file mode 100644 index 7a6001f..0000000 --- a/src/main/java/nl/knaw/dans/dvcli/action/Action.java +++ /dev/null @@ -1,19 +0,0 @@ -package nl.knaw.dans.dvcli.action; - -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public class Action implements Runnable { - private Runnable action; - private Report report; - - @Override - public void run() { - try { - action.run(); - } - catch (Exception e) { - report.report(e.getMessage()); - } - } -} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/ActionSource.java b/src/main/java/nl/knaw/dans/dvcli/action/ActionSource.java deleted file mode 100644 index 5ce7044..0000000 --- a/src/main/java/nl/knaw/dans/dvcli/action/ActionSource.java +++ /dev/null @@ -1,6 +0,0 @@ -package nl.knaw.dans.dvcli.action; - -public interface ActionSource extends AutoCloseable, Iterable { - - -} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/BatchProcessor.java b/src/main/java/nl/knaw/dans/dvcli/action/BatchProcessor.java index 47e3d94..5ea17a7 100644 --- a/src/main/java/nl/knaw/dans/dvcli/action/BatchProcessor.java +++ b/src/main/java/nl/knaw/dans/dvcli/action/BatchProcessor.java @@ -1,25 +1,75 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package nl.knaw.dans.dvcli.action; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import java.util.List; -import java.util.function.Function; +/** + * Processes a batch of labeled items by applying an action to each item. The labels are used for reporting. Typically, the label is the ID of the item. After each action, the processor waits for a + * delay, if specified. The processor reports the results of the actions to a report. + * + * @param the type of the items + * @param the type of action results + * @see Report for the interface that the report must implement + */ @Builder @Slf4j -public class BatchProcessor { - private Iterable items; - private Action action; +public class BatchProcessor { + /** + * The labeled items to process. + */ + private final Iterable> labeledItems; - private long delay; + /** + * The action to apply to each item. + */ + private final ThrowingFunction action; + + /** + * The report to which the results of the actions are reported. + */ + private final Report report; + + /** + * The delay in milliseconds between processing items. A delay of 0 or less means no delay. + */ + @Builder.Default + private final long delay = 1000; public void process() { + log.info("Starting batch processing"); int i = 0; - for (var item : items) { + for (var labeledItem : labeledItems) { delayIfNeeded(i); logStartAction(++i); - action.apply(item); + callAction(labeledItem.getFirst(), labeledItem.getSecond()); + } + log.info("Finished batch processing"); + } + + private void callAction(String label, I item) { + try { + R r = action.apply(item); + report.reportSuccess(label, item, r); + } + catch (Exception e) { + report.reportFailure(label, item, e); } } @@ -37,8 +87,8 @@ private void delayIfNeeded(int i) { } private void logStartAction(int i) { - if (items instanceof List) { - log.info("Processing item {} of {}", i, ((List) items).size()); + if (labeledItems instanceof List) { + log.info("Processing item {} of {}", i, ((List) labeledItems).size()); } else { log.info("Processing item {}", i); diff --git a/src/main/java/nl/knaw/dans/dvcli/action/ConsoleReport.java b/src/main/java/nl/knaw/dans/dvcli/action/ConsoleReport.java new file mode 100644 index 0000000..1525f3a --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvcli/action/ConsoleReport.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvcli.action; + +/** + * Implements a report to the console. + * + * @param the type of the item that was processed + * @param the type of the result of the action + */ +public class ConsoleReport implements Report { + + @Override + public void reportSuccess(String label, I i, R r) { + System.err.println(label + ": OK"); + } + + @Override + public void reportFailure(String label, I i, Exception e) { + System.err.println(label + ": FAILED: " + e.getMessage()); + } +} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/Item.java b/src/main/java/nl/knaw/dans/dvcli/action/Item.java deleted file mode 100644 index ed17425..0000000 --- a/src/main/java/nl/knaw/dans/dvcli/action/Item.java +++ /dev/null @@ -1,4 +0,0 @@ -package nl.knaw.dans.dvcli.action; - -public interface Item { -} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/Pair.java b/src/main/java/nl/knaw/dans/dvcli/action/Pair.java new file mode 100644 index 0000000..5d5678f --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvcli/action/Pair.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvcli.action; + +import lombok.Value; + +@Value +public class Pair { + F first; + S second; +} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/Report.java b/src/main/java/nl/knaw/dans/dvcli/action/Report.java index e271a88..3d1088e 100644 --- a/src/main/java/nl/knaw/dans/dvcli/action/Report.java +++ b/src/main/java/nl/knaw/dans/dvcli/action/Report.java @@ -1,6 +1,43 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package nl.knaw.dans.dvcli.action; -public interface Report { - - void report(String message); +/** + * Report the success or failure of an action. + * + * @param the type of the item that was processed + * @param the type of the result of the action + */ +public interface Report { + + /** + * Report a successful action. + * + * @param label a label for the item that was processed + * @param t the item that was processed + * @param r the result of the action + */ + void reportSuccess(String label, T t, R r); + + /** + * Report a failed action. + * + * @param label a label for the item for which the action was attempted + * @param t the item for which the action was attempted + * @param e the exception that was thrown + */ + void reportFailure(String label, T t, Exception e); } diff --git a/src/main/java/nl/knaw/dans/dvcli/action/SingleDatasetOrDatasetsFile.java b/src/main/java/nl/knaw/dans/dvcli/action/SingleDatasetOrDatasetsFile.java new file mode 100644 index 0000000..36328a8 --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvcli/action/SingleDatasetOrDatasetsFile.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvcli.action; + +import nl.knaw.dans.lib.dataverse.DatasetApi; +import nl.knaw.dans.lib.dataverse.DataverseClient; + +import java.io.IOException; +import java.util.stream.Stream; + +public class SingleDatasetOrDatasetsFile { + private final SingleIdOrIdsFile singleIdOrIdsFile; + private final DataverseClient dataverseClient; + + public SingleDatasetOrDatasetsFile(String singleDatasetOrDatasetsFile, DataverseClient dataverseClient) { + this.singleIdOrIdsFile = new SingleIdOrIdsFile(singleDatasetOrDatasetsFile); + this.dataverseClient = dataverseClient; + } + + public Stream> getDatasets() throws IOException { + return singleIdOrIdsFile.getPids().map( + id -> { + try { + var dbId = Integer.parseInt(id); + return new Pair<>(id, dataverseClient.dataset(dbId)); + } + catch (NumberFormatException e) { + // Assume it is a PID + } + return new Pair<>(id, dataverseClient.dataset(id)); + }); + } +} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/SingleIdOrIdsFile.java b/src/main/java/nl/knaw/dans/dvcli/action/SingleIdOrIdsFile.java new file mode 100644 index 0000000..1b32efc --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvcli/action/SingleIdOrIdsFile.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvcli.action; + +import lombok.AllArgsConstructor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.stream.Stream; + +@AllArgsConstructor +public class SingleIdOrIdsFile { + private final String singleIdOrIdFile; + + public Stream getPids() throws IOException { + var pidFile = Paths.get(singleIdOrIdFile); + if (Files.exists(pidFile)) { + return Files.readAllLines(pidFile).stream().map(String::trim); + } + return Stream.of(singleIdOrIdFile); + } +} diff --git a/src/main/java/nl/knaw/dans/dvcli/action/ThrowingFunction.java b/src/main/java/nl/knaw/dans/dvcli/action/ThrowingFunction.java new file mode 100644 index 0000000..bbc74b0 --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvcli/action/ThrowingFunction.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvcli.action; + +@FunctionalInterface +public interface ThrowingFunction { + R apply(T t) throws E; +} diff --git a/src/main/java/nl/knaw/dans/dvcli/command/DatasetCmd.java b/src/main/java/nl/knaw/dans/dvcli/command/DatasetCmd.java index b8d379c..d4e7250 100644 --- a/src/main/java/nl/knaw/dans/dvcli/command/DatasetCmd.java +++ b/src/main/java/nl/knaw/dans/dvcli/command/DatasetCmd.java @@ -15,41 +15,33 @@ */ package nl.knaw.dans.dvcli.command; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import nl.knaw.dans.dvcli.action.Pair; +import nl.knaw.dans.dvcli.action.SingleDatasetOrDatasetsFile; +import nl.knaw.dans.dvcli.action.SingleIdOrIdsFile; import nl.knaw.dans.lib.dataverse.DatasetApi; import nl.knaw.dans.lib.dataverse.DataverseClient; import picocli.CommandLine.Command; import picocli.CommandLine.Parameters; +import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; @Command(name = "dataset", mixinStandardHelpOptions = true, description = "Manage Dataverse datasets") @Slf4j public class DatasetCmd extends AbstractSubcommandContainer { - @Parameters(index = "0", paramLabel = "id", description = "The id or PID of the dataset") + @Parameters(index = "0", paramLabel = "id", description = "The ID or PID of the dataset, or a file with a list of IDs or PIDs.") private String id; public DatasetCmd(DataverseClient dataverseClient) { super(dataverseClient); } - - - List getDatasets() { - // If id is a number convert it to an integer - try { - var databaseId = Integer.parseInt(id); - log.debug("ID {} is a number, assuming it is a database ID", id); - return List.of(dataverseClient.dataset(databaseId)); - } - catch (NumberFormatException e) { - // Do nothing - } - log.debug("ID {} is not a number, assuming it is a PID", id); - return List.of(dataverseClient.dataset(id)); + List> getDatasets() throws IOException { + return new SingleDatasetOrDatasetsFile(id, dataverseClient).getDatasets().collect(Collectors.toList()); + } - } diff --git a/src/main/java/nl/knaw/dans/dvcli/command/DeleteDraft.java b/src/main/java/nl/knaw/dans/dvcli/command/DeleteDraft.java index 00627c2..1f8fd90 100644 --- a/src/main/java/nl/knaw/dans/dvcli/command/DeleteDraft.java +++ b/src/main/java/nl/knaw/dans/dvcli/command/DeleteDraft.java @@ -16,8 +16,9 @@ package nl.knaw.dans.dvcli.command; import nl.knaw.dans.dvcli.action.BatchProcessor; -import nl.knaw.dans.lib.dataverse.DataverseException; +import nl.knaw.dans.dvcli.action.ConsoleReport; import nl.knaw.dans.lib.dataverse.DatasetApi; +import nl.knaw.dans.lib.dataverse.DataverseException; import picocli.CommandLine.Command; import picocli.CommandLine.ParentCommand; @@ -32,11 +33,15 @@ public class DeleteDraft extends AbstractCmd { @Override public void doCall() throws IOException, DataverseException { - BatchProcessor.builder() - .items(datasetCmd.getDatasets()) - .action(d -> d.deleteDraft()) - .delay(1000L) - .build() - .process(); + BatchProcessor. builder() + .labeledItems(datasetCmd.getDatasets()) + .action(d -> { + var r = d.deleteDraft(); + return r.getEnvelopeAsString(); + }) + .report(new ConsoleReport<>()) + .delay(1000L) + .build() + .process(); } }