Skip to content

Commit

Permalink
WIP - Reporting and bug fixing
Browse files Browse the repository at this point in the history 
Signed-off-by: Alexandre Flores <[email protected]>
  • Loading branch information
@SugaryLump
SugaryLump committed Oct 31, 2024
1 parent 81e1733 commit 584ec13
Show file tree
Hide file tree
Showing 7 changed files with 368 additions and 338 deletions.
63 changes: 33 additions & 30 deletions ...java/org/roda/core/plugins/base/maintenance/backfill/GenerateActionLogBackfillPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.roda.core.RodaCoreFactory;
import org.roda.core.common.iterables.CloseableIterable;
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.common.RodaConstants.PreservationEventType;
Expand All @@ -35,6 +35,7 @@
import org.roda.core.data.v2.ip.StoragePath;
import org.roda.core.data.v2.jobs.Job;
import org.roda.core.data.v2.jobs.PluginParameter;
import org.roda.core.data.v2.jobs.PluginState;
import org.roda.core.data.v2.jobs.PluginType;
import org.roda.core.data.v2.jobs.Report;
import org.roda.core.data.v2.log.LogEntry;
Expand All @@ -44,7 +45,6 @@
import org.roda.core.plugins.Plugin;
import org.roda.core.plugins.PluginException;
import org.roda.core.plugins.PluginHelper;
import org.roda.core.plugins.RODAProcessingLogic;
import org.roda.core.plugins.base.maintenance.backfill.beans.Add;
import org.roda.core.plugins.base.maintenance.backfill.beans.DocType;
import org.roda.core.plugins.orchestrate.JobPluginInfo;
Expand Down Expand Up @@ -157,52 +157,55 @@ public Report execute(IndexService index, ModelService model, StorageService sto
}

protected void generateBackfill(ModelService model, IndexService index, StorageService storage, Report report,
JobPluginInfo jobPluginInfo, Job cachedJob) {
JobPluginInfo jobPluginInfo, Job cachedJob) throws PluginException {
report.setPluginState(PluginState.SUCCESS);
List<String> processedIds = new LinkedList<>();

CloseableIterable<OptionalWithCause<LogEntry>> objects = model.listLogEntries();
// TODO: Get this from config
int batchSize = 100;
int blockSize = 10;
int blockSize = RodaCoreFactory.getRodaConfigurationAsInt("core", "plugins", "internal", "backfill", "blockSize");
if (blockSize == 0) {
blockSize = 1000;
}
Add addBean = new Add();
int docCount = 0;
int addCount = 0;
for (OptionalWithCause<LogEntry> object : objects) {
if (object.isPresent() && (startDate == null || object.get().getDatetime().after(startDate))) {
// TODO Handle exceptions
try {
DocType docBean = GenerateBackfillPluginUtils.toDocBean(object.get(), LogEntry.class);
addBean.getDoc().add(docBean);
processedIds.addLast(object.get().getId());

if (!onlyGenerateInventory) {
DocType docBean = GenerateBackfillPluginUtils.toDocBean(object.get(), LogEntry.class);
addBean.getDoc().add(docBean);
}
jobPluginInfo.incrementObjectsProcessedWithSuccess();
docCount++;
if (docCount >= blockSize * batchSize) {
processedIds.addLast(object.get().getId());
} catch (AuthorizationDeniedException | RequestNotValidException | NotFoundException | NotSupportedException
| GenericException e) {
report.setPluginState(PluginState.FAILURE);
report.addPluginDetails("Exception while processing object: " + e.getMessage());
jobPluginInfo.incrementObjectsProcessedWithFailure();
throw new PluginException(e);
}
if (docCount >= blockSize && !onlyGenerateInventory) {
try {
StoragePath addPath = GenerateBackfillPluginUtils.constructAddOutputPath(outputDirectory, LogEntry.class,
Integer.toString(addCount));
GenerateBackfillPluginUtils.writeAddBean(storage, addPath, addBean);
addBean = new Add();
addCount++;
docCount = 0;
} catch (AlreadyExistsException | RequestNotValidException | GenericException | AuthorizationDeniedException
| NotFoundException e) {
report.setPluginState(PluginState.FAILURE);
report.addPluginDetails("Exception while creating XML: " + e.getMessage());
throw new PluginException(e);
}
} catch (AuthorizationDeniedException e) {
throw new RuntimeException(e);
} catch (RequestNotValidException e) {
throw new RuntimeException(e);
} catch (NotFoundException e) {
throw new RuntimeException(e);
} catch (NotSupportedException e) {
throw new RuntimeException(e);
} catch (GenericException e) {
throw new RuntimeException(e);
} catch (AlreadyExistsException e) {
throw new RuntimeException(e);
}
}
}
// TODO Handle exceptions
try {
objects.close();
if (docCount > 0) {
if (docCount > 0 && !onlyGenerateInventory) {
StoragePath addPath = GenerateBackfillPluginUtils.constructAddOutputPath(outputDirectory, LogEntry.class,
Integer.toString(addCount));
GenerateBackfillPluginUtils.writeAddBean(storage, addPath, addBean);
Expand All @@ -211,10 +214,10 @@ protected void generateBackfill(ModelService model, IndexService index, StorageS
LogEntry.class);
GenerateBackfillPluginUtils.writeInventoryPartial(storage, inventoryPath, processedIds);
} catch (AlreadyExistsException | RequestNotValidException | GenericException | AuthorizationDeniedException
| NotFoundException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
| NotFoundException | IOException e) {
report.setPluginState(PluginState.FAILURE);
report.addPluginDetails("Exception while writing output files: " + e.getMessage());
throw new PluginException(e);
}
}

Expand Down
9 changes: 7 additions & 2 deletions ...rg/roda/core/plugins/base/maintenance/backfill/GenerateAllRODAEntitiesBackfillPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.common.RodaConstants.PreservationEventType;
import org.roda.core.data.exceptions.InvalidParameterException;
import org.roda.core.data.exceptions.LogEntryJsonParseException;
import org.roda.core.data.exceptions.NotFoundException;
import org.roda.core.data.exceptions.RODAException;
import org.roda.core.data.v2.IsRODAObject;
Expand All @@ -30,6 +31,7 @@
import org.roda.core.data.v2.index.select.SelectedItems;
import org.roda.core.data.v2.index.select.SelectedItemsAll;
import org.roda.core.data.v2.index.select.SelectedItemsFilter;
import org.roda.core.data.v2.index.select.SelectedItemsNone;
import org.roda.core.data.v2.ip.DIPFile;
import org.roda.core.data.v2.ip.File;
import org.roda.core.data.v2.ip.Representation;
Expand All @@ -39,6 +41,7 @@
import org.roda.core.data.v2.jobs.PluginState;
import org.roda.core.data.v2.jobs.PluginType;
import org.roda.core.data.v2.jobs.Report;
import org.roda.core.data.v2.log.LogEntry;
import org.roda.core.index.IndexService;
import org.roda.core.model.ModelService;
import org.roda.core.plugins.AbstractPlugin;
Expand Down Expand Up @@ -159,7 +162,6 @@ public Report execute(IndexService index, ModelService model, StorageService sto
protected void generateBackfill(ModelService model, IndexService index, StorageService storage, Report report,
JobPluginInfo jobPluginInfo, Job cachedJob, List<Class<? extends IsRODAObject>> classes) {
for (Class<? extends IsRODAObject> clazz : classes) {
// TODO handle exceptions
Report reportItem = generateRODAObjectBackfill(model, clazz, jobPluginInfo);
if (reportItem != null) {
report.addReport(reportItem);
Expand Down Expand Up @@ -215,7 +217,10 @@ protected <T extends IsRODAObject> Job initGenerateBackfillJob(Class<T> clazz, S

job.setPlugin(GenerateBackfillPluginUtils.getGeneratedBackfillPluginName(clazz));
SelectedItems<?> selectedItems;
if (startDate != null) {
if (clazz.equals(LogEntry.class)) {
selectedItems = new SelectedItemsNone<>();
}
else if (startDate != null) {
SelectedItemsFilter<?> selectedItemsFilter = new SelectedItemsFilter<>();
Filter filter = new Filter();
DateIntervalFilterParameter dateIntervalFilterParameter = new DateIntervalFilterParameter();
Expand Down
9 changes: 5 additions & 4 deletions ...ain/java/org/roda/core/plugins/base/maintenance/backfill/GenerateBackfillPluginUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ public static void writeInventoryPartial(StorageService storage, StoragePath pat
storage.createBinary(path, payload, false);
}

public static Set<String> readOriginalProcessedIds(StorageService storage, String directoryPath)
/*public static Set<String> readOriginalProcessedIds(StorageService storage, String directoryPath)
throws RequestNotValidException, AuthorizationDeniedException, NotFoundException, GenericException, IOException {
StoragePath storagePath = DefaultStoragePath.parse(List.of(BACKFILL_ROOT_DIRECTORY, directoryPath));
CloseableIterable<Resource> resources = storage.listResourcesUnderDirectory(storagePath, false);
Expand All @@ -192,13 +192,13 @@ public static Set<String> readOriginalProcessedIds(StorageService storage, Strin
}
resources.close();
return processedIds;
}
}*/

public static Add readAddBean(Resource addXMLResource)
/*public static Add readAddBean(Resource addXMLResource)
throws GenericException, IOException {
ContentPayload payload = ((DefaultBinary) addXMLResource).getContent();
return XMLUtils.getObjectFromXML(payload.createInputStream(), Add.class);
}
}*/

public static void writeDeleteBean(StorageService storage, StoragePath path, Delete deleteBean)
throws RequestNotValidException, GenericException, AuthorizationDeniedException, AlreadyExistsException,
Expand Down Expand Up @@ -286,6 +286,7 @@ public static List<Class<? extends IsRODAObject>> getBackfillObjectClasses() {
list.add(DIP.class);
list.add(DIPFile.class);
list.add(File.class);
list.add(Job.class);
list.add(Report.class);
list.add(DisposalConfirmation.class);
list.add(Representation.class);
Expand Down
Loading

0 comments on commit 584ec13

Please sign in to comment.