From 50e4ffbac05e58d72ff07b877b6ba90270e891de Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 25 Oct 2024 09:00:22 +0100 Subject: [PATCH] Updated logic for generate backfill plugin Signed-off-by: Alexandre Flores --- .../roda/core/data/common/RodaConstants.java | 5 +- .../org/roda/core/data/utils/XMLUtils.java | 18 ++ .../backfill/GenerateAIPBackfillPlugin.java | 14 +- .../GenerateActionLogBackfillPlugin.java | 97 +------- ...GenerateAllRODAEntitiesBackfillPlugin.java | 105 +++++--- .../backfill/GenerateBackfillPluginUtils.java | 147 ++++++++--- .../backfill/GenerateDIPBackfillPlugin.java | 11 - ...ateDisposalConfirmationBackfillPlugin.java | 11 - .../GenerateIncidenceBackfillPlugin.java | 11 - .../backfill/GenerateJobBackfillPlugin.java | 14 -- .../GenerateNotificationBackfillPlugin.java | 99 +------- ...neratePreservationAgentBackfillPlugin.java | 99 +------- ...ervationRepositoryEventBackfillPlugin.java | 99 +------- .../GenerateRODAEntityBackfillPlugin.java | 234 ++++++++++-------- .../GenerateRODAMemberBackfillPlugin.java | 97 +------- ...presentationInformationBackfillPlugin.java | 11 - .../backfill/GenerateRiskBackfillPlugin.java | 14 -- ...rateTransferredResourceBackfillPlugin.java | 97 +------- 18 files changed, 373 insertions(+), 810 deletions(-) diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java index 5a91c7adf0..469a2cbdd6 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java @@ -1427,8 +1427,9 @@ public enum OrchestratorType { // Generate Backfill Plugin Parameters - public static final String PLUGIN_PARAMS_BLOCK_SIZE = "parameter.block_size"; - public static final String PLUGIN_PARAMS_VALIDATE_AGAINST = "parameter.validate_against"; + public static final String PLUGIN_PARAMS_OUTPUT_DIRECTORY = "parameter.output_directory"; + public static final String PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY = "parameter.only_generate_inventory"; + public static final String PLUGIN_PARAMS_START_DATE = "parameter.start_date"; public static final String PLUGIN_CATEGORY_CONVERSION = "conversion"; public static final String PLUGIN_CATEGORY_CHARACTERIZATION = "characterization"; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/utils/XMLUtils.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/utils/XMLUtils.java index b15e9799cc..320b65f1e0 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/utils/XMLUtils.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/utils/XMLUtils.java @@ -16,6 +16,7 @@ import javax.xml.parsers.SAXParserFactory; import javax.xml.transform.sax.SAXSource; +import jakarta.xml.bind.JAXBElement; import org.apache.commons.io.IOUtils; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.exceptions.GenericException; @@ -50,6 +51,23 @@ public static String getXMLFromObject(Object object) throws GenericException { return ret; } + public static String getXMLFragFromObject(JAXBElement object) throws GenericException { + String ret = null; + JAXBContext jaxbContext; + try { + jaxbContext = JAXBContext.newInstance(object.getValue().getClass()); + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FRAGMENT, true); + StringWriter writer = new StringWriter(); + marshaller.marshal(object, writer); + ret = writer.toString(); + } catch (JAXBException e) { + throw new GenericException(e); + } + + return ret; + } + public static T getObjectFromXML(InputStream xml, Class objectClass) throws GenericException { T ret; try { diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAIPBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAIPBackfillPlugin.java index e2e7596480..598f228d0f 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAIPBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAIPBackfillPlugin.java @@ -1,17 +1,12 @@ package org.roda.core.plugins.base.maintenance.backfill; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; +import java.util.List; + import org.roda.core.data.v2.index.IsIndexed; import org.roda.core.data.v2.ip.AIP; import org.roda.core.data.v2.ip.IndexedAIP; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; -import java.util.List; - /** * @author Alexandre Flores */ @@ -22,11 +17,6 @@ protected Class getIndexClass() { return (Class) IndexedAIP.class; } - @Override - protected AIP retrieveModelObject(ModelService model, String id) throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveAIP(id); - } - @Override public Plugin cloneMe() { return new GenerateAIPBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateActionLogBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateActionLogBackfillPlugin.java index 603027e2dc..9a458aeaf3 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateActionLogBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateActionLogBackfillPlugin.java @@ -2,111 +2,28 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; import org.roda.core.data.v2.log.LogEntry; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GenerateActionLogBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete action log index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } +public class GenerateActionLogBackfillPlugin extends GenerateRODAEntityBackfillPlugin { @Override - public String getPreservationEventDescription() { - return ""; + protected Class getIndexClass() { + return (Class) LogEntry.class; } - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } - - @Override - public List getCategories() { - return List.of(); - } - - @Override - public Plugin cloneMe() { - return new GenerateActionLogBackfillPlugin(); - } - - @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - + public Plugin cloneMe() { + return new GenerateActionLogBackfillPlugin(); } @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(LogEntry.class); } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAllRODAEntitiesBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAllRODAEntitiesBackfillPlugin.java index 3c47c9e6a7..a77d2929c4 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAllRODAEntitiesBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateAllRODAEntitiesBackfillPlugin.java @@ -7,11 +7,13 @@ */ package org.roda.core.plugins.base.maintenance.backfill; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; @@ -23,17 +25,16 @@ import org.roda.core.data.v2.IsRODAObject; import org.roda.core.data.v2.LiteOptionalWithCause; import org.roda.core.data.v2.Void; +import org.roda.core.data.v2.index.filter.DateIntervalFilterParameter; +import org.roda.core.data.v2.index.filter.Filter; +import org.roda.core.data.v2.index.select.SelectedItems; import org.roda.core.data.v2.index.select.SelectedItemsAll; -import org.roda.core.data.v2.index.select.SelectedItemsNone; -import org.roda.core.data.v2.ip.TransferredResource; -import org.roda.core.data.v2.ip.metadata.IndexedPreservationAgent; +import org.roda.core.data.v2.index.select.SelectedItemsFilter; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginParameter; import org.roda.core.data.v2.jobs.PluginState; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.data.v2.jobs.Report; -import org.roda.core.data.v2.log.LogEntry; -import org.roda.core.data.v2.user.RODAMember; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; import org.roda.core.plugins.AbstractPlugin; @@ -50,18 +51,33 @@ public class GenerateAllRODAEntitiesBackfillPlugin extends AbstractPlugin { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateAllRODAEntitiesBackfillPlugin.class); - private int blockSize = 100000; - private HashMap> initialIdsManifest = new HashMap<>(); - private HashMap> processedIdsManifest = new HashMap<>(); + private String outputDirectory = "."; + private boolean onlyGenerateInventory = false; + private Date startDate = null; - private static Map pluginParameters = new HashMap<>(); + private static final Map pluginParameters = new HashMap<>(); static { - pluginParameters.put(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE, + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY, PluginParameter - .getBuilder(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE, "Block size", PluginParameter.PluginParameterType.INTEGER) - .withDefaultValue("100000").isMandatory(false) - .withDescription("Number of documents in each index documents block.").build()); + .getBuilder(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY, "Output directory", + PluginParameter.PluginParameterType.STRING) + .withDefaultValue(".").isMandatory(true).withDescription("This job's output directory path").build()); + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY, + PluginParameter + .getBuilder(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY, "Only generate inventory", + PluginParameter.PluginParameterType.BOOLEAN) + .withDefaultValue("false").isMandatory(true) + .withDescription( + "Whether this job should only generate the inventory of RODA objects and not the index backfill files") + .build()); + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_START_DATE, + PluginParameter + .getBuilder(RodaConstants.PLUGIN_PARAMS_START_DATE, "Object starting date", + PluginParameter.PluginParameterType.STRING) + .isMandatory(false).withDescription( + "The last modified data for source objects to process. If not set, all objects will be processed.") + .build()); } @Override @@ -98,15 +114,28 @@ public String getVersionImpl() { @Override public List getParameters() { ArrayList parameters = new ArrayList<>(); - parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_START_DATE)); return parameters; } @Override public void setParameterValues(Map parameters) throws InvalidParameterException { super.setParameterValues(parameters); - if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)) { - blockSize = Integer.parseInt(parameters.get(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)); + if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY)) { + outputDirectory = parameters.get(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY); + } + if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)) { + onlyGenerateInventory = Boolean.parseBoolean(parameters.get(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)); + } + if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_START_DATE)) { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + try { + startDate = formatter.parse(parameters.get(RodaConstants.PLUGIN_PARAMS_START_DATE)); + } catch (ParseException e) { + throw new InvalidParameterException(e); + } } } @@ -141,7 +170,6 @@ protected Report generateRODAObjectBackfill(ModelService model, Class Job initGenerateBackfillJob(Class clazz, S job.setId(jobId); job.setName(jobName); - Map pluginParameters = new HashMap<>(); - pluginParameters.put(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE, String.valueOf(blockSize)); - job.setPluginParameters(pluginParameters); + Map localPluginParameters = new HashMap<>(); + localPluginParameters.put(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY, outputDirectory); + localPluginParameters.put(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY, + String.valueOf(onlyGenerateInventory)); + job.setPluginParameters(localPluginParameters); job.setPluginType(PluginType.MISC); job.setUsername(username); job.setPlugin(GenerateBackfillPluginUtils.getGeneratedBackfillPluginName(clazz)); - job.setSourceObjects(SelectedItemsAll.create(clazz)); + SelectedItems selectedItems; + if (startDate != null) { + SelectedItemsFilter selectedItemsFilter = new SelectedItemsFilter<>(); + Filter filter = new Filter(); + DateIntervalFilterParameter dateIntervalFilterParameter = new DateIntervalFilterParameter(); + dateIntervalFilterParameter.setFromValue(startDate); + filter.add(dateIntervalFilterParameter); + selectedItemsFilter.setFilter(filter); + selectedItems = selectedItemsFilter; + } else { + selectedItems = SelectedItemsAll.create(clazz); + } + job.setSourceObjects(selectedItems); return job; } @@ -209,28 +251,13 @@ public String getPreservationEventFailureMessage() { @Override public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) { - // Get all currently indexed IDs - /* - * List> classes = - * PluginHelper.getReindexObjectClasses(); for (Class - * clazz : classes) { // TODO handle exceptions try { Class - * indexClass = GenerateBackfillPluginUtils.getIndexClass(clazz); - * CloseableIterable> - * indexedObjects = index.list(indexClass, List.of("id")); HashSet - * objectIds = new HashSet<>(); indexedObjects.forEach(o -> - * objectIds.add(o.get().getId())); indexedObjects.close(); - * initialIdsManifest.put(clazz.getSimpleName(), objectIds); } catch - * (NotFoundException e) { throw new RuntimeException(e); } catch - * (RequestNotValidException e) { throw new RuntimeException(e); } catch - * (GenericException e) { throw new RuntimeException(e); } catch (IOException e) - * { throw new RuntimeException(e); } } - */ + // do nothing return null; } @Override public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) { - + // do nothing return null; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateBackfillPluginUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateBackfillPluginUtils.java index 615f93a235..f10c8a7f37 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateBackfillPluginUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateBackfillPluginUtils.java @@ -2,13 +2,16 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.roda.core.common.iterables.CloseableIterable; +import org.roda.core.data.common.RodaConstants; import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; @@ -51,6 +54,7 @@ import org.roda.core.storage.DefaultStoragePath; import org.roda.core.storage.Resource; import org.roda.core.storage.StorageService; +import org.roda.core.storage.StringContentPayload; import org.roda.core.storage.XMLContentPayload; /** @@ -65,6 +69,49 @@ private GenerateBackfillPluginUtils() { public static final String VALIDATE_AGAINST_INDEX = "Index"; public static final String VALIDATE_AGAINST_STORAGE = "Storage"; + public static final String BACKFILL_ROOT_DIRECTORY = "backfill"; + + public static StoragePath constructAddOutputPath(String outputDirectory, Class clazz, String addId) + throws RequestNotValidException { + return DefaultStoragePath + .parse(List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "add", "add_" + addId + ".xml")); + } + + public static StoragePath constructAddPartialOutputPath(String outputDirectory, Class clazz, String batchId) + throws RequestNotValidException { + return DefaultStoragePath.parse( + List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "add", "add_" + batchId + ".xmlfrag")); + } + + public static StoragePath constructAddPartialsDirectoryPath(String outputDirectory, Class clazz) + throws RequestNotValidException { + return DefaultStoragePath.parse(List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "add")); + } + + public static StoragePath constructInventoryOutputPath(String outputDirectory, Class clazz) + throws RequestNotValidException { + return DefaultStoragePath + .parse(List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "inventory", "inventory.lst")); + } + + public static StoragePath constructInventoryPartialOutputPath(String outputDirectory, Class clazz, String batchId) + throws RequestNotValidException { + return DefaultStoragePath.parse( + List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "inventory", "inv_" + batchId + ".lst")); + } + + public static StoragePath constructInventoryPartialsDirectoryPath(String outputDirectory, Class clazz) + throws RequestNotValidException { + return DefaultStoragePath + .parse(List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName(), "inventory")); + } + + public static StoragePath constructBeanOutputPath(String outputDirectory, Class clazz, int beanIndex) + throws RequestNotValidException { + return DefaultStoragePath + .parse(List.of(BACKFILL_ROOT_DIRECTORY, outputDirectory, clazz.getSimpleName() + "_" + beanIndex + ".xml")); + } + public static DocType toDocBean(M object, Class indexClass) throws AuthorizationDeniedException, RequestNotValidException, NotFoundException, NotSupportedException, GenericException { @@ -91,42 +138,79 @@ public static DocType toDocBean(M return docBean; } - public static void writeAddBean(StorageService storage, String filename, Add addBean) throws RequestNotValidException, + public static DocType toDocBean(I object) { + DocType docBean = new DocType(); + + for (Map.Entry field : object.getFields().entrySet()) { + Object value = field.getValue(); + if (value instanceof String string && !string.isEmpty()) { + FieldType fieldBean = new FieldType(); + fieldBean.setName(field.getKey()); + fieldBean.setValue(string); + docBean.getField().add(fieldBean); + } else if (value instanceof List multiValueField) { + for (Object multiValue : multiValueField) { + FieldType fieldBean = new FieldType(); + fieldBean.setName(field.getKey()); + fieldBean.setValue(multiValue.toString()); + docBean.getField().add(fieldBean); + } + } + } + + return docBean; + } + + public static void writeAddBean(StorageService storage, StoragePath path, Add addBean) + throws RequestNotValidException, GenericException, AuthorizationDeniedException, AlreadyExistsException, NotFoundException { - StoragePath storagePath = DefaultStoragePath.parse(List.of(".", filename)); String xml = XMLUtils.getXMLFromObject(addBean); ContentPayload payload = new XMLContentPayload(xml); - storage.createBinary(storagePath, payload, false); + storage.createBinary(path, payload, false); } - public static List readAddBeans(StorageService storage, String directoryPath) + public static void writeAddPartial(StorageService storage, StoragePath path, String addPartialXML) + throws AuthorizationDeniedException, RequestNotValidException, AlreadyExistsException, NotFoundException, + GenericException { + ContentPayload payload = new StringContentPayload(addPartialXML); + storage.createBinary(path, payload, false); + } + + public static void writeInventoryPartial(StorageService storage, StoragePath path, Collection processedIds) + throws AuthorizationDeniedException, RequestNotValidException, AlreadyExistsException, NotFoundException, + GenericException { + ContentPayload payload = new StringContentPayload(String.join("\n", processedIds)); + storage.createBinary(path, payload, false); + } + + public static Set readOriginalProcessedIds(StorageService storage, String directoryPath) throws RequestNotValidException, AuthorizationDeniedException, NotFoundException, GenericException, IOException { - StoragePath storagePath = DefaultStoragePath.parse(List.of(".", directoryPath)); + StoragePath storagePath = DefaultStoragePath.parse(List.of(BACKFILL_ROOT_DIRECTORY, directoryPath)); CloseableIterable resources = storage.listResourcesUnderDirectory(storagePath, false); - List addBeans = new ArrayList<>(); + Set processedIds = new HashSet<>(); for (Resource resource : resources) { - addBeans.add(readAddBean(resource)); + Add addBean = readAddBean(resource); + for (DocType docBean : addBean.getDoc()) { + processedIds.add(docBean.getField().stream().filter(field -> field.getName().equals(RodaConstants.INDEX_ID)) + .findFirst().get().getValue()); + } } - return addBeans; + resources.close(); + return processedIds; } public static Add readAddBean(Resource addXMLResource) - throws RequestNotValidException, GenericException, IOException { + throws GenericException, IOException { ContentPayload payload = ((DefaultBinary) addXMLResource).getContent(); return XMLUtils.getObjectFromXML(payload.createInputStream(), Add.class); } - public static void writeDeleteBean(StorageService storage, String filename, Delete deleteBean) + public static void writeDeleteBean(StorageService storage, StoragePath path, Delete deleteBean) throws RequestNotValidException, GenericException, AuthorizationDeniedException, AlreadyExistsException, NotFoundException { - StoragePath storagePath = DefaultStoragePath.parse(List.of(".", filename)); String xml = XMLUtils.getXMLFromObject(deleteBean); ContentPayload payload = new XMLContentPayload(xml); - storage.createBinary(storagePath, payload, false); - } - - public static void createAndWriteDeletionBeans(StorageService storage, List deletedIds) { - + storage.createBinary(path, payload, false); } public static String getGeneratedBackfillPluginName(Class clazz) throws NotFoundException { @@ -194,10 +278,11 @@ public static Class getIndexClass(Class HashSet generateBackfillForRODAObjects( - StorageService storage, List objects, int blockSize, Class indexClass, Report report) { + StorageService storage, List objects, int blockSize, Class indexClass, Report report, + String outputDirectory) { int totalBeans = 0; int count = 0; - Add addType = new Add(); + Add addBean = new Add(); HashSet processedIds = new HashSet<>(); for (T object : objects) { @@ -205,12 +290,12 @@ public static Hash try { processedIds.add(object.getId()); if (count == blockSize) { + StoragePath path = constructBeanOutputPath(outputDirectory, object.getClass(), totalBeans); + writeAddBean(storage, path, addBean); count = 0; totalBeans += 1; - GenerateBackfillPluginUtils.writeAddBean(storage, object.getClass().getName() + "_" + totalBeans + ".xml", - addType); } - addType.getDoc().add(GenerateBackfillPluginUtils.toDocBean(object, getIndexClass(indexClass))); + addBean.getDoc().add(GenerateBackfillPluginUtils.toDocBean(object, getIndexClass(indexClass))); count += 1; } catch (AuthorizationDeniedException e) { throw new RuntimeException(e); @@ -229,8 +314,8 @@ public static Hash // TODO Handle exceptions try { totalBeans += 1; - GenerateBackfillPluginUtils.writeAddBean(storage, - objects.getFirst().getClass().getSimpleName() + "_" + totalBeans + ".xml", addType); + StoragePath path = constructBeanOutputPath(BACKFILL_ROOT_DIRECTORY, objects.getFirst().getClass(), totalBeans); + writeAddBean(storage, path, addBean); } catch (AlreadyExistsException | RequestNotValidException | GenericException | AuthorizationDeniedException | NotFoundException e) { throw new RuntimeException(e); @@ -241,8 +326,8 @@ public static Hash public static List checkIndexForDeletedObjects(IndexService index, Class indexClass, List objectIds) throws RequestNotValidException, GenericException, IOException { Filter filter = new Filter(); - filter.add(new OneOfManyFilterParameter("id", objectIds)); - IterableIndexResult indexResult = index.findAll(indexClass, filter, List.of("id")); + filter.add(new OneOfManyFilterParameter(RodaConstants.INDEX_ID, objectIds)); + IterableIndexResult indexResult = index.findAll(indexClass, filter, List.of(RodaConstants.INDEX_ID)); List existingIndexIdsList = new ArrayList<>(); for (I indexedObject : indexResult) { existingIndexIdsList.add(indexedObject.getId()); @@ -251,19 +336,9 @@ public static List checkIndexForDeletedObjects(Ind return objectIds.stream().filter(id -> !existingIndexIdsList.contains(id)).toList(); } - /* - * public static List - * checkIndexForModifiedObjects(IndexService index, Class indexClass, - * List objectIds) throws RequestNotValidException, GenericException { - * Filter filter = new Filter(); filter.add(new OneOfManyFilterParameter("id", - * objectIds)); IndexResult result = index.find(indexClass, filter, - * Sorter.NONE, Sublist.NONE, List.of("id")); return - * result.getResults().stream().map(I::getId).toList(); } - */ - public static List checkIndexForAddedObjects(IndexService index, Class indexClass, Set objectIds) throws RequestNotValidException, GenericException, IOException { - IterableIndexResult indexResult = index.findAll(indexClass, Filter.ALL, List.of("id")); + IterableIndexResult indexResult = index.findAll(indexClass, Filter.ALL, List.of(RodaConstants.INDEX_ID)); List unprocessedIndexIds = new ArrayList<>(); for (I indexObject : indexResult) { if (!objectIds.contains(indexObject.getId())) { diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDIPBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDIPBackfillPlugin.java index ab0edd4d2f..1947e3193f 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDIPBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDIPBackfillPlugin.java @@ -2,14 +2,9 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.index.IsIndexed; import org.roda.core.data.v2.ip.DIP; import org.roda.core.data.v2.ip.IndexedDIP; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -22,12 +17,6 @@ protected Class getIndexClass() { return (Class) IndexedDIP.class; } - @Override - protected DIP retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveDIP(id); - } - @Override public Plugin cloneMe() { return new GenerateDIPBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDisposalConfirmationBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDisposalConfirmationBackfillPlugin.java index 7041a5420f..413200a02a 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDisposalConfirmationBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateDisposalConfirmationBackfillPlugin.java @@ -2,13 +2,8 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.disposal.confirmation.DisposalConfirmation; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -21,12 +16,6 @@ protected Class getIndexClass() { return (Class) DisposalConfirmation.class; } - @Override - protected DisposalConfirmation retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveDisposalConfirmation(id); - } - @Override public Plugin cloneMe() { return new GenerateDisposalConfirmationBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateIncidenceBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateIncidenceBackfillPlugin.java index 90802232de..05b093751b 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateIncidenceBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateIncidenceBackfillPlugin.java @@ -2,13 +2,8 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.index.IsIndexed; import org.roda.core.data.v2.risks.RiskIncidence; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -21,12 +16,6 @@ protected Class getIndexClass() { return (Class) RiskIncidence.class; } - @Override - protected RiskIncidence retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveRiskIncidence(id); - } - @Override public Plugin cloneMe() { return new GenerateIncidenceBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateJobBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateJobBackfillPlugin.java index 81f6fbe70c..5c8685841f 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateJobBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateJobBackfillPlugin.java @@ -2,16 +2,8 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.ip.AIP; -import org.roda.core.data.v2.ip.IndexedAIP; import org.roda.core.data.v2.jobs.Job; -import org.roda.core.data.v2.risks.RiskIncidence; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -24,12 +16,6 @@ protected Class getIndexClass() { return (Class) Job.class; } - @Override - protected Job retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveJob(id); - } - @Override public Plugin cloneMe() { diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateNotificationBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateNotificationBackfillPlugin.java index ee2f3ef621..233e4921f0 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateNotificationBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateNotificationBackfillPlugin.java @@ -2,111 +2,28 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; +import org.roda.core.data.v2.index.IsIndexed; +import org.roda.core.data.v2.notifications.Notification; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GenerateNotificationBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete notifications index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } +public class GenerateNotificationBackfillPlugin extends GenerateRODAEntityBackfillPlugin { @Override - public String getPreservationEventDescription() { - return ""; + protected Class getIndexClass() { + return (Class) Notification.class; } - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } - - @Override - public List getCategories() { - return List.of(); - } - - @Override - public Plugin cloneMe() { + public Plugin cloneMe() { return new GenerateNotificationBackfillPlugin(); } @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) - throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, - List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - - } - - @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(Notification.class); } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationAgentBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationAgentBackfillPlugin.java index c347c4d65f..7835e522fb 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationAgentBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationAgentBackfillPlugin.java @@ -2,111 +2,28 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; -import org.roda.core.data.v2.log.LogEntry; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; +import org.roda.core.data.v2.ip.metadata.IndexedPreservationAgent; +import org.roda.core.data.v2.ip.metadata.PreservationMetadata; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GeneratePreservationAgentBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete preservation agent index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } - - @Override - public String getPreservationEventDescription() { - return ""; - } - - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } - - @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } +public class GeneratePreservationAgentBackfillPlugin extends GenerateRODAEntityBackfillPlugin { @Override - public List getCategories() { - return List.of(); + protected Class getIndexClass() { + return (Class) IndexedPreservationAgent.class; } @Override - public Plugin cloneMe() { + public Plugin cloneMe() { return new GeneratePreservationAgentBackfillPlugin(); } @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - - } - - @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(PreservationMetadata.class); } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationRepositoryEventBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationRepositoryEventBackfillPlugin.java index dfcf79713d..cd4c69fb1a 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationRepositoryEventBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GeneratePreservationRepositoryEventBackfillPlugin.java @@ -2,109 +2,30 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; +import org.roda.core.data.v2.index.IsIndexed; +import org.roda.core.data.v2.ip.metadata.IndexedPreservationEvent; +import org.roda.core.data.v2.ip.metadata.PreservationMetadata; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GeneratePreservationRepositoryEventBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete preservation event index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } +public class GeneratePreservationRepositoryEventBackfillPlugin + extends GenerateRODAEntityBackfillPlugin { @Override - public String getPreservationEventDescription() { - return ""; + protected Class getIndexClass() { + return (Class) IndexedPreservationEvent.class; } - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } - - @Override - public List getCategories() { - return List.of(); - } - - @Override - public Plugin cloneMe() { + public Plugin cloneMe() { return new GeneratePreservationRepositoryEventBackfillPlugin(); } @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - - } - - @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(PreservationMetadata.class); } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAEntityBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAEntityBackfillPlugin.java index 445c75359c..af7e03bc60 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAEntityBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAEntityBackfillPlugin.java @@ -8,14 +8,18 @@ package org.roda.core.plugins.base.maintenance.backfill; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import javax.xml.namespace.QName; + +import org.roda.core.common.iterables.CloseableIterable; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.common.RodaConstants.PreservationEventType; import org.roda.core.data.exceptions.AlreadyExistsException; @@ -25,10 +29,12 @@ import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.NotSupportedException; import org.roda.core.data.exceptions.RequestNotValidException; +import org.roda.core.data.utils.XMLUtils; import org.roda.core.data.v2.IsModelObject; import org.roda.core.data.v2.IsRODAObject; import org.roda.core.data.v2.LiteOptionalWithCause; import org.roda.core.data.v2.index.IsIndexed; +import org.roda.core.data.v2.ip.StoragePath; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginParameter; import org.roda.core.data.v2.jobs.PluginType; @@ -39,42 +45,41 @@ import org.roda.core.plugins.PluginException; import org.roda.core.plugins.PluginHelper; import org.roda.core.plugins.RODAObjectsProcessingLogic; -import org.roda.core.plugins.base.maintenance.backfill.beans.Add; -import org.roda.core.plugins.base.maintenance.backfill.beans.Delete; +import org.roda.core.plugins.base.maintenance.backfill.beans.DocType; import org.roda.core.plugins.orchestrate.JobPluginInfo; +import org.roda.core.storage.DefaultBinary; +import org.roda.core.storage.Resource; import org.roda.core.storage.StorageService; +import org.roda.core.storage.StringContentPayload; +import org.roda.core.util.IdUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.xml.bind.JAXBElement; + public abstract class GenerateRODAEntityBackfillPlugin extends AbstractPlugin { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateRODAEntityBackfillPlugin.class); - private int blockSize = 100000; - private String validateAgainst = "None"; - - private final HashSet processedObjectIds = new HashSet<>(); + private String outputDirectory = "."; + private boolean onlyGenerateInventory = false; - private final static Map pluginParameters = new HashMap<>(); + private static final Map pluginParameters = new HashMap<>(); static { - pluginParameters.put(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE, + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY, PluginParameter - .getBuilder(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE, "Block size", PluginParameter.PluginParameterType.INTEGER) - .withDefaultValue("100000").isMandatory(false) - .withDescription("Number of documents in each index documents block.").build()); - pluginParameters - .put(RodaConstants.PLUGIN_PARAMS_VALIDATE_AGAINST, - PluginParameter - .getBuilder(RodaConstants.PLUGIN_PARAMS_VALIDATE_AGAINST, "Check parity against", - PluginParameter.PluginParameterType.DROPDOWN) - .withPossibleValues(Arrays.asList(GenerateBackfillPluginUtils.VALIDATE_AGAINST_NONE, - GenerateBackfillPluginUtils.VALIDATE_AGAINST_INDEX, GenerateBackfillPluginUtils.VALIDATE_AGAINST_STORAGE)) - .withDefaultValue(GenerateBackfillPluginUtils.VALIDATE_AGAINST_NONE).isMandatory(true) - .withDescription( - "Which collection the final result will be validated against to adjust for mid-execution changes.") - .build()); - + .getBuilder(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY, "Output directory", + PluginParameter.PluginParameterType.STRING) + .withDefaultValue(".").isMandatory(true).withDescription("This job's output directory path").build()); + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY, + PluginParameter + .getBuilder(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY, "Only generate inventory", + PluginParameter.PluginParameterType.BOOLEAN) + .withDefaultValue("false").isMandatory(true) + .withDescription( + "Whether this job should only generate the inventory of RODA objects and not the index backfill files") + .build()); } protected GenerateRODAEntityBackfillPlugin() { @@ -93,8 +98,7 @@ public void shutdown() { @Override public String getName() { - // Get from pom.xml - return getClass().getPackage().getImplementationTitle(); + return "Generate " + getObjectClasses().getFirst().getSimpleName() + " index backfill"; } @Override @@ -104,26 +108,25 @@ public String getDescription() { @Override public String getVersionImpl() { - // Get from pom.xml - return getClass().getPackage().getImplementationVersion(); + return "1.0"; } @Override public List getParameters() { ArrayList parameters = new ArrayList<>(); - parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)); - parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_VALIDATE_AGAINST)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)); return parameters; } @Override public void setParameterValues(Map parameters) throws InvalidParameterException { super.setParameterValues(parameters); - if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)) { - blockSize = Integer.parseInt(parameters.get(RodaConstants.PLUGIN_PARAMS_BLOCK_SIZE)); + if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY)) { + outputDirectory = parameters.get(RodaConstants.PLUGIN_PARAMS_OUTPUT_DIRECTORY); } - if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_VALIDATE_AGAINST)) { - validateAgainst = parameters.get(RodaConstants.PLUGIN_PARAMS_VALIDATE_AGAINST); + if (parameters != null && parameters.containsKey(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)) { + onlyGenerateInventory = Boolean.parseBoolean(parameters.get(RodaConstants.PLUGIN_PARAMS_ONLY_GENERATE_INVENTORY)); } } @@ -138,22 +141,17 @@ public Report execute(IndexService index, ModelService model, StorageService sto protected void generateBackfill(ModelService model, IndexService index, StorageService storage, Report report, JobPluginInfo jobPluginInfo, Job job, List objects) { - int totalBeans = 0; - int count = 0; - Add addType = new Add(); + StringBuilder docXMLs = new StringBuilder(); + List processedIds = new LinkedList<>(); + String batchId = IdUtils.createUUID(); for (T object : objects) { // TODO Handle exceptions try { - processedObjectIds.add(object.getId()); - if (count == blockSize) { - count = 0; - totalBeans += 1; - GenerateBackfillPluginUtils.writeAddBean(storage, pluginParameters.get(RodaConstants.PLUGIN_PARAMS_JOB_ID) - + "/" + object.getClass().getName() + "_" + totalBeans + ".xml", addType); - } - addType.getDoc().add(GenerateBackfillPluginUtils.toDocBean(object, getIndexClass())); - count += 1; + DocType docBean = GenerateBackfillPluginUtils.toDocBean(object, getIndexClass()); + JAXBElement rootElement = new JAXBElement<>(new QName("doc"), DocType.class, docBean); + docXMLs.append(XMLUtils.getXMLFragFromObject(rootElement)); + processedIds.addLast(object.getId()); } catch (AuthorizationDeniedException e) { throw new RuntimeException(e); } catch (RequestNotValidException e) { @@ -164,15 +162,16 @@ protected void generateBackfill(ModelService model, IndexService index, StorageS throw new RuntimeException(e); } catch (GenericException e) { throw new RuntimeException(e); - } catch (AlreadyExistsException e) { - throw new RuntimeException(e); } } // TODO Handle exceptions try { - totalBeans += 1; - GenerateBackfillPluginUtils.writeAddBean(storage, - objects.getFirst().getClass().getSimpleName() + "_" + totalBeans + ".xml", addType); + StoragePath addPartialPath = GenerateBackfillPluginUtils.constructAddPartialOutputPath(outputDirectory, + objects.getFirst().getClass(), batchId); + GenerateBackfillPluginUtils.writeAddPartial(storage, addPartialPath, docXMLs.toString()); + StoragePath inventoryPartialPath = GenerateBackfillPluginUtils + .constructInventoryPartialOutputPath(outputDirectory, objects.getFirst().getClass(), batchId); + GenerateBackfillPluginUtils.writeInventoryPartial(storage, inventoryPartialPath, processedIds); } catch (AlreadyExistsException | RequestNotValidException | GenericException | AuthorizationDeniedException | NotFoundException e) { throw new RuntimeException(e); @@ -181,18 +180,6 @@ protected void generateBackfill(ModelService model, IndexService index, StorageS protected abstract Class getIndexClass(); - protected List retrieveModelObjects(ModelService model, List ids) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - List result = new ArrayList<>(); - for (String id : ids) { - result.add(retrieveModelObject(model, id)); - } - return result; - } - - protected abstract T retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException; - @Override public PluginType getType() { return PluginType.AIP_TO_AIP; @@ -231,64 +218,95 @@ public Report beforeAllExecute(IndexService index, ModelService model, StorageSe @Override public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) { - if (validateAgainst.equals(GenerateBackfillPluginUtils.VALIDATE_AGAINST_INDEX)) { - validateAgainstIndex(index, model, storage); - } - return null; - + concatenateAndWriteAdds(storage); + concatenateAndWriteInventory(storage); + // TODO return report + return new Report(); } - protected Report validateAgainstIndex(IndexService index, ModelService model, StorageService storage) { + protected Report concatenateAndWriteAdds(StorageService storage) { + // TODO Get this from plugin config + int blockSize = 10; // TODO Handle exceptions try { - List originalAddBeans = GenerateBackfillPluginUtils.readAddBeans(storage, "test"); - List deletedObjectIds = GenerateBackfillPluginUtils.checkIndexForDeletedObjects(index, getIndexClass(), - processedObjectIds.stream().toList()); - List addedObjects = GenerateBackfillPluginUtils.checkIndexForAddedObjects(index, getIndexClass(), - processedObjectIds); - Add addBean = new Add(); - int addBeanCount = 0; - Delete deleteBean = new Delete(); - int deleteBeanCount = 0; - while (!deletedObjectIds.isEmpty() || !addedObjects.isEmpty()) { - for (String id : deletedObjectIds) { - // Write delete query and remove from the processed ids - deleteBean.getId().add(id); - processedObjectIds.remove(id); - if (deleteBean.getId().size() > blockSize) { - GenerateBackfillPluginUtils.writeDeleteBean(storage, - pluginParameters.get(RodaConstants.PLUGIN_PARAMS_JOB_ID) + "/index_deleted/" - + getIndexClass().getSimpleName() + "_" + deleteBeanCount + ".xml", - deleteBean); - deleteBean = new Delete(); - } + StoragePath addPartialsDirectory = GenerateBackfillPluginUtils.constructAddPartialsDirectoryPath(outputDirectory, + getObjectClasses().getFirst()); + CloseableIterable addPartials = storage.listResourcesUnderDirectory(addPartialsDirectory, false); + StringBuilder addStringBuilder = new StringBuilder(); + addStringBuilder.append(""); + int addedBatches = 0; + int writtenBlocks = 0; + for (Resource addPartialResource : addPartials) { + InputStream addPartialStream = ((DefaultBinary) addPartialResource).getContent().createInputStream(); + addStringBuilder.append(new String(addPartialStream.readAllBytes(), StandardCharsets.UTF_8)); + addPartialStream.close(); + addedBatches++; + if (addedBatches > blockSize) { + addStringBuilder.append(""); + StoragePath addPath = GenerateBackfillPluginUtils.constructAddOutputPath(outputDirectory, + getObjectClasses().getFirst(), Integer.toString(writtenBlocks)); + storage.createBinary(addPath, new StringContentPayload(addStringBuilder.toString()), false); + + addStringBuilder = new StringBuilder(); + addStringBuilder.append(""); + addedBatches = 0; + writtenBlocks++; } - for (String id : addedObjects) { - // Write add query and add to the processed ids - addBean.getDoc().add(GenerateBackfillPluginUtils.toDocBean(retrieveModelObject(model, id), getIndexClass())); - processedObjectIds.add(id); - if (addBean.getDoc().size() > blockSize) { - GenerateBackfillPluginUtils.writeAddBean(storage, - "index_added/" + getIndexClass().getSimpleName() + "_" + addBeanCount + ".xml", addBean); - addBean = new Add(); - } - } - deletedObjectIds = GenerateBackfillPluginUtils.checkIndexForDeletedObjects(index, getIndexClass(), - processedObjectIds.stream().toList()); - addedObjects = GenerateBackfillPluginUtils.checkIndexForAddedObjects(index, getIndexClass(), - processedObjectIds); + storage.deleteResource(addPartialResource.getStoragePath()); + } + addPartials.close(); + if (addedBatches > 0) { + addStringBuilder.append(""); + StoragePath addPath = GenerateBackfillPluginUtils.constructAddOutputPath(outputDirectory, + getObjectClasses().getFirst(), Integer.toString(writtenBlocks)); + storage.createBinary(addPath, new StringContentPayload(addStringBuilder.toString()), false); } - } catch (GenericException e) { - throw new RuntimeException(e); } catch (RequestNotValidException e) { throw new RuntimeException(e); + } catch (AuthorizationDeniedException e) { + throw new RuntimeException(e); + } catch (NotFoundException e) { + throw new RuntimeException(e); + } catch (GenericException e) { + throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); + } catch (AlreadyExistsException e) { + throw new RuntimeException(e); + } + // TODO return report + return null; + } + + protected Report concatenateAndWriteInventory(StorageService storage) { + // TODO Handle exceptions + try { + StoragePath inventoryPartialsDirectory = GenerateBackfillPluginUtils + .constructInventoryPartialsDirectoryPath(outputDirectory, getObjectClasses().getFirst()); + CloseableIterable inventoryPartials = storage.listResourcesUnderDirectory(inventoryPartialsDirectory, + false); + StringBuilder inventoryStringBuilder = new StringBuilder(); + for (Resource inventoryPartialResource : inventoryPartials) { + InputStream inventoryPartialStream = ((DefaultBinary) inventoryPartialResource).getContent() + .createInputStream(); + inventoryStringBuilder.append(new String(inventoryPartialStream.readAllBytes(), StandardCharsets.UTF_8)); + inventoryPartialStream.close(); + inventoryStringBuilder.append("\n"); + storage.deleteResource(inventoryPartialResource.getStoragePath()); + } + inventoryPartials.close(); + StoragePath inventoryPath = GenerateBackfillPluginUtils.constructInventoryOutputPath(outputDirectory, + getObjectClasses().getFirst()); + storage.createBinary(inventoryPath, new StringContentPayload(inventoryStringBuilder.toString()), false); + } catch (RequestNotValidException e) { + throw new RuntimeException(e); } catch (AuthorizationDeniedException e) { throw new RuntimeException(e); } catch (NotFoundException e) { throw new RuntimeException(e); - } catch (NotSupportedException e) { + } catch (GenericException e) { + throw new RuntimeException(e); + } catch (IOException e) { throw new RuntimeException(e); } catch (AlreadyExistsException e) { throw new RuntimeException(e); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAMemberBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAMemberBackfillPlugin.java index 0dc9971d30..3f66764ebf 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAMemberBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRODAMemberBackfillPlugin.java @@ -2,111 +2,28 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; -import org.roda.core.data.v2.log.LogEntry; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; +import org.roda.core.data.v2.user.RODAMember; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GenerateRODAMemberBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete RODA Member index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } +public class GenerateRODAMemberBackfillPlugin extends GenerateRODAEntityBackfillPlugin { @Override - public String getPreservationEventDescription() { - return ""; + protected Class getIndexClass() { + return (Class) RODAMember.class; } - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } - - @Override - public List getCategories() { - return List.of(); - } - - @Override - public Plugin cloneMe() { + public Plugin cloneMe() { return new GenerateRODAMemberBackfillPlugin(); } @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - - } - - @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(RODAMember.class); } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRepresentationInformationBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRepresentationInformationBackfillPlugin.java index 4c82cf1373..bd182ccb65 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRepresentationInformationBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRepresentationInformationBackfillPlugin.java @@ -2,13 +2,8 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.index.IsIndexed; import org.roda.core.data.v2.ri.RepresentationInformation; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -22,12 +17,6 @@ protected Class getIndexClass() { return (Class) RepresentationInformation.class; } - @Override - protected RepresentationInformation retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveRepresentationInformation(id); - } - @Override public Plugin cloneMe() { return new GenerateRepresentationInformationBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRiskBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRiskBackfillPlugin.java index d0f306122a..c7f6b1418a 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRiskBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateRiskBackfillPlugin.java @@ -2,17 +2,9 @@ import java.util.List; -import org.roda.core.data.exceptions.AuthorizationDeniedException; -import org.roda.core.data.exceptions.GenericException; -import org.roda.core.data.exceptions.NotFoundException; -import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.ip.AIP; -import org.roda.core.data.v2.ip.IndexedAIP; -import org.roda.core.data.v2.ri.RepresentationInformation; import org.roda.core.data.v2.risks.IndexedRisk; import org.roda.core.data.v2.risks.Risk; -import org.roda.core.model.ModelService; import org.roda.core.plugins.Plugin; /** @@ -25,12 +17,6 @@ protected Class getIndexClass() { return (Class) IndexedRisk.class; } - @Override - protected Risk retrieveModelObject(ModelService model, String id) - throws AuthorizationDeniedException, NotFoundException, GenericException, RequestNotValidException { - return model.retrieveRisk(id); - } - @Override public Plugin cloneMe() { return new GenerateRiskBackfillPlugin(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateTransferredResourceBackfillPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateTransferredResourceBackfillPlugin.java index 765d2fbb1e..e8252dcf30 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateTransferredResourceBackfillPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/maintenance/backfill/GenerateTransferredResourceBackfillPlugin.java @@ -2,111 +2,28 @@ import java.util.List; -import org.roda.core.data.common.RodaConstants; -import org.roda.core.data.v2.LiteOptionalWithCause; -import org.roda.core.data.v2.Void; import org.roda.core.data.v2.index.IsIndexed; -import org.roda.core.data.v2.jobs.PluginType; -import org.roda.core.data.v2.jobs.Report; -import org.roda.core.data.v2.log.LogEntry; -import org.roda.core.index.IndexService; -import org.roda.core.model.ModelService; -import org.roda.core.plugins.AbstractPlugin; +import org.roda.core.data.v2.ip.TransferredResource; import org.roda.core.plugins.Plugin; -import org.roda.core.plugins.PluginException; -import org.roda.core.storage.StorageService; /** * @author Alexandre Flores */ -public class GenerateTransferredResourceBackfillPlugin extends AbstractPlugin { - @Override - public String getName() { - return "Generate complete transferred resource index backfill"; - } - - @Override - public String getDescription() { - return ""; - } - - @Override - public RodaConstants.PreservationEventType getPreservationEventType() { - return null; - } +public class GenerateTransferredResourceBackfillPlugin extends GenerateRODAEntityBackfillPlugin { @Override - public String getPreservationEventDescription() { - return ""; + protected Class getIndexClass() { + return (Class) TransferredResource.class; } - @Override - public String getPreservationEventSuccessMessage() { - return ""; - } @Override - public String getPreservationEventFailureMessage() { - return ""; - } - - @Override - public String getPreservationEventSkippedMessage() { - return super.getPreservationEventSkippedMessage(); - } - - @Override - public PluginType getType() { - return PluginType.MISC; - } - - @Override - public List getCategories() { - return List.of(); - } - - @Override - public Plugin cloneMe() { + public Plugin cloneMe() { return new GenerateTransferredResourceBackfillPlugin(); } @Override - public boolean areParameterValuesValid() { - return false; - } - - @Override - public void init() throws PluginException { - - } - - @Override - public List> getObjectClasses() { - return List.of(Void.class); - } - - @Override - public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public Report execute(IndexService index, ModelService model, StorageService storage, List list) throws PluginException { - return null; - } - - @Override - public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { - return null; - } - - @Override - public void shutdown() { - - } - - @Override - public String getVersionImpl() { - return ""; + public List> getObjectClasses() { + return List.of(TransferredResource.class); } }