diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java index 1282759767..e53ab95d27 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java @@ -1422,6 +1422,7 @@ public enum OrchestratorType { public static final String PLUGIN_PARAMS_FORMAT = "parameter.format"; public static final String PLUGIN_PARAMS_FORMAT_VERSION = "parameter.format_version"; public static final String PLUGIN_PARAMS_PRONOM = "parameter.pronom"; + public static final String PLUGIN_PARAMS_CLEAR_INCIDENCES = "parameter.clear_incidences"; public static final String PLUGIN_CATEGORY_CONVERSION = "conversion"; public static final String PLUGIN_CATEGORY_CHARACTERIZATION = "characterization"; @@ -1572,6 +1573,9 @@ public enum OrchestratorType { public static final String RISK_INCIDENCE_FILE_PATH_COMPUTED_SEPARATOR = "/"; public static final String RISK_INCIDENCE_FILE_EXTENSION = ".json"; + /* Risk Ids */ + public static final String RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING = "urn:siegfried:r1"; + /* Representation information */ public static final String REPRESENTATION_INFORMATION_ID = "id"; public static final String REPRESENTATION_INFORMATION_NAME = "name"; @@ -1702,6 +1706,7 @@ public enum OrchestratorType { /* Siegfriend payload fields */ public static final String SIEGFRIED_PAYLOAD_MATCHES = "matches"; + public static final String SIEGFRIED_PAYLOAD_MATCH_WARNING = "warning"; public static final String SIEGFRIED_PAYLOAD_MATCH_NS = "ns"; public static final String SIEGFRIED_PAYLOAD_MATCH_NS_PRONOM = "pronom"; public static final String SIEGFRIED_PAYLOAD_MATCH_MIMETYPE = "mime"; @@ -1744,6 +1749,7 @@ public enum OrchestratorType { public static final String PRESERVATION_REGISTRY_MIME = "mime"; public static final String PRESERVATION_FORMAT_NOTE_MANUAL = "manual"; + public static final String PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING = "SIEGFRIED WARNING"; public static final String PREMIS_RELATIONSHIP_TYPE_STRUCTURAL = "structural"; public static final String PREMIS_RELATIONSHIP_SUBTYPE_HASPART = "hasPart"; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java index 8935c3492a..a75583cd39 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java @@ -49,7 +49,15 @@ public Filter getFilter() { return filter; } + public void setFilter(Filter filter) { + this.filter = filter; + } + public boolean isOnlyActive() { return onlyActive; } + + public void setOnlyActive(boolean onlyActive) { + this.onlyActive = onlyActive; + } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java b/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java index c5f0a0cfba..87e7d4dc9b 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java @@ -280,6 +280,46 @@ private static ExtensionComplexType getTechnicalMetadata(gov.loc.premis.v3.File return extensionComplexType; } + public static List getFormatNotes(ModelService model, String aipId, String representationId, + List fileDirectoryPath, String fileId, String username) { + Binary premisBin = null; + + try { + try { + premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId); + } catch (NotFoundException e) { + LOGGER.debug("PREMIS object skeleton does not exist yet. Creating PREMIS object!"); + List algorithms = RodaCoreFactory.getFixityAlgorithms(); + + if (fileId == null) { + PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms, + username); + } else { + File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId); + PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username); + } + + premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId); + LOGGER.debug("PREMIS object skeleton created"); + } + gov.loc.premis.v3.File premisFile = binaryToFile(premisBin.getContent(), false); + ObjectCharacteristicsComplexType objectCharacteristics; + if (premisFile.getObjectIdentifier() != null && !premisFile.getObjectIdentifier().isEmpty()) { + objectCharacteristics = premisFile.getObjectCharacteristics().get(0); + if (objectCharacteristics.getFormat() != null && !objectCharacteristics.getFormat().isEmpty()) { + for (FormatComplexType format : objectCharacteristics.getFormat()) { + if (format.getFormatNote() != null) { + return format.getFormatNote(); + } + } + } + } + } catch (RODAException | IOException e) { + LOGGER.error("PREMIS could not be checked due to an error", e); + } + return new ArrayList<>(); + } + public static boolean formatWasManuallyModified(ModelService model, String aipId, String representationId, List fileDirectoryPath, String fileId, String username) { Binary premisBin = null; @@ -1059,14 +1099,6 @@ public static void updateFormatPreservationMetadata(ModelService model, String a PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms, username); } else { - // File file; - // if (shallow) { - // file = model.retrieveFileInsideManifest(aipId, representationId, - // fileDirectoryPath, fileId); - // } else { - // file = model.retrieveFile(aipId, representationId, fileDirectoryPath, - // fileId); - // } File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId); PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username); } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java index eb1409bcae..ab2ce106ca 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java @@ -150,7 +150,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) { for (Representation representation : aip.getRepresentations()) { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, + List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual); if (!newSources.isEmpty()) { sources.addAll(newSources); @@ -180,7 +180,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) { for (Representation representation : aip.getRepresentations()) { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, + List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual); if (!newSources.isEmpty()) { sources.addAll(newSources); @@ -203,7 +203,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic } } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried {}: {}", aip.getId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); @@ -225,7 +225,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic for (Representation representation : filteredList) { try { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, + sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { state = PluginState.SKIPPED; @@ -283,7 +283,8 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St PluginHelper.updatePartialJobReport(this, model, reportItem, false, cachedJob); LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), representation.getAipId()); try { - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, cachedJob.getId(), + sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, + cachedJob.getId(), cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { jobPluginInfo.incrementObjectsProcessedWithSkipped(); @@ -295,7 +296,7 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St model.notifyRepresentationUpdated(representation).failOnError(); } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried {}: {}", representation.getAipId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); @@ -331,7 +332,8 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi file.getAipId()); try { - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnFile(model, file, cachedJob.getUsername(), overwriteManual)); + sources.addAll( + SiegfriedPluginUtils.runSiegfriedOnFile(model, index, file, cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { jobPluginInfo.incrementObjectsProcessedWithSkipped(); reportItem.setPluginState(PluginState.SKIPPED) @@ -341,7 +343,7 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi reportItem.setPluginState(PluginState.SUCCESS); } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried on file {}: {}", file.getId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java index 01bcc38b20..8eb42bf06a 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java @@ -21,18 +21,27 @@ import org.roda.core.RodaCoreFactory; import org.roda.core.common.PremisV3Utils; import org.roda.core.data.common.RodaConstants; +import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.utils.JsonUtils; import org.roda.core.data.v2.IsRODAObject; +import org.roda.core.data.v2.index.filter.Filter; +import org.roda.core.data.v2.index.filter.FilterParameter; +import org.roda.core.data.v2.index.filter.SimpleFilterParameter; import org.roda.core.data.v2.ip.File; import org.roda.core.data.v2.ip.Representation; import org.roda.core.data.v2.ip.StoragePath; import org.roda.core.data.v2.ip.metadata.LinkingIdentifier; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginType; +import org.roda.core.data.v2.risks.IncidenceStatus; +import org.roda.core.data.v2.risks.RiskIncidence; +import org.roda.core.data.v2.risks.SeverityLevel; +import org.roda.core.index.IndexService; +import org.roda.core.index.utils.IterableIndexResult; import org.roda.core.model.ModelService; import org.roda.core.model.utils.ModelUtils; import org.roda.core.plugins.PluginException; @@ -142,9 +151,9 @@ public static String getVersion() { } public static List runSiegfriedOnRepresentation(ModelService model, + IndexService index, Representation representation, String jobId, String username, boolean overwriteManual) throws GenericException, - RequestNotValidException, - NotFoundException, AuthorizationDeniedException, PluginException { + RequestNotValidException, NotFoundException, AuthorizationDeniedException, PluginException, AlreadyExistsException { StoragePath representationDataPath = ModelUtils.getRepresentationDataStoragePath(representation.getAipId(), representation.getId()); StorageService storageService; @@ -154,7 +163,7 @@ public static List runSiegfriedOnRep ModelUtils.getAIPStoragePath(representation.getAipId())); try (DirectResourceAccess directAccess = tmpStorageService.getDirectAccess(representationDataPath)) { Path representationFsPath = directAccess.getPath(); - return runSiegfriedOnRepresentationOrFile(model, representation.getAipId(), representation.getId(), + return runSiegfriedOnRepresentationOrFile(model, index, representation.getAipId(), representation.getId(), new ArrayList<>(), null, representationFsPath, username, overwriteManual); } catch (IOException e) { throw new GenericException(e); @@ -171,7 +180,7 @@ public static List runSiegfriedOnRep } else { try (DirectResourceAccess directAccess = model.getStorage().getDirectAccess(representationDataPath)) { Path representationFsPath = directAccess.getPath(); - return runSiegfriedOnRepresentationOrFile(model, representation.getAipId(), representation.getId(), + return runSiegfriedOnRepresentationOrFile(model, index, representation.getAipId(), representation.getId(), new ArrayList<>(), null, representationFsPath, username, overwriteManual); } catch (IOException e) { throw new GenericException(e); @@ -179,15 +188,15 @@ public static List runSiegfriedOnRep } } - public static List runSiegfriedOnFile(ModelService model, File file, + public static List runSiegfriedOnFile(ModelService model, + IndexService index, File file, String username, boolean overwriteManual) throws GenericException, RequestNotValidException, NotFoundException, - AuthorizationDeniedException, - PluginException { + AuthorizationDeniedException, PluginException, AlreadyExistsException { StoragePath fileStoragePath = ModelUtils.getFileStoragePath(file); try (DirectResourceAccess directAccess = model.getStorage().getDirectAccess(fileStoragePath)) { Path filePath = directAccess.getPath(); - List sources = runSiegfriedOnRepresentationOrFile(model, file.getAipId(), + List sources = runSiegfriedOnRepresentationOrFile(model, index, file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId(), filePath, username, overwriteManual); model.notifyFileUpdated(file).failOnError(); return sources; @@ -197,10 +206,11 @@ public static List runSiegfriedOnFil } private static List runSiegfriedOnRepresentationOrFile(ModelService model, - String aipId, String representationId, List fileDirectoryPath, String fileId, Path path, String username, + IndexService index, String aipId, String representationId, List fileDirectoryPath, String fileId, Path path, + String username, Boolean overwriteManual) - throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException, - PluginException { + throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException, PluginException, + AlreadyExistsException { List sources = new ArrayList<>(); if (FSUtils.exists(path)) { @@ -225,37 +235,95 @@ private static List runSiegfriedOnRe jsonFilePath.remove(jsonFilePath.size() - 1); - if (!PremisV3Utils.formatWasManuallyModified(model, aipId, representationId, jsonFilePath, jsonFileId, username) - || overwriteManual) { - ContentPayload payload = new StringContentPayload(file.toString()); - model.createOrUpdateOtherMetadata(aipId, representationId, jsonFilePath, jsonFileId, - SiegfriedPlugin.FILE_SUFFIX, RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED, payload, username, false); - - sources.add(PluginHelper.getLinkingIdentifier(aipId, representationId, jsonFilePath, jsonFileId, - RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE)); - - // Update PREMIS files - final JsonNode matches = file.get("matches"); - for (JsonNode match : matches) { - String format = null; - String version = null; - String pronom = null; - String mime = null; - - if ("pronom".equalsIgnoreCase(match.get("ns").textValue())) { - format = match.get("format").textValue(); - version = match.get("version").textValue(); - pronom = match.get("id").textValue(); - mime = match.get("mime").textValue(); + JsonNode matches = file.get(RodaConstants.SIEGFRIED_PAYLOAD_MATCHES); + if (matches != null) { + if (!PremisV3Utils.formatWasManuallyModified(model, aipId, representationId, jsonFilePath, jsonFileId, + username) || overwriteManual) { + ContentPayload payload = new StringContentPayload(file.toString()); + model.createOrUpdateOtherMetadata(aipId, representationId, jsonFilePath, jsonFileId, + SiegfriedPlugin.FILE_SUFFIX, RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED, payload, username, false); + + sources.add(PluginHelper.getLinkingIdentifier(aipId, representationId, jsonFilePath, jsonFileId, + RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE)); + + // Update PREMIS files + for (JsonNode match : matches) { + String format = null; + String version = null; + String pronom = null; + String mime = null; + + if ("pronom".equalsIgnoreCase(match.get("ns").textValue())) { + format = match.get("format").textValue(); + version = match.get("version").textValue(); + pronom = match.get("id").textValue(); + mime = match.get("mime").textValue(); + } + + JsonNode warning = match.get(RodaConstants.SIEGFRIED_PAYLOAD_MATCH_WARNING); + List notes = new ArrayList<>(); + if (StringUtils.isNotBlank(warning.textValue())) { + notes.add(RodaConstants.PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING + ": " + warning.textValue()); + updateFileRiskIncidences(model, index, aipId, representationId, fileId, jsonFilePath, + warning.textValue()); + } + PremisV3Utils.updateFormatPreservationMetadata(model, aipId, representationId, jsonFilePath, jsonFileId, + format, version, pronom, mime, notes, username, true); } - - PremisV3Utils.updateFormatPreservationMetadata(model, aipId, representationId, jsonFilePath, jsonFileId, - format, version, pronom, mime, new ArrayList<>(), username, true); } } } } - return sources; } + + private static void updateFileRiskIncidences(ModelService model, IndexService index, String aipId, + String representationId, String fileId, List filePath, String warning) throws RequestNotValidException, + GenericException, AuthorizationDeniedException, AlreadyExistsException, NotFoundException { + // Mitigate previous incidences + for (RiskIncidence incidence : getPreviousSiegfriedIncidences(model, index, fileId)) { + incidence.setStatus(IncidenceStatus.MITIGATED); + model.updateRiskIncidence(incidence, true); + } + // Create a new incidence + RiskIncidence riskIncidence = new RiskIncidence(); + if (fileId != null) { + riskIncidence.setFileId(fileId); + riskIncidence.setFilePath(filePath); + riskIncidence.setObjectClass(File.class.getName()); + } else { + riskIncidence.setObjectClass(Representation.class.getName()); + } + riskIncidence.setRiskId(RodaConstants.RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING); + riskIncidence.setDetectedBy(SiegfriedPlugin.getStaticName()); + riskIncidence.setByPlugin(true); + riskIncidence.setStatus(IncidenceStatus.UNMITIGATED); + riskIncidence.setRepresentationId(representationId); + riskIncidence.setAipId(aipId); + riskIncidence.setDescription(warning); + riskIncidence.setFileId(fileId); + riskIncidence.setSeverity(SeverityLevel.LOW); + model.createRiskIncidence(riskIncidence, true); + } + + public static List getPreviousSiegfriedIncidences(ModelService model, IndexService index, + String fileId) throws RequestNotValidException, GenericException, AuthorizationDeniedException, NotFoundException { + List riskIncidences = new ArrayList<>(); + Filter filter = new Filter(); + List filterParameters = new ArrayList<>(); + filterParameters.add(new SimpleFilterParameter("riskId", RodaConstants.RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING)); + filterParameters.add(new SimpleFilterParameter("fileId", fileId)); + filterParameters.add(new SimpleFilterParameter("status", IncidenceStatus.UNMITIGATED.name())); + filter.add(filterParameters); + try (IterableIndexResult results = index.findAll(RiskIncidence.class, filter, true, + Arrays.asList("id", "status"))) { + for (RiskIncidence incidence : results) { + RiskIncidence modelIncidence = model.retrieveRiskIncidence(incidence.getId()); + riskIncidences.add(modelIncidence); + } + } catch (IOException e) { + LOGGER.error("Error finding file id {}'s associated risk incidences", fileId, e); + } + return riskIncidences; + } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java index be0b086492..cd5f6360fd 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java @@ -28,6 +28,8 @@ import org.roda.core.data.v2.jobs.PluginState; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.data.v2.jobs.Report; +import org.roda.core.data.v2.risks.IncidenceStatus; +import org.roda.core.data.v2.risks.RiskIncidence; import org.roda.core.data.v2.validation.ValidationException; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; @@ -37,6 +39,7 @@ import org.roda.core.plugins.PluginException; import org.roda.core.plugins.PluginHelper; import org.roda.core.plugins.RODAObjectsProcessingLogic; +import org.roda.core.plugins.base.characterization.SiegfriedPluginUtils; import org.roda.core.plugins.orchestrate.JobPluginInfo; import org.roda.core.storage.DirectResourceAccess; import org.roda.core.storage.StorageService; @@ -68,12 +71,20 @@ public class EditFileFormatPlugin extends AbstractPlugin { PluginParameter .getBuilder(RodaConstants.PLUGIN_PARAMS_PRONOM, "PRONOM", PluginParameter.PluginParameterType.STRING) .withDescription("The PRONOM identifier to set for all selected files.").build()); + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES, + PluginParameter + .getBuilder(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES, "Clear Siegfried risk incidences", + PluginParameter.PluginParameterType.BOOLEAN) + .withDescription( + "Have this plugin clear risk incidences caused by Siegfried file format identification warnings.") + .build()); } private String mimetype; private String format; private String formatVersion; private String pronom; + private boolean clearIncidences; public static String getStaticName() { return "Edit File Format"; @@ -115,6 +126,7 @@ public List getParameters() { parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT)); parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT_VERSION)); parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_PRONOM)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)); return parameters; } @@ -125,6 +137,11 @@ public void setParameterValues(Map parameters) throws InvalidPar format = parameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT); formatVersion = parameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT_VERSION); pronom = parameters.get(RodaConstants.PLUGIN_PARAMS_PRONOM); + if (parameters.containsKey(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)) { + clearIncidences = Boolean.parseBoolean(parameters.get(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)); + } else { + clearIncidences = false; + } } @Override @@ -153,7 +170,7 @@ private void processFiles(IndexService index, ModelService model, StorageService } else { List sources = new ArrayList<>(); try { - sources.add(setFileFormatMetadata(model, file, cachedJob.getId(), cachedJob.getUsername())); + sources.add(setFileFormatMetadata(model, index, file, cachedJob.getId(), cachedJob.getUsername())); jobPluginInfo.incrementObjectsProcessedWithSuccess(); reportItem.setPluginState(PluginState.SUCCESS); } catch (RequestNotValidException | NotFoundException | AuthorizationDeniedException | GenericException @@ -199,7 +216,8 @@ private Report validateParameters() { return reportItem; } - private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, String jobId, String username) + private LinkingIdentifier setFileFormatMetadata(ModelService model, IndexService index, File file, String jobId, + String username) throws GenericException, RequestNotValidException, NotFoundException, AuthorizationDeniedException, PluginException { StoragePath fileDataPath = ModelUtils.getFileStoragePath(file); @@ -227,9 +245,11 @@ private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, S String updatedPronomIdentifier = pronom; String updatedMimetype = mimetype; + List notes = mitigatePreviousIncidencesAndCreateNotes(model, index, file.getAipId(), + file.getRepresentationId(), file.getId(), jsonFilePath, username); PremisV3Utils.updateFormatPreservationMetadata(model, file.getAipId(), file.getRepresentationId(), jsonFilePath, jsonFileId, updatedFormat, updatedFormatVersion, updatedPronomIdentifier, updatedMimetype, - Arrays.asList(RodaConstants.PRESERVATION_FORMAT_NOTE_MANUAL), username, true); + notes, username, true); source = PluginHelper.getLinkingIdentifier(file.getAipId(), file.getRepresentationId(), jsonFilePath, jsonFileId, RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE); @@ -240,6 +260,30 @@ private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, S return source; } + private List mitigatePreviousIncidencesAndCreateNotes(ModelService model, IndexService index, String aipId, + String representationId, String fileId, List filePath, String username) + throws AuthorizationDeniedException, RequestNotValidException, NotFoundException, GenericException { + List notes = new ArrayList<>(); + notes.add(RodaConstants.PRESERVATION_FORMAT_NOTE_MANUAL); + List siegfriedRiskIncidences = SiegfriedPluginUtils.getPreviousSiegfriedIncidences(model, index, + fileId); + if (!siegfriedRiskIncidences.isEmpty()) { + if (clearIncidences) { + for (RiskIncidence incidence : siegfriedRiskIncidences) { + incidence.setStatus(IncidenceStatus.MITIGATED); + model.updateRiskIncidence(incidence, true); + } + } else { + for (String note : PremisV3Utils.getFormatNotes(model, aipId, representationId, filePath, fileId, username)) { + if (note.contains(RodaConstants.PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING)) { + notes.add(note); + } + } + } + } + return notes; + } + @Override public RodaConstants.PreservationEventType getPreservationEventType() { return RodaConstants.PreservationEventType.UPDATE; diff --git a/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:drambora:r79.json b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:drambora:r79.json new file mode 100644 index 0000000000..0c3ad3463e --- /dev/null +++ b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:drambora:r79.json @@ -0,0 +1,37 @@ + + { + + + "categories": ["Hardware, software or communications equipment and facilities","Operations and service delivery"], + + + "id": "urn:drambora:r79", + + "name": "Siegried file format identification warning", + + "description": "File format identification via Siegfried has produced warnings.", + + "notes": "PLACEHOLDER NOTES", + + "preMitigationNotes": "PLACEHOLDER PRE-MITIGATION NOTES", + + "mitigationOwner": "Preservation", + + "mitigationStrategy": "Avoidance strategies:\n- Define realistic service levels and implement policies and procedures for their review and adjustment\n- Secure and allocate resources based on business priorities\n- Establish mechanisms to regularly review and if necessary adjust policies and procedures in order to ensure objectives are realised In the event of risk’s execution:\n- Undertake appropriate internal enquiries to determine the shortcomings that led to failure and update policies accordingly ", + + "preMitigationProbability": 4 , + + "preMitigationImpact": 3, + + "preMitigationSeverity": 4, + "preMitigationSeverityLevel": "LOW", + + "identifiedBy":"PLACEHOLDER IDENTIFIED BY", + "mitigationOwnerType": "Human", + "createdBy": "admin", + "createdOn": 1721257200000, + "updatedBy": "admin", + "updatedOn": 1721257200000, + "identifiedOn": "1721257200000" + } + \ No newline at end of file diff --git a/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json new file mode 100644 index 0000000000..ae93a12aa2 --- /dev/null +++ b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json @@ -0,0 +1,24 @@ +{ + "categories": [ + "Content characterization" + ], + "id": "urn:siegfried:r1", + "name": "File format identification issue", + "description": "File format identification process has identified possible issues while identifying file formats.", + "notes": "The file format identification process may have reported warnings during the matching process of the file format identifier signatures. These are not strictly errors but may still warrant further investigation. Common warnings include being unable to detect the file format or a mismatch between the identified file format via identifier signatures and the file name extension.", + "preMitigationNotes": "\n- The repository cannot reliably confirm the technical metadata of the affected files", + "mitigationOwner": "Preservation", + "mitigationStrategy": "Manually review and/or edit the technical metadata of affected files in order to verify the relevance of the produced warnings.", + "preMitigationProbability": 4, + "preMitigationImpact": 3, + "preMitigationSeverity": 12, + "preMitigationSeverityLevel": "MODERATE", + "identifiedBy": "Risk automatically detected by the File Format Identification plugin \"roda.core.plugins.plugins.base.characterization.SiegfriedPlugin\".", + "mitigationOwnerType": "", + "createdBy": "admin", + "createdOn": 1721257200000, + "updatedBy": "admin", + "updatedOn": 1721257200000, + "identifiedOn": "1721257200000" +} + \ No newline at end of file diff --git a/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java b/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java index 8b52957ac4..4ca0a254e8 100644 --- a/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java +++ b/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java @@ -73,7 +73,7 @@ public IndexResult find(@RequestBody FindRequest findRequest, Str } @Override - public LongResponse count(CountRequest countRequest) { + public LongResponse count(@RequestBody CountRequest countRequest) { RequestContext requestContext = RequestUtils.parseHTTPRequest(request); if (UserUtility.hasPermissions(requestContext.getUser(), RodaConstants.PERMISSION_METHOD_FIND_RISK_INCIDENCE)) { return new LongResponse(indexService.count(RiskIncidence.class, countRequest, requestContext));