From e3f1fa5e4eb4da79be9818ff4f8c6d535539183c Mon Sep 17 00:00:00 2001 From: pautri Date: Tue, 12 Dec 2023 11:33:14 +0100 Subject: [PATCH 1/9] EPICify v2 --- pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 6d05a4a..10e6e67 100644 --- a/pom.xml +++ b/pom.xml @@ -38,17 +38,17 @@ fluent-hc 4.5.1 - - com.opencsv - opencsv - 5.2 - + + com.opencsv + opencsv + 5.2 + net.sf.supercsv super-csv 2.4.0 - + eu.clarin.cmdi @@ -89,7 +89,7 @@ nl.knaw.meertens.pid epicify-lib - 1.0 + 2.0-SNAPSHOT ch.qos.logback From b4b90b0a508ac71b1b5fcffc6330b34028507294 Mon Sep 17 00:00:00 2001 From: pautri Date: Tue, 12 Dec 2023 14:09:30 +0100 Subject: [PATCH 2/9] Update config-log.xsl --- src/main/resources/WorkspaceLog/config-log.xsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/WorkspaceLog/config-log.xsl b/src/main/resources/WorkspaceLog/config-log.xsl index 54b94e5..06f06ec 100644 --- a/src/main/resources/WorkspaceLog/config-log.xsl +++ b/src/main/resources/WorkspaceLog/config-log.xsl @@ -15,7 +15,7 @@ /devel.log - true + true return (mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); @@ -29,7 +29,7 @@ /user-log-events.xml - true\n" + + true\n" + \n" + return (level >= INFO && mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); DENY From 0b3c13f313205f6b0da9178f6c136711db4d17a7 Mon Sep 17 00:00:00 2001 From: pautri Date: Thu, 11 Jan 2024 14:39:20 +0100 Subject: [PATCH 3/9] stopping log appenders such that they can be initialised again in a resumed run --- .../deposit/action/WorkspaceLogCleanup.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogCleanup.java b/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogCleanup.java index 790756d..1612959 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogCleanup.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogCleanup.java @@ -17,22 +17,35 @@ package nl.mpi.tla.flat.deposit.action; import nl.mpi.tla.flat.deposit.Context; -import org.slf4j.Logger; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; import org.slf4j.LoggerFactory; import org.slf4j.MDC; +import java.util.Iterator; /** * - * @author menzowi + * @author menzowi, pautri */ public class WorkspaceLogCleanup extends AbstractAction { - - private static final Logger logger = LoggerFactory.getLogger(WorkspaceLogCleanup.class.getName()); - + + private static final LoggerContext ctx = (LoggerContext) LoggerFactory.getILoggerFactory(); + private static final Logger logger = ctx.getLogger("nl.mpi.tla.flat.deposit"); + @Override public boolean perform(Context context) { + + for (Iterator> index = logger.iteratorForAppenders(); index.hasNext();) { + Appender appender = index.next(); + if (appender.getName().equals("USER") || appender.getName().equals("DEVEL")) { + appender.stop(); + } + } + MDC.remove("sip"); return true; } - + } From 531744aa6488ca0fdb6c187ea36ad07105125a75 Mon Sep 17 00:00:00 2001 From: pautri Date: Mon, 15 Jan 2024 10:17:42 +0100 Subject: [PATCH 4/9] Update config-log.xsl "prudent" probably not helpful as we don't have multiple jvms, might even slow things down --- .../resources/WorkspaceLog/config-log.xsl | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/main/resources/WorkspaceLog/config-log.xsl b/src/main/resources/WorkspaceLog/config-log.xsl index 06f06ec..83e04d0 100644 --- a/src/main/resources/WorkspaceLog/config-log.xsl +++ b/src/main/resources/WorkspaceLog/config-log.xsl @@ -14,11 +14,14 @@ - /devel.log - true + + +/devel.log + true - return (mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); + return (mdc.get("sip") != null && ((String)mdc.get("sip")).equals(" +")); DENY ACCEPT @@ -28,23 +31,26 @@ - /user-log-events.xml - true\n" + - \n" + return (level >= INFO && mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); - - DENY - ACCEPT - - - - - - - - - - - - + + +/user-log-events.xml + true\n" + + \n" + return (level >= INFO && mdc.get("sip") != null && ((String)mdc.get("sip")).equals(" +")); + + DENY + ACCEPT + + + + + + + + + + + + From 9f578fb366a3b3bd02b14892c4735f49f7011a5a Mon Sep 17 00:00:00 2001 From: pautri Date: Tue, 28 Jan 2025 16:11:04 +0100 Subject: [PATCH 5/9] attempt to fix issue with logging within threads --- .../tla/flat/deposit/action/ACLUpdate.java | 20 +- .../deposit/action/EPICHandleCreation.java | 52 ++--- .../nl/mpi/tla/flat/deposit/action/FITS.java | 183 ++++++++++-------- .../tla/flat/deposit/action/FOXCreate.java | 38 ++-- .../tla/flat/deposit/action/FedoraAction.java | 18 +- .../tla/flat/deposit/action/FedoraDelete.java | 2 +- .../flat/deposit/action/FedoraInteract.java | 2 +- .../action/FedoraLoadCollectionHierarchy.java | 22 +-- .../tla/flat/deposit/action/FedoraUser.java | 20 +- .../nl/mpi/tla/flat/deposit/action/Mail.java | 23 +-- .../tla/flat/deposit/action/PurgeUpdates.java | 32 +-- .../deposit/action/UpdateCollections.java | 32 +-- .../deposit/action/WorkspaceLogSetup.java | 31 ++- 13 files changed, 241 insertions(+), 234 deletions(-) diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/ACLUpdate.java b/src/main/java/nl/mpi/tla/flat/deposit/action/ACLUpdate.java index 1d8b757..9850977 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/ACLUpdate.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/ACLUpdate.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2018 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -46,17 +46,17 @@ * @author menzowi */ public class ACLUpdate extends FedoraAction { - + private static final Logger logger = LoggerFactory.getLogger(ACLUpdate.class.getName()); - + private static XsltTransformer strip = null; @Override public boolean perform(Context context) throws DepositException { try { - + String namespace = context.getProperty("activeFedoraNamespace", "lat").toString(); - + connect(context); File dir = new File(getParameter("dir","./acl")); @@ -65,7 +65,7 @@ public boolean perform(Context context) throws DepositException { } SIPInterface sip = context.getSIP(); - + // check owner.xml File owner = new File(dir + "/owner.xml"); if (!owner.exists()) { @@ -93,9 +93,9 @@ public boolean perform(Context context) throws DepositException { own.transform(); Saxon.save(destination,owner); } - + // check POLICY and RELS-EXT - if (sip.isUpdate()) { + if (sip.isUpdate()) { this.check(dir,sip.getFID(true).toString(),"AIP"); for (Resource res:sip.getResources()) { if (res.isInsert()) { @@ -120,7 +120,7 @@ else if (res.hasPID()) { } return true; } - + void check(File dir,String fid,String tpe) throws Exception { // POLICY File policy = new File(dir + "/"+fid.replaceAll("[^a-zA-Z0-9]", "_")+".xml"); @@ -169,5 +169,5 @@ void check(File dir,String fid,String tpe) throws Exception { } } } - + } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/EPICHandleCreation.java b/src/main/java/nl/mpi/tla/flat/deposit/action/EPICHandleCreation.java index d39b333..61597bc 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/EPICHandleCreation.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/EPICHandleCreation.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -43,17 +43,17 @@ * @author pavsri */ public class EPICHandleCreation extends AbstractAction { - + private static final Logger logger = LoggerFactory.getLogger(EPICHandleCreation.class.getName()); @Override public boolean perform(Context context) throws DepositException { - + try { - + String namespace = context.getProperty("activeFedoraNamespace", "lat").toString(); XdmValue namespaces = context.getProperty("fedoraNamespace", "lat"); - + String fedora = this.getParameter("fedoraConfig"); String epic = this.getParameter("epicConfig"); @@ -89,13 +89,13 @@ public boolean perform(Context context) throws DepositException { return false; } logger.debug("EPIC configuration["+config.getAbsolutePath()+"]"); - + XMLConfiguration xConfig = new XMLConfiguration(config); - + boolean isTest = xConfig.getString("status") != null && xConfig.getString("status").equals("test"); PIDService ps = new PIDService(xConfig, null); - + if (context.getSIP().hasPID() && context.getSIP().hasFID()) { String fid = context.getSIP().getFID().toString().replaceAll("#.*",""); @@ -134,13 +134,13 @@ public boolean perform(Context context) throws DepositException { String pid = col.getPID().toString().replaceAll("^http(s?)://hdl.handle.net/","hdl:"); String prefix = pid.replaceAll("hdl:([^/]*)/.*","$1"); String uuid = pid.replaceAll(".*/",""); - + String loc = server+"/objects/"+fid+"/datastreams/"+dsid+"/content?asOfDateTime="+asof; logger.info("Lookup handle["+prefix+"/"+uuid+"]"); String cur = (isTest?null:ps.getPIDLocation(prefix+"/"+uuid)); logger.info("Looked up handle["+prefix+"/"+uuid+"] -> URI["+cur+"]"); - + if (cur == null) { logger.info("Create handle["+pid+"]["+uuid+"] -> URI["+loc+"]"); context.registerRollbackEvent(this, "epic creation", "uuid", uuid, "loc", loc); @@ -188,7 +188,7 @@ public boolean perform(Context context) throws DepositException { } } } - + Map pids = context.getPIDs(); for (URI pid:pids.keySet()) { URI red = pids.get(pid); @@ -214,7 +214,7 @@ public boolean perform(Context context) throws DepositException { } if (c) continue; - + String pidStr = pid.toString().replaceAll("^http(s?)://hdl.handle.net/","hdl:"); String prefix = pidStr.replaceAll("hdl:([^/]*)/.*","$1"); String uuid = pidStr.replaceAll(".*/",""); @@ -222,7 +222,7 @@ public boolean perform(Context context) throws DepositException { logger.info("Lookup handle["+prefix+"/"+uuid+"]"); String cur = (isTest?null:ps.getPIDLocation(prefix+"/"+uuid)); logger.info("Looked up handle["+prefix+"/"+uuid+"] -> URI["+cur+"]"); - + if (cur == null) { logger.info("Create handle["+pid+"]["+uuid+"] -> URI["+red+"]"); context.registerRollbackEvent(this, "epic creation", "uuid", uuid, "loc", red.toString()); @@ -238,7 +238,7 @@ public boolean perform(Context context) throws DepositException { } catch(Exception e) { throw new DepositException(e); } - + return true; } public void rollback(Context context, List events) { @@ -248,7 +248,7 @@ public void rollback(Context context, List events) { try { epic = this.getParameter("epicConfig"); File config = new File(epic); - + if (!config.exists()) { logger.error("The EPIC configuration["+epic+"] doesn't exist!"); return; @@ -259,16 +259,16 @@ public void rollback(Context context, List events) { logger.error("The EPIC configuration["+epic+"] can't be read!"); return; } - + logger.debug("EPIC configuration["+config.getAbsolutePath()+"]"); - + XMLConfiguration xConfig = new XMLConfiguration(config); - + boolean isTest = xConfig.getString("status") != null && xConfig.getString("status").equals("test"); String tombstone = xConfig.getString("tombstone"); - + PIDService ps = new PIDService(xConfig, null); - + for (ListIterator iter = events.listIterator(events.size()); iter.hasPrevious();) { XdmItem event = iter.previous(); try { @@ -276,7 +276,7 @@ public void rollback(Context context, List events) { if (tpe.equals("epic creation")) { String uuid = Saxon.xpath2string(event, "param[@name='uuid']/@value"); String loc = Saxon.xpath2string(event, "param[@name='loc']/@value"); - + if(delMode){ try { ps.deleteHandle(uuid); @@ -287,7 +287,7 @@ public void rollback(Context context, List events) { logger.info("Rollback for deleting the handle[" + uuid + "] for event[" + tpe + "]not possible. Hence opting for tombstone"); } } - + if(!delMode) { ps.updateLocation(uuid, tombstone); logger.debug("rollback action[" + this.getName() + "] event[" + tpe + "] updated handle [" + uuid + "]" + " to " + tombstone); @@ -297,7 +297,7 @@ else if (tpe.equals("epic Update")){ String uuid = Saxon.xpath2string(event, "param[@name='uuid']/@value"); String loc = Saxon.xpath2string(event, "param[@name='loc']/@value"); String cur = Saxon.xpath2string(event, "param[@name='cur']/@value"); - + ps.updateLocation(uuid, cur); logger.debug("rollback action[" + this.getName() + "] event[" + tpe + "] updated handle [" + uuid + "]" + " to " + cur); } @@ -308,14 +308,14 @@ else if (tpe.equals("epic Update")){ logger.error("rollback action[" + this.getName() + "] event[" + event + "] failed!", ex); } } - + } catch (Exception e) { logger.error("rollback action[" + this.getName() + " failed!", e); } - - + + } } } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java index eb83a93..0f92c75 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -17,10 +17,7 @@ package nl.mpi.tla.flat.deposit.action; import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; @@ -28,6 +25,7 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -41,19 +39,14 @@ import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.IntConsumer; -import java.util.stream.IntStream; -import javax.swing.text.Document; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.Source; import javax.xml.transform.stream.StreamSource; import net.sf.saxon.s9api.SaxonApiException; import net.sf.saxon.s9api.XdmAtomicValue; import net.sf.saxon.s9api.XdmItem; import net.sf.saxon.s9api.XdmNode; +import net.sf.saxon.s9api.XdmValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,7 +99,7 @@ public boolean perform(Context context) throws DepositException { threadLimit = 1; // default number of fits thread } if (hasParameter("waitLimit")) { - waitLimit = Integer.valueOf(getParameter("threadLimit")); + waitLimit = Integer.valueOf(getParameter("waitLimit")); } else { waitLimit = 15; // default waiting of fits thread } @@ -118,7 +111,7 @@ public boolean perform(Context context) throws DepositException { } URL fitsURL = null; try { - fitsURL = new URL(fitsService); + fitsURL = URI.create(fitsService).toURL(); } catch (MalformedURLException ex) { throw new DepositException(ex); } @@ -144,7 +137,7 @@ public boolean perform(Context context) throws DepositException { logger.debug("resource[" + file + "] mimetype?"); result = null; try { - URL call = new URL(fitsURL, "examine?file=" + file.getAbsolutePath()); + URL call = fitsURL.toURI().resolve("examine?file=" + file.getAbsolutePath()).toURL(); if (threadCounter <= threadLimit) { future = obj.submit(() -> { threadCounter = threadCounter + 1; @@ -239,12 +232,18 @@ public boolean perform(Context context) throws DepositException { try { // loop over /mimetypes/mimetype boolean bCheck1 = false; // tells if a mimetype was found for the resource + ThreadLocal threadLogger = new ThreadLocal() { + @Override + protected Logger initialValue() { + return LoggerFactory.getLogger(FITS.class); + } + }; for (Iterator iter = Saxon.xpathIterator(mimetypes, "/mimetypes/mimetype", null, NAMESPACES); iter.hasNext();) { XdmItem mt = iter.next(); String mime = Saxon.xpath2string(mt, "normalize-space(@value)"); - logger.debug(". . mimetype[" + mime + "] check"); - Boolean bCheck2 = new Boolean(true); // tells if all assertion groups succeeded + threadLogger.get().debug(". . mimetype[" + mime + "] check"); + Boolean bCheck2 = Boolean.TRUE; // tells if all assertion groups succeeded if (Saxon.xpath2boolean(mt, "exists(assertions)")) { for (Iterator iter2 = Saxon.xpathIterator(mt, "assertions", null, NAMESPACES); iter2.hasNext();) { @@ -255,18 +254,18 @@ public boolean perform(Context context) throws DepositException { // no xpath, so fall back to the default xpath xp = MIMETYPE_XPATH; } - logger.debug(". . . assertions[" + xp + "] check"); + threadLogger.get().debug(". . . assertions[" + xp + "] check"); // evaluate xpath - Map vars = new HashMap(); + Map vars = new HashMap<>(); vars.put("mime", new XdmAtomicValue(mime)); if (!Saxon.xpath2boolean(result, xp, vars, NAMESPACES)) { // the assertions XPath failed, continue to the next // /mimetypes/mimetype - logger.debug(". . . assertions[" + xp + "] check failed"); + threadLogger.get().debug(". . . assertions[" + xp + "] check failed"); bCheck2 = null; break; } - logger.debug(". . . assertions[" + xp + "] check succeeded"); + threadLogger.get().debug(". . . assertions[" + xp + "] check succeeded"); // the assertions XPath succeeded, check the assertions for this // mimetype Boolean bCheck3 = true; // tells if all mimetype assertions succeeded @@ -281,18 +280,18 @@ public boolean perform(Context context) throws DepositException { bCheck3 = Saxon.xpath2boolean(result, axp, null, NAMESPACES); if (!bCheck3) { // assertion fails, print the AVT log message - logger.debug(". . . . assert[" + axp + "] check failed"); - logger.error( + threadLogger.get().debug(". . . . assert[" + axp + "] check failed"); + threadLogger.get().error( "File '{}' has a mimetype '{}' which is ALLOWED in this repository, but fails an assertion!", file, mime); - logger.error("Message from FITS file: " + Saxon.avt( + threadLogger.get().error("Message from FITS file: " + Saxon.avt( Saxon.xpath2string(a, "@message"), result, context.getProperties(), NAMESPACES)); // break out of the assertion loop break; } // assertion is positive, go to next - logger.debug(". . . . assert[" + axp + "] check succeeded"); + threadLogger.get().debug(". . . . assert[" + axp + "] check succeeded"); } else { // the assertion xpath does not exist throw new DepositException( @@ -303,11 +302,11 @@ public boolean perform(Context context) throws DepositException { } if (!bCheck3) { // some assertion of this assertions failed - logger.debug(". . . assertions[" + xp + "] failed"); - bCheck2 = new Boolean(false); + threadLogger.get().debug(". . . assertions[" + xp + "] failed"); + bCheck2 = Boolean.FALSE; break; } else - logger.debug(". . . assertions[" + xp + "] succeeded"); + threadLogger.get().debug(". . . assertions[" + xp + "] succeeded"); } } else { // no assertions, use just the path @@ -317,41 +316,41 @@ public boolean perform(Context context) throws DepositException { xp = MIMETYPE_XPATH; } // evaluate xpath - Map vars = new HashMap(); + Map vars = new HashMap<>(); vars.put("mime", new XdmAtomicValue(mime)); if (!Saxon.xpath2boolean(result, xp, vars, NAMESPACES)) { // the assertions XPath failed, continue to the next /mimetypes/mimetype - logger.debug(". . . assertions[" + xp + "] check failed"); + threadLogger.get().debug(". . . assertions[" + xp + "] check failed"); bCheck2 = null; continue; } - logger.debug(". . . assertions[" + xp + "] check succeeded"); + threadLogger.get().debug(". . . assertions[" + xp + "] check succeeded"); } if (bCheck2 == null) { - logger.debug(". . continue to next mimetype"); + threadLogger.get().debug(". . continue to next mimetype"); continue; } if (bCheck2.booleanValue()) { // all assertions succeeded - logger.debug(". . mimetype[" + mime + "] succeeded"); + threadLogger.get().debug(". . mimetype[" + mime + "] succeeded"); bCheck1 = true; - logger.info( + threadLogger.get().info( "Resource[{}] has a mimetype which is ALLOWED in this repository and satisfies all assertions: '{}'", file, mime); if (resource.hasMime() && !resource.getMime().equals(mime)) { logger.warn("Resource mimetype changed from '{}' to '{}'", resource.getMime(), mime); } - logger.debug("Setting resource mimetype to '{}'", mime); + threadLogger.get().debug("Setting resource mimetype to '{}'", mime); resource.setMime(mime); } else - logger.debug(". . mimetype[" + mime + "] failed"); + threadLogger.get().debug(". . mimetype[" + mime + "] failed"); break; } if (!bCheck1) { // no mimetype was found, look for the otherwise - logger.debug(". mimetypes failed, checking otherwise"); + threadLogger.get().debug(". mimetypes failed, checking otherwise"); XdmItem o = Saxon.xpathSingle(mimetypes, "/mimetypes/otherwise"); if (o != null) { // check for an xpath @@ -362,26 +361,26 @@ public boolean perform(Context context) throws DepositException { if (!fallback.equals("")) { // use the non-empty fallback value as mimetype for the resource bCheck1 = true; - logger.error("Use fallback mimetype[{}] for resource[{}]", fallback, + threadLogger.get().error("Use fallback mimetype[{}] for resource[{}]", fallback, file); if (resource.hasMime() && !resource.getMime().equals(fallback)) { - logger.warn("Resource mimetype changed from '{}' to '{}'", + threadLogger.get().warn("Resource mimetype changed from '{}' to '{}'", resource.getMime(), fallback); } - logger.debug("Setting resource mimetype to '{}'", fallback); + threadLogger.get().debug("Setting resource mimetype to '{}'", fallback); resource.setMime(fallback); } } } else - logger.debug(". mimetypes failed, no otherwise"); + threadLogger.get().debug(". mimetypes failed, no otherwise"); if (!bCheck1) { // no allowed or fallback mimetype was found for this resource - logger.debug(". mimetypes failed"); - logger.error("No mimetype found for resource[{}]", file); + threadLogger.get().debug(". mimetypes failed"); + threadLogger.get().error("No mimetype found for resource[{}]", file); unallowed++; } else - logger.debug(". mimetypes succeeded"); + threadLogger.get().debug(". mimetypes succeeded"); } } catch (Exception ex) { throw new DepositException(ex); @@ -400,37 +399,45 @@ public boolean perform(Context context) throws DepositException { logger.debug("ERROR in one of the threads! Abort execution! (isAnyError) --> " + isAnyError); } } - // Check if all the threads are completed - Start - while (threadCounter > 0) { - try { - logger.debug("Threadcounter > 0 ---> " + threadCounter + " > 0"); - logger.debug("Wait for all threads to finish (" + waitLimit + " seconds). . . . . . . . . . . . . . . ."); - executor.awaitTermination(waitLimit, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new DepositException(e); + // Initiate an orderly shutdown + executor.shutdown(); + + // Wait for all threads to complete or timeout + boolean terminated; + try { + terminated = executor.awaitTermination(waitLimit, TimeUnit.SECONDS); + if (!terminated) { + logger.error("Executor did not terminate in {} seconds", waitLimit); + executor.shutdownNow(); // Force shutdown + throw new DepositException("Thread execution timed out"); } + } catch (InterruptedException e) { + throw new DepositException("Thread execution was interrupted", e); } + // Check results from all futures for (Future fut : list) { try { - // because Future.get() waits for task to get completed - logger.info(new Date() + "::" + fut.get()); + synchronized (FITS.class) { + Integer threadResult = fut.get(); // Get results or exception + logger.info(new Date() + "::Thread completed with counter=" + threadResult); + } } catch (InterruptedException | ExecutionException e) { - throw new DepositException("Exception occurred from thread with msg: " + e); + logger.error("Thread execution failed: " + e.getMessage()); } } - logger.debug("Execution of all threads successfully completed! We can now safely close the executor."); - // Check if all the threads are completed - End + logger.debug("All threads completed successfully"); executor.shutdown(); logger.debug("Executor shutdown done!"); - if (unallowed > 0) + if (unallowed > 0) { logger.error("{} resources were not allowed!", unallowed); + } return (unallowed == 0); } - public class TaskLimitSemaphore { + class TaskLimitSemaphore { private final ExecutorService executor; private final Semaphore semaphore; @@ -439,37 +446,45 @@ public TaskLimitSemaphore(ExecutorService executor, int limit) { this.executor = executor; this.semaphore = new Semaphore(limit); } - public Future submit(final Callable task) throws Exception { Future future = null; - logger.debug("isAnyError Boolean value inside thread: "+isAnyError); - if (isAnyError) { - throw new Exception("ERROR! So don't acquire anymore threads!"); - } else { - semaphore.acquire(); - logger.debug("semaphore.acquire()..."); - } - try { - future = executor.submit(() -> { - try { - return task.call(); - } finally { - semaphore.release(); - logger.debug("semaphore.release()..."); - threadCounter--; - logger.debug("Thread counter decreased to --> " + threadCounter + " . . . . . . ."); - + synchronized (FITS.class) { + logger.debug("isAnyError Boolean value inside thread: " + isAnyError); + if (isAnyError) { + throw new Exception("ERROR! So don't acquire anymore threads!"); + } + try { + semaphore.acquire(); + logger.debug("semaphore.acquire()..."); + future = executor.submit((Callable) () -> { + try { + return task.call(); + } catch (Exception e) { + synchronized (FITS.class) { + isAnyError = true; + logger.debug("set isAnyError TRUE"); + } + threadCounter--; + logger.debug("ERROR: Thread counter decreased to --> " + threadCounter + " . . . . . . ."); + throw new Exception("ERROR occurred in thread! -> " + e); + } finally { + semaphore.release(); + logger.debug("semaphore.release()..."); + synchronized (FITS.class) { + threadCounter--; + logger.debug("Thread counter decreased to --> " + threadCounter + " . . . . . . ."); + } + } + }); + } catch (Exception e) { + synchronized (FITS.class) { + isAnyError = true; + logger.debug("set isAnyError TRUE"); } - }); - } catch (Exception e) { - isAnyError = true; - semaphore.release(); - logger.debug("ERROR: semaphore.release()..."); - threadCounter--; - logger.debug("ERROR: Thread counter decreased to --> " + threadCounter + " . . . . . . ."); - throw new Exception("ERROR occurred in thread! -> " + e); + throw e; + } } return future; } + } } -} diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FOXCreate.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FOXCreate.java index 6a3d684..2cfaa1d 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FOXCreate.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FOXCreate.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -56,10 +56,10 @@ public class FOXCreate extends AbstractAction { public boolean perform(Context context) throws DepositException { URIResolver org = Saxon.getXsltCompiler().getURIResolver(); try { - + String namespace = context.getProperty("activeFedoraNamespace", "lat").toString(); XdmValue namespaces = context.getProperty("fedoraNamespace", "lat"); - + // check for the user profile File owner = new File(getParameter("owner")); if (!owner.exists()) { @@ -73,7 +73,7 @@ public boolean perform(Context context) throws DepositException { return false; } XMLConfiguration profile = new XMLConfiguration(owner); - + File fedora = new File(getParameter("fedoraConfig")); if (!fedora.exists()) { logger.error("The Fedora configuration["+fedora+"] doesn't exist!"); @@ -86,24 +86,24 @@ public boolean perform(Context context) throws DepositException { return false; } logger.debug("Fedora configuration["+fedora.getAbsolutePath()+"]"); - + File dir = new File(getParameter("dir","./fox")); if (!dir.exists()) FileUtils.forceMkdir(dir); - + if (hasParameter("jar_cmd2fox")) { Saxon.getXsltCompiler().setURIResolver(new JarURIResolver(org,new File(getParameter("jar_cmd2fox")))); } - + File xsl = new File(getParameter("cmd2fox")); XsltExecutable cmd2fox = Saxon.buildTransformer(xsl); - + XsltTransformer fox = cmd2fox.load(); SaxonListener listener = new SaxonListener("FOXCreate",MDC.get("sip")); fox.setMessageListener(listener); fox.setErrorListener(listener); - + // fixed parameters fox.setParameter(new QName("owner"), new XdmAtomicValue(profile.getString("name"))); fox.setParameter(new QName("fox-base"), new XdmAtomicValue(dir.toString())); @@ -111,7 +111,7 @@ public boolean perform(Context context) throws DepositException { fox.setParameter(new QName("repository"), new XdmAtomicValue((new XMLConfiguration(fedora)).getString("publicServer"))); fox.setParameter(new QName("namespace"), new XdmAtomicValue(namespace)); fox.setParameter(new QName("namespaces"), namespaces); - + // optional parameters if (hasParameter("management")) fox.setParameter(new QName("management-dir"), params.get("management")); @@ -133,19 +133,19 @@ public boolean perform(Context context) throws DepositException { fox.setParameter(new QName("license-uri"), params.get("license-uri")); if (hasParameter("overwrite-resource-label")) fox.setParameter(new QName("overwrite-resource-label"),new XdmAtomicValue(getParameter("overwrite-resource-label").toLowerCase().contains("t"))); - + // additional parameters for (String param:this.params.keySet()) { if (param.startsWith("xsl-param-")) fox.setParameter(new QName(param.replaceFirst("^xsl-param-","")), params.get(param)); } - + // go fox.setSource(new DOMSource(context.getSIP().getRecord(),context.getSIP().getBase().toURI().toString())); XdmDestination destination = new XdmDestination(); fox.setDestination(destination); fox.transform(); - + String fid = Saxon.xpath2string(destination.getXdmNode(),"/*/@PID").replaceAll("#.*","").replaceAll("[^a-zA-Z0-9]", "_"); File out = new File(dir + "/"+fid+"_CMD.xml"); if (out.exists()) { @@ -153,7 +153,7 @@ public boolean perform(Context context) throws DepositException { } TransformerFactory.newInstance().newTransformer().transform(destination.getXdmNode().asSource(),new StreamResult(out)); logger.info("created FOX["+out.getAbsolutePath()+"]"); - + XdmNode cmd = Saxon.buildDocument(new StreamSource(new File(dir+"/"+fid+"_CMD.xml"))); XdmItem self = Saxon.xpathSingle(cmd,"//cmd:CMD/cmd:Header/cmd:MdSelfLink",null,NAMESPACES); if (self!=null) { @@ -179,23 +179,23 @@ public boolean perform(Context context) throws DepositException { } return true; } - + static class JarURIResolver implements URIResolver { - + private URIResolver resolver = null; private File xsl = null; - + public JarURIResolver(URIResolver resolver, File xsl) { this.resolver = resolver; this.xsl = xsl; } - + public Source resolve(String href,String base) throws TransformerException { if (href.equals("jar:cmd2fox.xsl") && xsl!=null) { return new javax.xml.transform.stream.StreamSource(this.xsl); } else { return resolver.resolve(href,base); } - } + } } } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraAction.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraAction.java index ab2100a..026a96e 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraAction.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraAction.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -36,14 +36,14 @@ abstract public class FedoraAction extends AbstractAction { private static final Logger logger = LoggerFactory.getLogger(FedoraAction.class.getName()); - + private static String user = null; - + private XMLConfiguration fedoraConfig = null; public void connect(Context context) throws DepositException { try { - fedoraConfig = new XMLConfiguration(new File(getParameter("fedoraConfig"))); + fedoraConfig = new XMLConfiguration(new File(getParameter("fedoraConfig"))); String user = fedoraConfig.getString("userName"); if (!FedoraRequest.isDefaultClientSet() || this.user==null || this.user!=user ) { @@ -59,11 +59,11 @@ public void connect(Context context) throws DepositException { throw new DepositException("Connecting to Fedora Commons failed!",e); } } - + public String getFedoraUser() { return this.user; } - + public URI lookupFID(URI pid) throws DepositException { URI fid = null; try { @@ -83,7 +83,7 @@ public URI lookupFID(URI pid) throws DepositException { } return fid; } - + public URI lookupPID(URI fid) throws DepositException { URI pid = null; try { @@ -103,7 +103,7 @@ public URI lookupPID(URI fid) throws DepositException { } return pid; } - + public Date lookupAsOfDateTime(URI fid) throws DepositException { try { return getObjectProfile(fid.toString().replaceAll("#.*","")).execute().getLastModifiedDate(); @@ -111,5 +111,5 @@ public Date lookupAsOfDateTime(URI fid) throws DepositException { throw new DepositException("Connecting to Fedora Commons failed!",e); } } - + } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraDelete.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraDelete.java index 3059ad3..65a1955 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraDelete.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraDelete.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraInteract.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraInteract.java index fea637e..01a381e 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraInteract.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraInteract.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraLoadCollectionHierarchy.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraLoadCollectionHierarchy.java index 83e7c87..2cf414a 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraLoadCollectionHierarchy.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraLoadCollectionHierarchy.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -46,17 +46,17 @@ * @author pavsri */ public class FedoraLoadCollectionHierarchy extends FedoraAction { - + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FedoraLoadCollectionHierarchy.class.getName()); - + @Override - public boolean perform(Context context) throws DepositException { + public boolean perform(Context context) throws DepositException { try { connect(context); - + String namespace = context.getProperty("activeFedoraNamespace", "lat").toString(); XdmValue namespaces = context.getProperty("fedoraNamespace", "lat"); - + SIPInterface sip = context.getSIP(); if (sip.hasCollections()) { // check if known collections are complete @@ -115,10 +115,10 @@ public boolean perform(Context context) throws DepositException { throw ex; } catch (Exception ex) { throw new DepositException(ex); - } + } return true; } - + private void loadParentCollections(Deque hist,Collection col, String namespace, XdmValue namespaces) throws Exception { // fetch parent collections String sparql = "SELECT ?fid WHERE { ?fid } "; @@ -155,7 +155,7 @@ private void loadParentCollections(Deque hist,Collection col, String namesp } } } - + protected void completeFID(Collection col) throws DepositException { try { if (col!=null) { @@ -164,7 +164,7 @@ protected void completeFID(Collection col) throws DepositException { fid = col.getFID(true); else if (col.hasPID()) fid = this.lookupFID(col.getPID()); - else + else throw new DepositException("Unknown Collection["+col+"]!"); if (hasCMDDatastream(fid)) { Date asof = getObjectProfile(fid.toString()).execute().getLastModifiedDate(); @@ -176,7 +176,7 @@ else if (col.hasPID()) throw new DepositException("Completing the FID of Collection["+col+"] failed!",e); } } - + protected boolean hasCMDDatastream(URI fid) throws DepositException { try { for(DatastreamProfile p:getDatastreams(fid.toString()).execute().getDatastreamProfiles()) { diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraUser.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraUser.java index 9a7093c..48d55d5 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraUser.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FedoraUser.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -80,7 +80,7 @@ public boolean perform(Context context) throws DepositException { logger.error("The fedora config can't be read!"); return false; } - + // check for the user-specific Fedora config File usr = new File(getParameter("userFedoraConfig")); if (usr.exists() && !usr.isFile()) { @@ -90,26 +90,26 @@ public boolean perform(Context context) throws DepositException { logger.error("The user-specific fedora config can't be written!"); return false; } - + // get the user and the temp pass XMLConfiguration userProfile = new XMLConfiguration(user); //String pass = UUID.randomUUID().toString(); - + // export the php script to a tmp dir File tmp = Files.createTempDir(); - File php = tmp.toPath().resolve("./pass.php").toFile(); + File php = tmp.toPath().resolve("./pass.php").toFile(); FileUtils.copyURLToFile(FedoraUser.class.getResource("/FedoraUser/pass.php"), php); // exec the drush script ProcessBuilder pb = new ProcessBuilder(getParameter("drush"), "php-script", "pass", "--script-path="+tmp, userProfile.getString("name")/*, pass*/); pb.directory(new File(getParameter("drupal"))); pb.redirectErrorStream(true); - + Process process = pb.start(); InputStream is = process.getInputStream(); InputStreamReader isr = new InputStreamReader(is); BufferedReader br = new BufferedReader(isr); - + logger.debug("Setting up the user-specific access to fedora:"); String pass = null; String line; @@ -129,7 +129,7 @@ public boolean perform(Context context) throws DepositException { logger.error("Failed to set up the user-specific access to fedora! No pass :-("); res = false; } - + // clean up the tmp dir php.delete(); tmp.delete(); @@ -145,9 +145,9 @@ public boolean perform(Context context) throws DepositException { XdmDestination destination = new XdmDestination(); embed.setDestination(destination); embed.transform(); - + Saxon.save(destination.getXdmNode().asSource(),usr); - + } catch (Exception e) { throw new DepositException("Setting up the user-specific access to fedora failed!", e); } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/Mail.java b/src/main/java/nl/mpi/tla/flat/deposit/action/Mail.java index 76250fd..2739c11 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/Mail.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/Mail.java @@ -25,27 +25,20 @@ import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import javax.xml.transform.dom.DOMSource; - -import java.io.File; -import java.net.URI; - import javax.xml.transform.stream.StreamSource; - +import java.io.File; import net.sf.saxon.s9api.QName; import net.sf.saxon.s9api.XdmAtomicValue; import net.sf.saxon.s9api.XdmDestination; import net.sf.saxon.s9api.XdmNode; -import net.sf.saxon.s9api.XdmValue; import net.sf.saxon.s9api.XsltExecutable; import net.sf.saxon.s9api.XsltTransformer; import nl.mpi.tla.flat.deposit.Context; import nl.mpi.tla.flat.deposit.DepositException; -import nl.mpi.tla.flat.deposit.sip.SIPInterface; import nl.mpi.tla.flat.deposit.util.Saxon; import nl.mpi.tla.flat.deposit.util.SaxonListener; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.jena.ext.com.google.common.base.Throwables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -79,9 +72,9 @@ public boolean perform(Context context) throws DepositException { XdmNode mailNode = (XdmNode) Saxon.xpathSingle(mailNodeConfig, "/mailConfig"); String sendWhenSuccess = Saxon.xpath2string(mailNode, "./sendWhenSuccess"); - logger.debug("sendWhenSuccess[" + sendWhenSuccess + "]"); - String sendOnFailedValidation = Saxon.xpath2string(mailNode, "./sendOnFailedValidation"); - logger.debug("sendOnFailedValidation[" + sendOnFailedValidation + "]"); + logger.debug("sendWhenSuccess[" + sendWhenSuccess + "]"); + String sendOnFailedValidation = Saxon.xpath2string(mailNode, "./sendOnFailedValidation"); + logger.debug("sendOnFailedValidation[" + sendOnFailedValidation + "]"); String server = Saxon.xpath2string(mailNode, "./server"); logger.debug("Server[" + server + "]"); String port = Saxon.xpath2string(mailNode, "./port"); @@ -130,7 +123,7 @@ public boolean perform(Context context) throws DepositException { logger.info("No email sent! (mail-config.xml has value false) "); return true; } - + if (context.getFlow().getStop()!=null) { if (!swordStatus.booleanValue()) { //Validation failed outcome = "FAILED"; @@ -155,9 +148,9 @@ public boolean perform(Context context) throws DepositException { logger.info("But Code stops: " + context.getFlow().getStop()); logger.info("No email sent! (this was only a partial DoorKeeper run) "); return true; - } + } } - + if (swordStatus == null || context.hasException()) { outcome = "FAILED"; logger.info("Email sent! Normal Doorkeeper run with outcome: "+ outcome); @@ -165,7 +158,7 @@ public boolean perform(Context context) throws DepositException { String stackTrace = ExceptionUtils.getFullStackTrace(context.getException()); fox.setParameter(new QName("stacktrace"), new XdmAtomicValue(stackTrace)); } - + if(outcome.equals("SUCCESS")) { logger.info("Email sent! Normal Doorkeeper run with outcome: "+ outcome); if (context.getSIP().hasPID()) { diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/PurgeUpdates.java b/src/main/java/nl/mpi/tla/flat/deposit/action/PurgeUpdates.java index f1d7d1a..91e490d 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/PurgeUpdates.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/PurgeUpdates.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -50,13 +50,13 @@ public class PurgeUpdates extends FedoraAction { public boolean perform(Context context) throws DepositException { try { - + // connect to Fedora connect(context); - + String ext = getParameter("ext",".FITS.xml"); String xp = getParameter("path","normalize-space(//fits:md5checksum)"); - + Path dir = Paths.get(getParameter("dir","./fits")).toAbsolutePath(); if (!dir.toFile().exists()) throw new DepositException("directory["+dir+"] doesn't exist!"); @@ -64,15 +64,15 @@ public boolean perform(Context context) throws DepositException { throw new DepositException("directory["+dir+"] isn't a directory!"); if (!dir.toFile().canRead()) throw new DepositException("directory["+dir+"] can't be read!"); - + SIPInterface sip = context.getSIP(); Set resources = context.getSIP().getResources(); - + for (Resource res:resources) { if (res.isUpdate()) { if (!res.hasFID()) throw new DepositException("Update of Resource["+res.getURI()+"] but location in the Repository is unknown!"); - + // get local checksum String name = res.getFile().getPath().replaceAll("[^a-zA-Z0-9\\-]", "_"); if (res instanceof CMDResource) @@ -85,27 +85,27 @@ public boolean perform(Context context) throws DepositException { throw new DepositException("file for Resource["+res.getURI()+"] isn't a file!"); if (!f.canRead()) throw new DepositException("file for Resource["+res.getURI()+"] can't be read!"); - + XdmNode fd = Saxon.buildDocument(new StreamSource(f)); String checksum = Saxon.xpath2string(fd,xp,null,Global.NAMESPACES); - + if (checksum.isEmpty()) throw new DepositException("checksum for Resource["+res.getURI()+"] is unknown!"); - + logger.debug("checksum["+checksum+"] for Resource["+res.getURI()+"]"); - - // get repository checksum (from FC DO DC) + + // get repository checksum (from FC DO DC) FedoraResponse resp = getDatastreamDissemination(res.getFID().toString(),"DC").execute(); if (resp.getStatus()!=200) throw new DepositException("Unexpected status["+resp.getStatus()+"] while querying Fedora Commons!"); - + XdmNode fc = Saxon.buildDocument(new StreamSource(resp.getEntityInputStream())); logger.debug("DC["+fc.toString()+"]"); String fcChecksum = Saxon.xpath2string(fc, "normalize-space(//dc:identifier[starts-with(.,'md5:')])",null,Global.NAMESPACES).replace("md5:",""); if (fcChecksum.isEmpty()) throw new DepositException("Stored checksum for Resource["+res.getURI()+"] is unknown!"); - + logger.debug("FC checksum["+fcChecksum+"] for Resource["+res.getURI()+"]"); if (fcChecksum.equals(checksum)) { @@ -116,11 +116,11 @@ public boolean perform(Context context) throws DepositException { } } - + } catch (Exception ex) { throw new DepositException("Couldn't complete CMDI resource mapping", ex); } return true; } - + } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/UpdateCollections.java b/src/main/java/nl/mpi/tla/flat/deposit/action/UpdateCollections.java index 628fddf..6abcf5a 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/UpdateCollections.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/UpdateCollections.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -56,19 +56,19 @@ * @author pavsri */ public class UpdateCollections extends FedoraAction { - + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(UpdateCollections.class.getName()); - + private File dir = null; private File first = null; - + private XsltTransformer upsert = null; - + @Override - public boolean perform(Context context) throws DepositException { + public boolean perform(Context context) throws DepositException { try { connect(context); - + String namespace = context.getProperty("activeFedoraNamespace", "lat").toString(); XdmValue namespaces = context.getProperty("fedoraNamespace", "lat"); @@ -76,20 +76,20 @@ public boolean perform(Context context) throws DepositException { dir = new File(getParameter("dir","./fox")); if (!dir.exists()) FileUtils.forceMkdir(dir); - + if (this.hasParameter("firstDir")) { first = new File(getParameter("firstDir")); if (!first.exists()) FileUtils.forceMkdir(first); } else first = dir; - + // prep the stylesheet upsert = Saxon.buildTransformer(UpdateCollections.class.getResource("/UpdateCollections/upsert-collection.xsl")).load(); SaxonListener listener = new SaxonListener("UpdateCollections",MDC.get("sip")); upsert.setMessageListener(listener); - upsert.setErrorListener(listener); + upsert.setErrorListener(listener); upsert.setParameter(new QName("fid"),new XdmAtomicValue(context.getSIP().getFID())); upsert.setParameter(new QName("new-pid"),new XdmAtomicValue(context.getSIP().getPID())); if (context.getSIP().isUpdate()) { @@ -149,7 +149,7 @@ else if (col.hasFID() && col.getFID().equals(context.getSIP().getFID())) throw new DepositException("Unexpected status["+res.getStatus()+"] while querying Fedora Commons!"); } catch(FedoraClientException e) { if (e.getStatus()==404) { - logger.debug("Collection["+col.getFID()+"] status["+e.getStatus()+"] has no CMD datastream."); + logger.debug("Collection["+col.getFID()+"] status["+e.getStatus()+"] has no CMD datastream."); } else throw new DepositException("Unexpected status["+e.getStatus()+"] while querying Fedora Commons!",e); } @@ -161,10 +161,10 @@ else if (col.hasFID() && col.getFID().equals(context.getSIP().getFID())) } } catch (Exception ex) { throw new DepositException(ex); - } + } return true; } - + private void updateCollection(Deque hist, Collection col, URI fidPart, String oldPart, String newPart, String namespace, XdmValue namespaces) throws Exception { try { // load the collection's CMD @@ -223,9 +223,9 @@ private void updateCollection(Deque hist, Collection col, URI fidPart, Stri throw new DepositException("Unexpected status["+e.getStatus()+"] while querying Fedora Commons!",e); } } - + private XsltTransformer dc = null; - + private void updateDC(File fox, URI fid, URI pid) throws FedoraClientException, SaxonApiException, TransformerConfigurationException, TransformerException, DepositException { FedoraResponse res = getDatastreamDissemination(fid.toString(),"DC").execute(); if (res.getStatus()==200) { @@ -250,5 +250,5 @@ private void updateDC(File fox, URI fid, URI pid) throws FedoraClientException, } else throw new DepositException("Unexpected status["+res.getStatus()+"] while querying Fedora Commons!"); } - + } diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogSetup.java b/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogSetup.java index 8a71dfe..6e08013 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogSetup.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/WorkspaceLogSetup.java @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2015-2017 The Language Archive * * This program is free software: you can redistribute it and/or modify @@ -20,7 +20,6 @@ import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.util.StatusPrinter; import java.io.File; -import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Paths; import java.util.UUID; @@ -43,9 +42,9 @@ * @author menzowi */ public class WorkspaceLogSetup extends AbstractAction { - + private static final Logger logger = LoggerFactory.getLogger(WorkspaceLogSetup.class.getName()); - + @Override public boolean perform(Context context) { if (MDC.get("sip")==null) @@ -66,19 +65,19 @@ public boolean perform(Context context) { return false; } } - + File dir = new File(getParameter("dir","./logs")); if (!dir.exists()) FileUtils.forceMkdir(dir); - + File logback = dir.toPath().resolve("./logback.xml").toFile(); - + if (!logback.exists()) { // create {$work}/logs/logback-dk.xml logback = dir.toPath().resolve("./logback-dk.xml").toFile(); if (logback.exists()) logback.delete(); - + XsltTransformer configure = Saxon.buildTransformer(WorkspaceLogSetup.class.getResource("/WorkspaceLog/config-log.xsl")).load(); SaxonListener listener = new SaxonListener("WorkspaceLogSetup", MDC.get("sip")); configure.setMessageListener(listener); @@ -86,17 +85,17 @@ public boolean perform(Context context) { configure.setParameter(new QName("dir"), new XdmAtomicValue(dir.toString())); configure.setParameter(new QName("sip"), new XdmAtomicValue(MDC.get("sip"))); - + Source in = (config!=null?new StreamSource(config):new StreamSource(WorkspaceLogSetup.class.getResource("/WorkspaceLog/empty-config.xml").toString())); configure.setSource(in); - + XdmDestination destination = new XdmDestination(); configure.setDestination(destination); - + configure.transform(); - + Saxon.save(destination,logback); - + if (!dir.toPath().resolve("./user-log.xml").toFile().exists()) { // copy user-log.xml to {$work}/logs/user-log.xml Files.copy(FOXCreate.class.getResourceAsStream("/WorkspaceLog/user-log.xml"), dir.toPath().resolve("./user-log.xml")); @@ -106,7 +105,7 @@ public boolean perform(Context context) { Files.copy(FOXCreate.class.getResourceAsStream("/WorkspaceLog/log4j.dtd"), dir.toPath().resolve("./log4j.dtd")); } } - + Logger logger = LoggerFactory.getLogger(nl.mpi.tla.flat.deposit.Flow.class); LoggerContext logctxt = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); @@ -120,10 +119,10 @@ public boolean perform(Context context) { "But you can never leave!\"\n" ); } catch (Exception ex) { - this.logger.error("Couldn't setup the deposit log!",ex); + logger.error("Couldn't setup the deposit log!",ex); return false; } return true; } - + } From cf4e6d545290eb15222bb595dd11cde47a24acfe Mon Sep 17 00:00:00 2001 From: pautri Date: Thu, 30 Jan 2025 13:55:55 +0100 Subject: [PATCH 6/9] Update config-log.xsl --- src/main/resources/WorkspaceLog/config-log.xsl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/resources/WorkspaceLog/config-log.xsl b/src/main/resources/WorkspaceLog/config-log.xsl index 83e04d0..e7f38c2 100644 --- a/src/main/resources/WorkspaceLog/config-log.xsl +++ b/src/main/resources/WorkspaceLog/config-log.xsl @@ -14,14 +14,11 @@ - - -/devel.log + /devel.log true - return (mdc.get("sip") != null && ((String)mdc.get("sip")).equals(" -")); + return (mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); DENY ACCEPT @@ -32,11 +29,9 @@ - -/user-log-events.xml + /user-log-events.xml true\n" + - \n" + return (level >= INFO && mdc.get("sip") != null && ((String)mdc.get("sip")).equals(" -")); + \n" + return (level >= INFO && mdc.get("sip") != null && ((String)mdc.get("sip")).equals("")); DENY ACCEPT From 87716f84c8c461370ac9607d0faa33fa558dec31 Mon Sep 17 00:00:00 2001 From: pautri Date: Thu, 30 Jan 2025 14:37:42 +0100 Subject: [PATCH 7/9] Update FITS.java --- src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java index 0f92c75..83d4dc6 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java @@ -232,12 +232,7 @@ public boolean perform(Context context) throws DepositException { try { // loop over /mimetypes/mimetype boolean bCheck1 = false; // tells if a mimetype was found for the resource - ThreadLocal threadLogger = new ThreadLocal() { - @Override - protected Logger initialValue() { - return LoggerFactory.getLogger(FITS.class); - } - }; + ThreadLocal threadLogger = ThreadLocal.withInitial(() -> logger); for (Iterator iter = Saxon.xpathIterator(mimetypes, "/mimetypes/mimetype", null, NAMESPACES); iter.hasNext();) { XdmItem mt = iter.next(); From 102329c9485e4db326171d847c77ef3a15721aff Mon Sep 17 00:00:00 2001 From: pautri Date: Thu, 30 Jan 2025 15:39:58 +0100 Subject: [PATCH 8/9] Update FITS.java --- .../nl/mpi/tla/flat/deposit/action/FITS.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java index 83d4dc6..caf2eba 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java @@ -232,7 +232,7 @@ public boolean perform(Context context) throws DepositException { try { // loop over /mimetypes/mimetype boolean bCheck1 = false; // tells if a mimetype was found for the resource - ThreadLocal threadLogger = ThreadLocal.withInitial(() -> logger); + Logger threadLogger = logger; for (Iterator iter = Saxon.xpathIterator(mimetypes, "/mimetypes/mimetype", null, NAMESPACES); iter.hasNext();) { XdmItem mt = iter.next(); @@ -249,18 +249,18 @@ public boolean perform(Context context) throws DepositException { // no xpath, so fall back to the default xpath xp = MIMETYPE_XPATH; } - threadLogger.get().debug(". . . assertions[" + xp + "] check"); + threadLogger.debug(". . . assertions[" + xp + "] check"); // evaluate xpath Map vars = new HashMap<>(); vars.put("mime", new XdmAtomicValue(mime)); if (!Saxon.xpath2boolean(result, xp, vars, NAMESPACES)) { // the assertions XPath failed, continue to the next // /mimetypes/mimetype - threadLogger.get().debug(". . . assertions[" + xp + "] check failed"); + threadLogger.debug(". . . assertions[" + xp + "] check failed"); bCheck2 = null; break; } - threadLogger.get().debug(". . . assertions[" + xp + "] check succeeded"); + threadLogger.debug(". . . assertions[" + xp + "] check succeeded"); // the assertions XPath succeeded, check the assertions for this // mimetype Boolean bCheck3 = true; // tells if all mimetype assertions succeeded @@ -275,18 +275,18 @@ public boolean perform(Context context) throws DepositException { bCheck3 = Saxon.xpath2boolean(result, axp, null, NAMESPACES); if (!bCheck3) { // assertion fails, print the AVT log message - threadLogger.get().debug(". . . . assert[" + axp + "] check failed"); - threadLogger.get().error( + threadLogger.debug(". . . . assert[" + axp + "] check failed"); + threadLogger.error( "File '{}' has a mimetype '{}' which is ALLOWED in this repository, but fails an assertion!", file, mime); - threadLogger.get().error("Message from FITS file: " + Saxon.avt( + threadLogger.error("Message from FITS file: " + Saxon.avt( Saxon.xpath2string(a, "@message"), result, context.getProperties(), NAMESPACES)); // break out of the assertion loop break; } // assertion is positive, go to next - threadLogger.get().debug(". . . . assert[" + axp + "] check succeeded"); + threadLogger.debug(". . . . assert[" + axp + "] check succeeded"); } else { // the assertion xpath does not exist throw new DepositException( @@ -297,11 +297,11 @@ public boolean perform(Context context) throws DepositException { } if (!bCheck3) { // some assertion of this assertions failed - threadLogger.get().debug(". . . assertions[" + xp + "] failed"); + threadLogger.debug(". . . assertions[" + xp + "] failed"); bCheck2 = Boolean.FALSE; break; } else - threadLogger.get().debug(". . . assertions[" + xp + "] succeeded"); + threadLogger.debug(". . . assertions[" + xp + "] succeeded"); } } else { // no assertions, use just the path @@ -315,37 +315,37 @@ public boolean perform(Context context) throws DepositException { vars.put("mime", new XdmAtomicValue(mime)); if (!Saxon.xpath2boolean(result, xp, vars, NAMESPACES)) { // the assertions XPath failed, continue to the next /mimetypes/mimetype - threadLogger.get().debug(". . . assertions[" + xp + "] check failed"); + threadLogger.debug(". . . assertions[" + xp + "] check failed"); bCheck2 = null; continue; } - threadLogger.get().debug(". . . assertions[" + xp + "] check succeeded"); + threadLogger.debug(". . . assertions[" + xp + "] check succeeded"); } if (bCheck2 == null) { - threadLogger.get().debug(". . continue to next mimetype"); + threadLogger.debug(". . continue to next mimetype"); continue; } if (bCheck2.booleanValue()) { // all assertions succeeded - threadLogger.get().debug(". . mimetype[" + mime + "] succeeded"); + threadLogger.debug(". . mimetype[" + mime + "] succeeded"); bCheck1 = true; - threadLogger.get().info( + threadLogger.info( "Resource[{}] has a mimetype which is ALLOWED in this repository and satisfies all assertions: '{}'", file, mime); if (resource.hasMime() && !resource.getMime().equals(mime)) { logger.warn("Resource mimetype changed from '{}' to '{}'", resource.getMime(), mime); } - threadLogger.get().debug("Setting resource mimetype to '{}'", mime); + threadLogger.debug("Setting resource mimetype to '{}'", mime); resource.setMime(mime); } else - threadLogger.get().debug(". . mimetype[" + mime + "] failed"); + threadLogger.debug(". . mimetype[" + mime + "] failed"); break; } if (!bCheck1) { // no mimetype was found, look for the otherwise - threadLogger.get().debug(". mimetypes failed, checking otherwise"); + threadLogger.debug(". mimetypes failed, checking otherwise"); XdmItem o = Saxon.xpathSingle(mimetypes, "/mimetypes/otherwise"); if (o != null) { // check for an xpath @@ -356,26 +356,26 @@ public boolean perform(Context context) throws DepositException { if (!fallback.equals("")) { // use the non-empty fallback value as mimetype for the resource bCheck1 = true; - threadLogger.get().error("Use fallback mimetype[{}] for resource[{}]", fallback, + threadLogger.error("Use fallback mimetype[{}] for resource[{}]", fallback, file); if (resource.hasMime() && !resource.getMime().equals(fallback)) { - threadLogger.get().warn("Resource mimetype changed from '{}' to '{}'", + threadLogger.warn("Resource mimetype changed from '{}' to '{}'", resource.getMime(), fallback); } - threadLogger.get().debug("Setting resource mimetype to '{}'", fallback); + threadLogger.debug("Setting resource mimetype to '{}'", fallback); resource.setMime(fallback); } } } else - threadLogger.get().debug(". mimetypes failed, no otherwise"); + threadLogger.debug(". mimetypes failed, no otherwise"); if (!bCheck1) { // no allowed or fallback mimetype was found for this resource - threadLogger.get().debug(". mimetypes failed"); - threadLogger.get().error("No mimetype found for resource[{}]", file); + threadLogger.debug(". mimetypes failed"); + threadLogger.error("No mimetype found for resource[{}]", file); unallowed++; } else - threadLogger.get().debug(". mimetypes succeeded"); + threadLogger.debug(". mimetypes succeeded"); } } catch (Exception ex) { throw new DepositException(ex); From e74c4e304b0d150fd99d467b5cff57990d9f8103 Mon Sep 17 00:00:00 2001 From: pautri Date: Thu, 30 Jan 2025 15:49:26 +0100 Subject: [PATCH 9/9] Update FITS.java --- src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java index caf2eba..70d52fc 100644 --- a/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java +++ b/src/main/java/nl/mpi/tla/flat/deposit/action/FITS.java @@ -237,7 +237,7 @@ public boolean perform(Context context) throws DepositException { null, NAMESPACES); iter.hasNext();) { XdmItem mt = iter.next(); String mime = Saxon.xpath2string(mt, "normalize-space(@value)"); - threadLogger.get().debug(". . mimetype[" + mime + "] check"); + threadLogger.debug(". . mimetype[" + mime + "] check"); Boolean bCheck2 = Boolean.TRUE; // tells if all assertion groups succeeded if (Saxon.xpath2boolean(mt, "exists(assertions)")) { for (Iterator iter2 = Saxon.xpathIterator(mt, "assertions", null,