From d6c06004410a40405d55b6b00d059ed8338c284e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 19 Sep 2023 22:54:39 +0200 Subject: [PATCH 01/69] feat(mail): add MTA settings for mail to JvmSettings #7424 --- .../edu/harvard/iq/dataverse/settings/JvmSettings.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 738d63e924f..2088dcbc5e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -123,6 +123,15 @@ public enum JvmSettings { SCOPE_MAIL(PREFIX, "mail"), SUPPORT_EMAIL(SCOPE_MAIL, "support-email"), CC_SUPPORT_ON_CONTACT_EMAIL(SCOPE_MAIL, "cc-support-on-contact-email"), + MAIL_DEBUG(SCOPE_MAIL, "debug"), + // Mail Transfer Agent settings + SCOPE_MAIL_MTA(SCOPE_MAIL, "mta"), + MAIL_MTA_HOST(SCOPE_MAIL_MTA, "host"), + MAIL_MTA_AUTH(SCOPE_MAIL_MTA, "auth"), + MAIL_MTA_USER(SCOPE_MAIL_MTA, "user"), + MAIL_MTA_PASSWORD(SCOPE_MAIL_MTA, "password"), + // Placeholder setting for a large list of extra settings + MAIL_MTA_SETTING(SCOPE_MAIL_MTA), // UI SETTINGS SCOPE_UI(PREFIX, "ui"), From de759c1a23d6a997767e540ec8503cbdb0a6e26d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 12:58:34 +0200 Subject: [PATCH 02/69] feat(mail): add minimal implementation of a mail session factory #7424 --- .../dataverse/util/MailSessionProducer.java | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java new file mode 100644 index 00000000000..7728fa338ca --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -0,0 +1,87 @@ +package edu.harvard.iq.dataverse.util; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Named; +import jakarta.mail.Authenticator; +import jakarta.mail.PasswordAuthentication; +import jakarta.mail.Session; + +import java.util.List; +import java.util.Properties; + +@ApplicationScoped +public class MailSessionProducer { + + // NOTE: We do not allow "from" here, as we want the transport to get it from the message being sent, enabling + // matching addresses. If "from" in transport and "from" in the message differ, some MTAs may reject or + // classify as spam. + // NOTE: Complete list including descriptions at https://eclipse-ee4j.github.io/angus-mail/docs/api/org.eclipse.angus.mail/org/eclipse/angus/mail/smtp/package-summary.html + static final List smtpStringProps = List.of( + "localhost", "localaddress", "auth.mechanisms", "auth.ntlm.domain", "submitter", "dsn.notify", "dsn.ret", + "sasl.mechanisms", "sasl.authorizationid", "sasl.realm", "ssl.trust", "ssl.protocols", "ssl.ciphersuites", + "proxy.host", "proxy.port", "proxy.user", "proxy.password", "socks.host", "socks.port", "mailextension" + ); + static final List smtpIntProps = List.of( + "port", "connectiontimeout", "timeout", "writetimeout", "localport", "auth.ntlm.flag" + ); + static final List smtpBoolProps = List.of( + "ehlo", "auth.login.disable", "auth.plain.disable", "auth.digest-md5.disable", "auth.ntlm.disable", + "auth.xoauth2.disable", "allow8bitmime", "sendpartial", "sasl.enable", "sasl.usecanonicalhostname", + "quitwait", "quitonsessionreject", "ssl.enable", "ssl.checkserveridentity", "starttls.enable", + "starttls.required", "userset", "noop.strict" + ); + + private static final String PREFIX = "mail.stmp."; + + Session systemMailSession; + + @Produces + @Named("mail/systemSession") + public Session getSession() { + if (systemMailSession == null) { + // Initialize with null (= no authenticator) is a valid argument for the session factory method. + Authenticator authenticator = null; + + // In case we want auth, create an authenticator (default = false from microprofile-config.properties) + if (JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class)) { + authenticator = new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(JvmSettings.MAIL_MTA_USER.lookup(), JvmSettings.MAIL_MTA_PASSWORD.lookup()); + } + }; + } + + this.systemMailSession = Session.getInstance(getMailProperties(), authenticator); + } + return systemMailSession; + } + + Properties getMailProperties() { + Properties configuration = new Properties(); + + // See https://jakarta.ee/specifications/mail/2.1/apidocs/jakarta.mail/jakarta/mail/package-summary + configuration.put("mail.transport.protocol", "smtp"); + configuration.put("mail.debug", JvmSettings.MAIL_DEBUG.lookupOptional(Boolean.class).orElse(false).toString()); + + configuration.put("mail.smtp.host", JvmSettings.MAIL_MTA_HOST.lookup()); + // default = false from microprofile-config.properties + configuration.put("mail.smtp.auth", JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class).toString()); + + // Map properties 1:1 to mail.smtp properties for the mail session. + smtpStringProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(prop).ifPresent( + string -> configuration.put(PREFIX + prop, string))); + smtpBoolProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(Boolean.class, prop).ifPresent( + bool -> configuration.put(PREFIX + prop, bool.toString()))); + smtpIntProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(Integer.class, prop).ifPresent( + number -> configuration.put(PREFIX + prop, number.toString()))); + + return configuration; + } + +} From 72cdde93acba7270c9cae55129162ecf8bcc2c0d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 12:59:35 +0200 Subject: [PATCH 03/69] feat(mail): add sane defaults for mail jvm settings at app level #7424 We only default to no authentication. We still require people to configure an SMTP host, only in containers we do default to "smtp" as a hostname for that (see our compose file). Username/password cannot have a default and all other special settings should not be done here. These are highly setup specific. --- src/main/resources/META-INF/microprofile-config.properties | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index 7c16495f870..e71542c3c89 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -36,6 +36,11 @@ dataverse.rserve.user=rserve dataverse.rserve.password=rserve dataverse.rserve.tempdir=/tmp/Rserv +# MAIL +dataverse.mail.mta.auth=false +# In containers, default to hostname smtp, a container on the same network +%ct.dataverse.mail.mta.host=smtp + # OAI SERVER dataverse.oai.server.maxidentifiers=100 dataverse.oai.server.maxrecords=10 From 02f1c3d534822a0bd285a29949ae6ef26cb84f3d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 13:47:12 +0200 Subject: [PATCH 04/69] feat(mail): inject mail session via CDI from factory in MailServiceBean #7424 --- src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index f17732df7b6..3463aa211bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -26,9 +26,10 @@ import java.util.List; import java.util.Set; import java.util.logging.Logger; -import jakarta.annotation.Resource; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.inject.Inject; +import jakarta.inject.Named; import jakarta.mail.Address; import jakarta.mail.Message; import jakarta.mail.MessagingException; @@ -79,7 +80,8 @@ public class MailServiceBean implements java.io.Serializable { public MailServiceBean() { } - @Resource(name = "mail/notifyMailSession") + @Inject + @Named("mail/systemSession") private Session session; public boolean sendSystemEmail(String to, String subject, String messageText) { From 1f79f57556ec6b92de344e3a0a305a25f0196103 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 13:47:44 +0200 Subject: [PATCH 05/69] fix(mail): make error logs about missing mapping file go away #7424 --- src/main/resources/META-INF/javamail.default.address.map | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/main/resources/META-INF/javamail.default.address.map diff --git a/src/main/resources/META-INF/javamail.default.address.map b/src/main/resources/META-INF/javamail.default.address.map new file mode 100644 index 00000000000..b1115c9dc8c --- /dev/null +++ b/src/main/resources/META-INF/javamail.default.address.map @@ -0,0 +1,2 @@ +# See https://jakartaee.github.io/mail-api/docs/api/jakarta.mail/jakarta/mail/Session.html +rfc822=smtp From 03b11bf7342f21b5486f4f473094de08eba26c3b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 13:48:31 +0200 Subject: [PATCH 06/69] feat(ct,mail): no longer configure mail in containers manually #7424 --- src/main/docker/scripts/init_2_configure.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/docker/scripts/init_2_configure.sh b/src/main/docker/scripts/init_2_configure.sh index a98f08088c1..b31cfac37b7 100755 --- a/src/main/docker/scripts/init_2_configure.sh +++ b/src/main/docker/scripts/init_2_configure.sh @@ -31,10 +31,6 @@ echo "# Dataverse postboot configuration for Payara" > "${DV_POSTBOOT}" # EE 8 code annotations or at least glassfish-resources.xml # NOTE: postboot commands is not multi-line capable, thus spaghetti needed. -# JavaMail -echo "INFO: Defining JavaMail." -echo "create-javamail-resource --mailhost=${DATAVERSE_MAIL_HOST:-smtp} --mailuser=${DATAVERSE_MAIL_USER:-dataversenotify} --fromaddress=${DATAVERSE_MAIL_FROM:-dataverse@localhost} mail/notifyMailSession" >> "${DV_POSTBOOT}" - # 3. Domain based configuration options # Set Dataverse environment variables echo "INFO: Defining system properties for Dataverse configuration options." From ee88cfd8e3ec42d8244575a4f5a77cca914a8346 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 13:51:10 +0200 Subject: [PATCH 07/69] doc(ct): remove mail env vars from app image docs #7424 --- doc/sphinx-guides/source/container/app-image.rst | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 29f6d6ac1d4..4720760544e 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -134,19 +134,6 @@ In addition, the application image provides the following tunables: 1. Simply pick a JVM option from the list and replace any ``.`` with ``_``. 2. Replace any ``-`` in the option name with ``__``. - * - ``DATAVERSE_MAIL_HOST`` - - ``smtp`` - - String - - A hostname (w/o port!) where to reach a Mail MTA on port 25. - * - ``DATAVERSE_MAIL_USER`` - - ``dataversenotify`` - - String - - A username to use with the Mail MTA - * - ``DATAVERSE_MAIL_FROM`` - - ``dataverse@localhost`` - - Mail address - - The "From" field for all outbound mail. Make sure to set :ref:`systemEmail` to the same value or no mail will - be sent. Note that the script ``init_2_configure.sh`` will apply a few very important defaults to enable quick usage From 600d209fbd7b70893fe29f7601ba8049cb415d8f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 13:53:29 +0200 Subject: [PATCH 08/69] build(test): enable using GenericContainer without JUnit4 around As a hack to work around testcontainers/testcontainers-java#970, we add these fake, empty classes. Copied from Spring project. See also: https://github.com/testcontainers/testcontainers-java/issues/970 --- src/test/java/org/junit/rules/TestRule.java | 11 +++++++++++ src/test/java/org/junit/runners/model/Statement.java | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 src/test/java/org/junit/rules/TestRule.java create mode 100644 src/test/java/org/junit/runners/model/Statement.java diff --git a/src/test/java/org/junit/rules/TestRule.java b/src/test/java/org/junit/rules/TestRule.java new file mode 100644 index 00000000000..4f94d8e6922 --- /dev/null +++ b/src/test/java/org/junit/rules/TestRule.java @@ -0,0 +1,11 @@ +package org.junit.rules; + +/** + * "Fake" class used as a replacement for Junit4-dependent classes. + * See more at: + * GenericContainer run from Jupiter tests shouldn't require JUnit 4.x library on runtime classpath + * . + */ +@SuppressWarnings("unused") +public interface TestRule { +} diff --git a/src/test/java/org/junit/runners/model/Statement.java b/src/test/java/org/junit/runners/model/Statement.java new file mode 100644 index 00000000000..b80ca0abc86 --- /dev/null +++ b/src/test/java/org/junit/runners/model/Statement.java @@ -0,0 +1,11 @@ +package org.junit.runners.model; + +/** + * "Fake" class used as a replacement for Junit4-dependent classes. + * See more at: + * GenericContainer run from Jupiter tests shouldn't require JUnit 4.x library on runtime classpath + * . + */ +@SuppressWarnings("unused") +public class Statement { +} From 6f6a9b7b4cbccf1cb1e6521c2b65a9eb81c85af6 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 14:52:32 +0200 Subject: [PATCH 09/69] fix(mail): correct typo in mail.smtp prefix string #7424 --- .../edu/harvard/iq/dataverse/util/MailSessionProducer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 7728fa338ca..87475359215 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -33,7 +33,7 @@ public class MailSessionProducer { "starttls.required", "userset", "noop.strict" ); - private static final String PREFIX = "mail.stmp."; + private static final String PREFIX = "mail.smtp."; Session systemMailSession; @@ -66,9 +66,9 @@ Properties getMailProperties() { configuration.put("mail.transport.protocol", "smtp"); configuration.put("mail.debug", JvmSettings.MAIL_DEBUG.lookupOptional(Boolean.class).orElse(false).toString()); - configuration.put("mail.smtp.host", JvmSettings.MAIL_MTA_HOST.lookup()); + configuration.put(PREFIX + "host", JvmSettings.MAIL_MTA_HOST.lookup()); // default = false from microprofile-config.properties - configuration.put("mail.smtp.auth", JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class).toString()); + configuration.put(PREFIX + "auth", JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class).toString()); // Map properties 1:1 to mail.smtp properties for the mail session. smtpStringProps.forEach( From c23ccded546910b9175c35841930ee2901af1c10 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 14:53:20 +0200 Subject: [PATCH 10/69] chore(build,test): upgrade to Testcontainers v1.19.0 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index c45d59e4f5f..29743aa3974 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -168,7 +168,7 @@ 5.1.0 - 1.15.0 + 1.19.0 2.10.1 5.10.0 From eb1664f08fa896013adb18be0171c1b5bddd5609 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 14:53:57 +0200 Subject: [PATCH 11/69] feat(mail): add explicit injection constructor to MailServiceBean #7424 Necessary to add some integration testing, verifying sending mails actually should work. --- .../java/edu/harvard/iq/dataverse/MailServiceBean.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 3463aa211bb..df24099f60c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -79,6 +79,14 @@ public class MailServiceBean implements java.io.Serializable { */ public MailServiceBean() { } + + /** + * Creates a new instance of MailServiceBean with explicit injection, as used during testing. + */ + public MailServiceBean(Session session, SettingsServiceBean settingsService) { + this.session = session; + this.settingsService = settingsService; + } @Inject @Named("mail/systemSession") From 086f766c714feb61065ab4420783e31f57c91e99 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 14:58:19 +0200 Subject: [PATCH 12/69] test(mail): add integration test for mail session configuration and usage #7424 --- .../dataverse/util/MailSessionProducerIT.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java new file mode 100644 index 00000000000..b41fa69ff52 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java @@ -0,0 +1,103 @@ +package edu.harvard.iq.dataverse.util; + +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.MailServiceBean; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import io.restassured.RestAssured; +import jakarta.mail.Session; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * An integration test using a fake SMTP MTA to check for outgoing mails. + * LIMITATION: This test cannot possibly check if the production and injection of the session via CDI + * works, as it is not running within a servlet container. This would require usage of Arquillian + * or and end-to-end API test with a deployed application. + */ +@Testcontainers +@ExtendWith(MockitoExtension.class) +class MailSessionProducerIT { + + private static final Integer PORT_SMTP = 1025; + private static final Integer PORT_HTTP = 1080; + + Integer smtpPort; + String smtpHost; + + @Mock + SettingsServiceBean settingsServiceBean; + @Mock + DataverseServiceBean dataverseServiceBean; + + @Container + static GenericContainer maildev = new GenericContainer<>("maildev/maildev:2.1.0") + .withExposedPorts(PORT_HTTP, PORT_SMTP) + .waitingFor(Wait.forHttp("/")); + + @BeforeEach + void setUp() { + smtpHost = maildev.getHost(); + smtpPort = maildev.getMappedPort(PORT_SMTP); + Integer httpPort = maildev.getMappedPort(PORT_HTTP); + + RestAssured.baseURI = "http://" + smtpHost; + RestAssured.port = httpPort; + + // Setup mocks + Mockito.when(settingsServiceBean.getValueForKey(SettingsServiceBean.Key.SystemEmail)).thenReturn("noreply@example.org"); + BrandingUtil.injectServices(dataverseServiceBean, settingsServiceBean); + + // TODO: Once we merge PR 9273 (https://github.com/IQSS/dataverse/pull/9273), + // we can use methods to inject the settings in @JvmSetting + System.setProperty(JvmSettings.MAIL_MTA_HOST.getScopedKey(), smtpHost); + System.setProperty(JvmSettings.MAIL_MTA_SETTING.insert("port"), smtpPort.toString()); + } + + @AfterEach + void tearDown() { + System.clearProperty(JvmSettings.MAIL_MTA_HOST.getScopedKey()); + System.clearProperty(JvmSettings.MAIL_MTA_SETTING.insert("port")); + } + + @Test + //@JvmSetting(key = JvmSettings.MAIL_DEBUG, value = "true") + void createSessionWithoutAuth() { + given().when().get("/email") + .then() + .statusCode(200) + .body("size()", is(0)); + + // given + Session session = new MailSessionProducer().getSession(); + MailServiceBean mailer = new MailServiceBean(session, settingsServiceBean); + + // when + boolean sent = mailer.sendSystemEmail("test@example.org", "Test", "Test", false); + + // then + assertTrue(sent); + //RestAssured.get("/email").body().prettyPrint(); + given().when().get("/email") + .then() + .statusCode(200) + .body("size()", is(1)) + .body("[0].subject", equalTo("Test")); + } + +} \ No newline at end of file From 51af5e12051db26eebdbb7a4afe23173104c5043 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 20 Sep 2023 14:58:42 +0200 Subject: [PATCH 13/69] build(mail): exclude geronimo javamail spec from dependencies If not excluded, the very old Javamail 1.4 spec is being used during local testing, obviously incompatible with Jakarta EE Mail definition. Exclusion is the only way around this, as we cannot possibly change the upstream dependencies. --- pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom.xml b/pom.xml index 7ba22d2a076..e781cca3b7c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,17 @@ abdera-i18n 1.1.3 + + org.apache.abdera + abdera-parser + 1.1.3 + + + org.apache.geronimo.specs + geronimo-javamail_1.4_spec + + + 1.19.0 2.10.1 + 1.19.1 5.10.0 5.4.0 0.8.10 From 4bfda6c86d9de90bc23026ce9f7dda146bcd710e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 11 Oct 2023 14:06:23 +0200 Subject: [PATCH 35/69] chore(build): update to SmallRye Config 3.4.1 --- modules/dataverse-parent/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index ab5bd54c934..4ff2a167e6c 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -168,9 +168,8 @@ 5.1.0 - 1.19.0 - 2.10.1 1.19.1 + 3.4.1 5.10.0 5.4.0 0.8.10 From b74d60f2af45148d3d59b459668733963915ec6a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 11 Oct 2023 14:12:30 +0200 Subject: [PATCH 36/69] chore(build): update Mockito to v5.6.0 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 4ff2a167e6c..94ee48bef9b 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -171,7 +171,7 @@ 1.19.1 3.4.1 5.10.0 - 5.4.0 + 5.6.0 0.8.10 9.3 From 9397cc38690fe1b8adcffd7bb82080488e6e85ad Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 8 Nov 2023 22:58:16 +0100 Subject: [PATCH 37/69] fix(mail): lookup legacy mail session programmatically #7424 Using @Resource on the field triggers deployments to fail if the resource is not provided by the app server. Using a programmatic lookup, we can catch and ignore the exception. --- .../dataverse/util/MailSessionProducer.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 93f3ac29e44..25f5970274e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -9,8 +9,12 @@ import jakarta.mail.PasswordAuthentication; import jakarta.mail.Session; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; import java.util.List; import java.util.Properties; +import java.util.logging.Level; import java.util.logging.Logger; @ApplicationScoped @@ -41,13 +45,24 @@ public class MailSessionProducer { Session systemMailSession; /** - * Inject the application server provided (user defined) javamail resource to enable backwards compatibility. + * Cache the application server provided (user defined) javamail resource to enable backwards compatibility. + * No direct JNDI lookup on the field to avoid deployment failures when not present. * @deprecated This should be removed with the next major release of Dataverse, as it would be a breaking change. */ @Deprecated(forRemoval = true, since = "6.1") - @Resource(name = "mail/notifyMailSession") Session appserverProvidedSession; + public MailSessionProducer() { + try { + // Do JNDI lookup of legacy mail session programmatically to avoid deployment errors when not found. + Context initialContext = new InitialContext(); + this.appserverProvidedSession = (Session)initialContext.lookup("mail/notifyMailSession"); + } catch (NamingException e) { + // This exception simply means the appserver did not provide the legacy mail session. + // Debug level output is just fine. + logger.log(Level.FINE, "Error during mail resource lookup", e); + } + } @Produces @Named("mail/systemSession") @@ -104,4 +119,14 @@ Properties getMailProperties() { return configuration; } + /** + * Determine if the session returned by {@link #getSession()} has been provided by the application server + * @return True if injected as resource from app server, false otherwise + * @deprecated This is supposed to be removed when {@link #appserverProvidedSession} is removed. + */ + @Deprecated(forRemoval = true, since = "6.1") + public boolean hasSessionFromAppServer() { + return this.appserverProvidedSession != null; + } + } From d650725f794ae1e471939e3b48b18a24e5456ffa Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 8 Nov 2023 23:08:26 +0100 Subject: [PATCH 38/69] build(mail): add .map files to be included in resources #7424 Without this change, the javamail maps would not be included in the artifact and trigger error messages in the logs about them being missed. The error message will still be present as long as payara/Payara#6254 is not fixed, released and we updated to a newer version of Payara. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 843afc22acb..75b0cf1100f 100644 --- a/pom.xml +++ b/pom.xml @@ -688,6 +688,7 @@ **/firstNames/*.* **/*.xsl **/services/* + **/*.map From 11826d9a5decf521daba183035bfd02ec248fc89 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 9 Nov 2023 00:07:08 +0100 Subject: [PATCH 39/69] feat(mail): add startup checks for mail configuration #7424 During the deployment of Dataverse we check for conditions of the mail system that might not be done as people intend to use it. We'll only issue warnings in the log messages, nothing critical here. --- .../settings/ConfigCheckService.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java index a2c3f53d59d..c4f4fc6610b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java @@ -1,16 +1,22 @@ package edu.harvard.iq.dataverse.settings; +import edu.harvard.iq.dataverse.MailServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; import edu.harvard.iq.dataverse.util.FileUtil; - +import edu.harvard.iq.dataverse.util.MailSessionProducer; import jakarta.annotation.PostConstruct; import jakarta.ejb.DependsOn; import jakarta.ejb.Singleton; import jakarta.ejb.Startup; +import jakarta.inject.Inject; +import jakarta.mail.internet.InternetAddress; + import java.io.IOException; import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -20,6 +26,11 @@ public class ConfigCheckService { private static final Logger logger = Logger.getLogger(ConfigCheckService.class.getCanonicalName()); + + @Inject + MailSessionProducer mailSessionProducer; + @Inject + MailServiceBean mailService; public static class ConfigurationError extends RuntimeException { public ConfigurationError(String message) { @@ -32,6 +43,9 @@ public void startup() { if (!checkSystemDirectories()) { throw new ConfigurationError("Not all configuration checks passed successfully. See logs above."); } + + // Only checks resulting in warnings, nothing critical that needs to stop deployment + checkSystemMailSetup(); } /** @@ -77,5 +91,36 @@ public boolean checkSystemDirectories() { } return success; } + + /** + * This method is not expected to make a deployment fail, but send out clear warning messages about missing or + * wrong configuration settings. + */ + public void checkSystemMailSetup() { + // Check if a system mail setting has been provided or issue warning about disabled mail notifications + Optional mailAddress = mailService.getSystemAddress(); + + // Not present -> warning + if (mailAddress.isEmpty()) { + logger.warning("Could not find a system mail setting in database (key :" + Key.SystemEmail + ", deprecated) or JVM option '" + JvmSettings.SYSTEM_EMAIL.getScopedKey() + "'"); + logger.warning("Mail notifications and system messages are deactivated until you provide a configuration"); + } + + // If there is an app server provided mail config, let's determine if the setup is matching + // TODO: when support for appserver provided mail session goes away, this code can be deleted + if (mailSessionProducer.hasSessionFromAppServer()) { + if (mailAddress.isEmpty()) { + logger.warning("Found a mail session provided by app server, but no system mail address (see logs above)"); + // Check if the "from" in the session is the same as the system mail address (see issue 4210) + } else { + String sessionFrom = mailSessionProducer.getSession().getProperty("mail.from"); + if (! mailAddress.get().toString().equals(sessionFrom)) { + logger.warning(() -> String.format( + "Found app server mail session provided 'from' (%s) does not match system mail setting (%s)", + sessionFrom, mailAddress.get())); + } + } + } + } } From 213b0256fc41ec745979f4a85dd3e259e2c218c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 Feb 2024 14:46:50 +0100 Subject: [PATCH 40/69] test(mail): add more tests for mail session producer with invalid config Also fix minor linting with visibility of test methods. --- .../iq/dataverse/MailServiceBeanTest.java | 4 +- .../dataverse/util/MailSessionProducerIT.java | 48 +++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanTest.java index f8a01c53298..afcc12949d6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanTest.java @@ -35,11 +35,11 @@ class Delegation { * We need to reset the BrandingUtil mocks for every test, as we rely on them being set to default. */ @BeforeEach - private void setup() { + void setup() { BrandingUtilTest.setupMocks(); } @AfterAll - private static void tearDown() { + static void tearDown() { BrandingUtilTest.tearDownMocks(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java index dcf04b7644a..8280578a343 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.MailServiceBean; import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.branding.BrandingUtilTest; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.testing.JvmSetting; @@ -10,14 +11,12 @@ import edu.harvard.iq.dataverse.util.testing.Tags; import io.restassured.RestAssured; import jakarta.mail.Session; -import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.testcontainers.containers.GenericContainer; @@ -30,6 +29,9 @@ import static io.restassured.RestAssured.given; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -43,6 +45,8 @@ @Tag(Tags.USES_TESTCONTAINERS) @Testcontainers(disabledWithoutDocker = true) @ExtendWith(MockitoExtension.class) +@LocalJvmSettings +@JvmSetting(key = JvmSettings.SYSTEM_EMAIL, value = "test@test.com") class MailSessionProducerIT { private static final Integer PORT_SMTP = 1025; @@ -51,15 +55,21 @@ class MailSessionProducerIT { static SettingsServiceBean settingsServiceBean = Mockito.mock(SettingsServiceBean.class);; static DataverseServiceBean dataverseServiceBean = Mockito.mock(DataverseServiceBean.class);; + /** + * We need to reset the BrandingUtil mocks for every test, as we rely on them being set to default. + */ @BeforeAll static void setUp() { // Setup mocks behavior, inject as deps BrandingUtil.injectServices(dataverseServiceBean, settingsServiceBean); } + @AfterAll + static void tearDown() { + BrandingUtilTest.tearDownMocks(); + } @Nested @LocalJvmSettings - @JvmSetting(key = JvmSettings.SYSTEM_EMAIL, value = "test@test.com") @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") class WithoutAuthentication { @@ -113,7 +123,6 @@ void createSession() { @Nested @LocalJvmSettings - @JvmSetting(key = JvmSettings.SYSTEM_EMAIL, value = "test@test.com") @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") @JvmSetting(key = JvmSettings.MAIL_MTA_AUTH, value = "yes") @@ -169,4 +178,33 @@ void createSession() { } + @Nested + @LocalJvmSettings + class InvalidConfiguration { + @Test + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, value = "1234", varArgs = "invalid") + void invalidConfigItemsAreIgnoredOnSessionBuild() { + assertDoesNotThrow(() -> new MailSessionProducer().getSession()); + + Session mailSession = new MailSessionProducer().getSession(); + MailServiceBean mailer = new MailServiceBean(mailSession, settingsServiceBean); + assertFalse(mailer.sendSystemEmail("test@example.org", "Test", "Test", false)); + } + + @Test + @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, value = "foobar") + void invalidHostnameIsFailingWhenSending() { + assertDoesNotThrow(() -> new MailSessionProducer().getSession()); + + Session mailSession = new MailSessionProducer().getSession(); + MailServiceBean mailer = new MailServiceBean(mailSession, settingsServiceBean); + assertFalse(mailer.sendSystemEmail("test@example.org", "Test", "Test", false)); + } + + @Test + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, varArgs = "port" , value = "foobar") + void invalidPortWithLetters() { + assertThrows(IllegalArgumentException.class, () -> new MailSessionProducer().getSession()); + } + } } \ No newline at end of file From 084fa3219a7bbe609f6180eba50f380d9a450247 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 08:25:48 +0100 Subject: [PATCH 41/69] chore(test): remove leftover JUnit 4 rules --- src/test/java/org/junit/rules/TestRule.java | 11 ----------- src/test/java/org/junit/runners/model/Statement.java | 11 ----------- 2 files changed, 22 deletions(-) delete mode 100644 src/test/java/org/junit/rules/TestRule.java delete mode 100644 src/test/java/org/junit/runners/model/Statement.java diff --git a/src/test/java/org/junit/rules/TestRule.java b/src/test/java/org/junit/rules/TestRule.java deleted file mode 100644 index 4f94d8e6922..00000000000 --- a/src/test/java/org/junit/rules/TestRule.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.junit.rules; - -/** - * "Fake" class used as a replacement for Junit4-dependent classes. - * See more at: - * GenericContainer run from Jupiter tests shouldn't require JUnit 4.x library on runtime classpath - * . - */ -@SuppressWarnings("unused") -public interface TestRule { -} diff --git a/src/test/java/org/junit/runners/model/Statement.java b/src/test/java/org/junit/runners/model/Statement.java deleted file mode 100644 index b80ca0abc86..00000000000 --- a/src/test/java/org/junit/runners/model/Statement.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.junit.runners.model; - -/** - * "Fake" class used as a replacement for Junit4-dependent classes. - * See more at: - * GenericContainer run from Jupiter tests shouldn't require JUnit 4.x library on runtime classpath - * . - */ -@SuppressWarnings("unused") -public class Statement { -} From 4d3904f66f20bc78a0fba718557543fa694280a2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 08:30:15 +0100 Subject: [PATCH 42/69] test(mail): verify SMTP over SSL/TLS works Adding an integration test with self-signed certificates to enable verification SMTP over SSL works. --- .../dataverse/util/MailSessionProducerIT.java | 60 +++++++++++++++++++ src/test/resources/mail/cert.pem | 24 ++++++++ src/test/resources/mail/key.pem | 28 +++++++++ 3 files changed, 112 insertions(+) create mode 100644 src/test/resources/mail/cert.pem create mode 100644 src/test/resources/mail/key.pem diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java index 8280578a343..c4893652153 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java @@ -23,6 +23,7 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; import java.util.Map; @@ -118,6 +119,65 @@ void createSession() { } + @Nested + @LocalJvmSettings + @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, varArgs = "ssl.enable", value = "true") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, varArgs = "ssl.trust", value = "*") + class WithSSLWithoutAuthentication { + @Container + static GenericContainer maildev = new GenericContainer<>("maildev/maildev:2.1.0") + .withCopyFileToContainer(MountableFile.forClasspathResource("mail/cert.pem"), "/cert.pem") + .withCopyFileToContainer(MountableFile.forClasspathResource("mail/key.pem"), "/key.pem") + .withExposedPorts(PORT_HTTP, PORT_SMTP) + .withEnv(Map.of( + "MAILDEV_INCOMING_SECURE", "true", + "MAILDEV_INCOMING_CERT", "/cert.pem", + "MAILDEV_INCOMING_KEY", "/key.pem" + )) + .waitingFor(Wait.forHttp("/")); + + static String tcSmtpHost() { + return maildev.getHost(); + } + + static String tcSmtpPort() { + return maildev.getMappedPort(PORT_SMTP).toString(); + } + + @BeforeAll + static void setup() { + RestAssured.baseURI = "http://" + tcSmtpHost(); + RestAssured.port = maildev.getMappedPort(PORT_HTTP); + } + + @Test + void createSession() { + given().when().get("/email") + .then() + .statusCode(200) + .body("size()", is(0)); + + // given + Session session = new MailSessionProducer().getSession(); + MailServiceBean mailer = new MailServiceBean(session, settingsServiceBean); + + // when + boolean sent = mailer.sendSystemEmail("test@example.org", "Test", "Test", false); + + // then + assertTrue(sent); + //RestAssured.get("/email").body().prettyPrint(); + given().when().get("/email") + .then() + .statusCode(200) + .body("size()", is(1)) + .body("[0].subject", equalTo("Test")); + } + + } + static final String username = "testuser"; static final String password = "supersecret"; diff --git a/src/test/resources/mail/cert.pem b/src/test/resources/mail/cert.pem new file mode 100644 index 00000000000..6115183d413 --- /dev/null +++ b/src/test/resources/mail/cert.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEFTCCAv0CFAIjr/AvBVg4EX5/rk5+eFdfsquOMA0GCSqGSIb3DQEBCwUAMIHG +MQswCQYDVQQGEwJEVjEaMBgGA1UECAwRRGF0YXZlcnNlIENvdW50cnkxFzAVBgNV +BAcMDkRhdGF2ZXJzZSBDaXR5MS4wLAYDVQQKDCVHbG9iYWwgRGF0YXZlcnNlIENv +bW11bml0eSBDb25zb3J0aXVtMRswGQYDVQQLDBJUZXN0aW5nIERlcGFydG1lbnQx +FDASBgNVBAMMC2V4YW1wbGUub3JnMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1w +bGUub3JnMB4XDTI0MDIyMDA3MTkxOVoXDTM0MDIxNzA3MTkxOVowgcYxCzAJBgNV +BAYTAkRWMRowGAYDVQQIDBFEYXRhdmVyc2UgQ291bnRyeTEXMBUGA1UEBwwORGF0 +YXZlcnNlIENpdHkxLjAsBgNVBAoMJUdsb2JhbCBEYXRhdmVyc2UgQ29tbXVuaXR5 +IENvbnNvcnRpdW0xGzAZBgNVBAsMElRlc3RpbmcgRGVwYXJ0bWVudDEUMBIGA1UE +AwwLZXhhbXBsZS5vcmcxHzAdBgkqhkiG9w0BCQEWEHRlc3RAZXhhbXBsZS5vcmcw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCzQ55QKM/sVJMb9c5MKtc/ +YW3+MlCrCnGlo42DCjl6noZg8Gji4dOEMo29UcRtYqhOsx7HOXZ5ulj3YKiBfzht ++QV/ZofhMIN9F/N5XCi4MRPorFz+mPck5NDzH1SqYn5zGm5APPqFJlwBWxDKEfqe +6ir5gG91MzHHuJJSQq3nrSDq+/DXRwg/7L2O7da6pBqti7nYU0T5ql88nddkRhR8 +7NdeZndI+UVmkcnal/3ZpybW8ZNzpiP8nCJO3ASz9kXRC3cITS0zgKxl6USDZs+8 +NAM6R0r8icB89L+i8bOfbyU7nkN9T+xUTTOmalSmsYrMIedIBmcB7NuqbXPLEpeJ +AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAA4U/uhswbeJB0gX4vfVqYf30A131Rvu +J4eaVrVLzuByP1R0MvbBCMMYZBlDVDhiFqRh4KdoVWBvTfxf/4McYZ1FhXkgRlOb +mv/mxVBqnXEu5msviApYmoLzMqgd91F3T4CWs66QIWVTJYh2McRKLG0+IfGp3aox +YKC/W2RPsUO2fKFnUDkYetXMuWg1KJYKuqE6u2lcoV3uHFphXplClnlwN+IwtWWY +cgfNBBRpwx6RXTk2XXgpCKYRBthBu1rowp7qiAwX7R5am6wDx0EIbevfR32bDReX +oAV8c9soJWwAUwH63jqq7KTO8Dg1oGHveZMk4HHGkCqZeGCjbDPaak4= +-----END CERTIFICATE----- diff --git a/src/test/resources/mail/key.pem b/src/test/resources/mail/key.pem new file mode 100644 index 00000000000..84d34efdce8 --- /dev/null +++ b/src/test/resources/mail/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzQ55QKM/sVJMb +9c5MKtc/YW3+MlCrCnGlo42DCjl6noZg8Gji4dOEMo29UcRtYqhOsx7HOXZ5ulj3 +YKiBfzht+QV/ZofhMIN9F/N5XCi4MRPorFz+mPck5NDzH1SqYn5zGm5APPqFJlwB +WxDKEfqe6ir5gG91MzHHuJJSQq3nrSDq+/DXRwg/7L2O7da6pBqti7nYU0T5ql88 +nddkRhR87NdeZndI+UVmkcnal/3ZpybW8ZNzpiP8nCJO3ASz9kXRC3cITS0zgKxl +6USDZs+8NAM6R0r8icB89L+i8bOfbyU7nkN9T+xUTTOmalSmsYrMIedIBmcB7Nuq +bXPLEpeJAgMBAAECggEAQ3h3TQ9XVslsRxFIsLVNJ49JoWuZng7DwIai3AfMo4Cn +7jN+HqrFfBO08mUkq9D+rQRQ2MYhd+Zx1sXcFkVmXUnlTlKuYMzsKHiLzIkp0E20 +gxXguHilSI8Qr/kCWlDQ7AyuI2JwHg5WgbIfSxbiP86+FwNGsBNxMI0hEXIEV1ZY +OFXO6AWO63D4zwbwMT30k8cjfyjGvjEtoGmjnBJcrJLSADCIWLcFCw+Cm8vcRkCd +BEpfRzeEos/NVdOqCpi1ea3OkGAY94mXxz6gaFRbeJFj9b6st7oVZLBOiMx1eafH +hgB9JkfVtDogl9B13MkqRN8WAiOgAjIo2Ukq8x1ZkwKBgQD88sdh8k1eldO9UXG1 +BjEsB2mEnzp1hvjuRlMQtnvOjDakbqozzbNQlq9YJxocphLyUPM/BKTsIGp0SPpd +vo0lgspDJ5eLnHd/Xf/guYvKg90NsHZR6V7hf9Z4JcrwrwvXpf7Lp/m95Jwd930j +/kPXw25gRFmpJ8Q9ciIk0PF0NwKBgQC1bUTK8iarZHhDGnR+/AhjkfSnb0z725Qb +w7MYRvicRNWT0wnk3njMMfXYS0rbxw7O5LlSoyCf+n6dGtHqJWCS1+lYuCjCz1vr +hMVFbpcEhob0OAhg8YMgzQRsmeJcBm8slVEOrmmVhQQZPRBjAaQw2f6cjW/ZhzZd +JHSiDw3yPwKBgQDLSleB2Zni3al56v3mzh4w05gzVUFHeX2RCoXx1ad1He1AhAxY +bAakSyaLQ4nR4osxomuMhzAA8iB8araFJwMLVa03AZfjRZIolCR0uMqnrQi42syN +EnEF7JcyorUScKyk2S0JAmxN+HCcCO7TQaPGwbNwvR4OO/6Un6jfS+nySwKBgH6n +4bashkJwyWRPO7TKzjB03I9nLB9Hk4YugQEZysWNaGzij62vgjVLS43MQl5cAQJ+ +usHuEACfJ3UWHCWSInFhOg4twob9q/YnonBuXA9UuzITTAYhlKF5fvUyGMyV0VcW +hpfxOtSfH9Vew+naY32XMiCovMTnmBQ+Nw5L5DiRAoGAV5/JT4z57Y+8npBCRr1m +NJZBXjQ8rmjYBCs+jOQ48wK2mEgcgARIgVGgi9MZZ2BUFHPThGS1o4OYE+fdqD95 +bvg1XInVpNwebLP6UZa9xZ8oGd3Auxfsav1WJB+CZo2tOX5Qt+GnwiumEr3Dlf1d +UVXDNM5A/sl1IDL3T3IEdSw= +-----END PRIVATE KEY----- From 53e964ae68b227793cde00774108adeef586eebb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 08:30:50 +0100 Subject: [PATCH 43/69] style(mail): update deprecation tags for DV v6.2 --- .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 2 +- .../edu/harvard/iq/dataverse/util/MailSessionProducer.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 45189ac6c3a..63566b62395 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -232,7 +232,7 @@ public enum Key { * @deprecated Please replace usages with {@link edu.harvard.iq.dataverse.MailServiceBean#getSystemAddress}, * which is backward compatible with this setting. */ - @Deprecated(since = "6.1", forRemoval = true) + @Deprecated(since = "6.2", forRemoval = true) SystemEmail, /* size limit for Tabular data file ingests */ /* (can be set separately for specific ingestable formats; in which diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 25f5970274e..13fedb94014 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -49,7 +49,7 @@ public class MailSessionProducer { * No direct JNDI lookup on the field to avoid deployment failures when not present. * @deprecated This should be removed with the next major release of Dataverse, as it would be a breaking change. */ - @Deprecated(forRemoval = true, since = "6.1") + @Deprecated(forRemoval = true, since = "6.2") Session appserverProvidedSession; public MailSessionProducer() { @@ -124,7 +124,7 @@ Properties getMailProperties() { * @return True if injected as resource from app server, false otherwise * @deprecated This is supposed to be removed when {@link #appserverProvidedSession} is removed. */ - @Deprecated(forRemoval = true, since = "6.1") + @Deprecated(forRemoval = true, since = "6.2") public boolean hasSessionFromAppServer() { return this.appserverProvidedSession != null; } From abcb131e79bd7f16ee8e003b8d7bf2e33f3e0259 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 08:32:11 +0100 Subject: [PATCH 44/69] style(settings): ignore SonarCube rule S115 for DB settings The DB settings names are not compliant with usual Java enum name rules. Ignoring to avoid unnecessary clutter, hiding more important problems. --- .../edu/harvard/iq/dataverse/settings/SettingsServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 63566b62395..864307d536f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -45,6 +45,7 @@ public class SettingsServiceBean { * over your shoulder when typing strings in various places of a large app. * So there. */ + @SuppressWarnings("java:S115") public enum Key { AllowApiTokenLookupViaApi, /** From b0d268d281331549f5c8b9ef17f760131686d079 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 10:39:06 +0100 Subject: [PATCH 45/69] doc(settings): add section on secure password storage in security section The section about securing your installation was missing hints about how to store and access passwords in a safe manner. Now having a single place to reference from everywhere makes the config bits for passwords much more readable, as we do not need to provide as many examples. --- .../source/installation/config.rst | 121 ++++++++++-------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c233e594fa7..32c61009524 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -88,6 +88,51 @@ See the :ref:`payara` section of :doc:`prerequisites` for details and init scrip Related to this is that you should remove ``/root/.payara/pass`` to ensure that Payara isn't ever accidentally started as root. Without the password, Payara won't be able to start as root, which is a good thing. +.. _secure-password-storage: + +Secure Password Storage +^^^^^^^^^^^^^^^^^^^^^^^ + +In development or demo scenarios, we suggest not to store passwords in files permanently. +We recommend the use of at least environment variables or production-grade mechanisms to supply passwords. + +In a production setup, permanently storing passwords as plaintext should be avoided at all cost. +Environment variables are dangerous in shared environments and containers, as they may be easily exploited; we suggest not to use them. +Depending on your deployment model and environment, you can make use of the following techniques to securely store and access passwords. + +**Password Aliases** + +A `password alias`_ allows you to have a plaintext reference to an encrypted password stored on the server, with the alias being used wherever the password is needed. +This method is especially useful in a classic deployment, as it does not require any external secrets management. + +Password aliases are consumable as a MicroProfile Config source and can be referrenced by their name in a `property expression`_. +You may also reference them within a `variable substitution`_, e.g. in your ``domain.xml``. + +Creation example for an alias named *my.alias.name*: + +.. code-block:: shell + + echo "AS_ADMIN_ALIASPASSWORD=changeme" > /tmp/p.txt + asadmin create-password-alias --passwordfile "/tmp/p.txt" "my.alias.name" + rm /tmp/p.txt + +Note: omitting the ``--passwordfile`` parameter allows creating the alias in an interactive fashion with a prompt. + +**Secrets Files** + +Payara has a builtin MicroProfile Config source to consume values from files in a directory on your filesystem. +This `directory config source`_ is most useful and secure with external secrets management in place, temporarily mounting cleartext passwords as files. +Examples are Kubernetes / OpenShift `Secrets `_ or tools like `Vault Agent `_. + +Please follow the `directory config source`_ documentation to learn about its usage. + +**Cloud Providers** + +Running Dataverse on a cloud platform or running an external secret management system like `Vault `_ enables accessing secrets without any intermediate storage of cleartext. +Obviously this is the most secure option for any deployment model, but it may require more resources to set up and maintain - your mileage may vary. + +Take a look at `cloud sources`_ shipped with Payara to learn about their usage. + Enforce Strong Passwords for User Accounts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -365,16 +410,8 @@ Basic Database Settings 1. Any of these settings can be set via system properties (see :ref:`jvm-options` starting at :ref:`dataverse.db.name`), environment variables or other MicroProfile Config mechanisms supported by the app server. `See Payara docs for supported sources `_. -2. Remember to protect your secrets. For passwords, use an environment variable (bare minimum), a password alias named the same - as the key (OK) or use the "dir config source" of Payara (best). - - Alias creation example: - - .. code-block:: shell - - echo "AS_ADMIN_ALIASPASSWORD=changeme" > /tmp/p.txt - asadmin create-password-alias --passwordfile /tmp/p.txt dataverse.db.password - rm /tmp/p.txt +2. Remember to protect your secrets. + See :ref:`secure-password-storage` for more information. 3. Environment variables follow the key, replacing any dot, colon, dash, etc. into an underscore "_" and all uppercase letters. Example: ``dataverse.db.host`` -> ``DATAVERSE_DB_HOST`` @@ -603,6 +640,8 @@ Then create a password alias by running (without changes): The second command will trigger an interactive prompt asking you to input your Swift password. +Note: you may choose a different way to secure this password, depending on your use case. See :ref:`secure-password-storage` for more options. + Second, update the JVM option ``dataverse.files.storage-driver-id`` by running the delete command: ``./asadmin $ASADMIN_OPTS delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` @@ -872,9 +911,8 @@ Optionally, you may provide static credentials for each S3 storage using MicroPr You may provide the values for these via any `supported MicroProfile Config API source`_. **WARNING:** - *For security, do not use the sources "environment variable" or "system property" (JVM option) in a production context!* -*Rely on password alias, secrets directory or cloud based sources instead!* +*Rely on password alias, secrets directory or cloud based sources as described at* :ref:`secure-password-storage` *instead!* **NOTE:** @@ -1946,15 +1984,9 @@ dataverse.db.password The PostgreSQL users password to connect with. -Preferrably use a JVM alias, as passwords in environment variables aren't safe. - -.. code-block:: shell - - echo "AS_ADMIN_ALIASPASSWORD=change-me-super-secret" > /tmp/password.txt - asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.db.password - rm /tmp/password.txt +See :ref:`secure-password-storage` to learn about options to securely store this password. -Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PASSWORD``. +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_DB_PASSWORD`` (although you shouldn't use environment variables for passwords). dataverse.db.host +++++++++++++++++ @@ -2201,14 +2233,7 @@ Once you have a username from DataCite, you can enter it like this: dataverse.pid.datacite.password +++++++++++++++++++++++++++++++ -Once you have a password from your provider, you should create a password alias. -This avoids storing it in clear text, although you could use a JVM option `to reference -a different place `__. - -``./asadmin create-password-alias dataverse.pid.datacite.password`` - -It will allow you to enter the password while not echoing the characters. -To manage these, read up on `Payara docs about password aliases `__. +Once you have a password from your provider, you should create a password alias called *dataverse.pid.datacite.password* or use another method described at :ref:`secure-password-storage` to safeguard it. **Notes:** @@ -2219,7 +2244,7 @@ To manage these, read up on `Payara docs about password aliases `. Provide a passphrase to decrypt the :ref:`private key file `. +See :ref:`secure-password-storage` for ways to do this securely. The key file may (and should) be encrypted with a passphrase (used for encryption with AES-128). See also chapter 1.4 "Authentication" of the @@ -2260,10 +2286,10 @@ encryption with AES-128). See also chapter 1.4 "Authentication" of the Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_PID_HANDLENET_KEY_PASSPHRASE`` (although you shouldn't use -environment variables for passwords). This setting was formerly known as -``dataverse.handlenet.admprivphrase`` and has been renamed. You should delete -the old JVM option and the wrapped password alias, then recreate as shown for -:ref:`dataverse.pid.datacite.password` but with this option as alias name. +environment variables for passwords). + +This setting was formerly known as ``dataverse.handlenet.admprivphrase`` and has been renamed. +You should delete the old JVM option and the wrapped password alias, then recreate as shown for :ref:`dataverse.pid.datacite.password` but with this option as alias name. .. _dataverse.pid.handlenet.index: @@ -2457,20 +2483,11 @@ The key used to sign a URL is created from the API token of the creating user pl signature-secret makes it impossible for someone who knows an API token from forging signed URLs and provides extra security by making the overall signing key longer. -Since the signature-secret is sensitive, you should treat it like a password. Here is an example how to set your shared secret -with the secure method "password alias": +**WARNING**: +*Since the signature-secret is sensitive, you should treat it like a password.* +*See* :ref:`secure-password-storage` *to learn about ways to safeguard it.* -.. code-block:: shell - - echo "AS_ADMIN_ALIASPASSWORD=change-me-super-secret" > /tmp/password.txt - asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.api.signature-secret - rm /tmp/password.txt - -Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable -``DATAVERSE_API_SIGNATURE_SECRET``. - -**WARNING:** For security, do not use the sources "environment variable" or "system property" (JVM option) in a -production context! Rely on password alias, secrets directory or cloud based sources instead! +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_SIGNATURE_SECRET`` (although you shouldn't use environment variables for passwords) . .. _dataverse.api.allow-incomplete-metadata: @@ -4147,10 +4164,7 @@ A true(default)/false option determining whether datafiles listed on the dataset :AllowUserManagementOfOrder +++++++++++++++++++++++++++ -A true/false (default) option determining whether the dataset datafile table display includes checkboxes enabling users to turn folder ordering and/or category ordering (if an order is defined by :CategoryOrder) on and off dynamically. - -.. _supported MicroProfile Config API source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html - +A true/false (default) option determining whether the dataset datafile table display includes checkboxes enabling users to turn folder ordering and/or category ordering (if an order is defined by :CategoryOrder) on and off dynamically. .. _:UseStorageQuotas: @@ -4173,3 +4187,10 @@ tab. files saved with these headers on S3 - since they no longer have to be generated and added to the streamed file on the fly. The setting is ``false`` by default, preserving the legacy behavior. + +.. _supported MicroProfile Config API source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html +.. _password alias: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Server%20Configuration%20And%20Management/Configuration%20Options/Password%20Aliases.html +.. _variable substitution: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Server%20Configuration%20And%20Management/Configuration%20Options/Variable%20Substitution/Usage%20of%20Variables.html +.. _property expression: https://download.eclipse.org/microprofile/microprofile-config-3.1/microprofile-config-spec-3.1.html#property-expressions +.. _directory config source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Directory.html +.. _cloud sources: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Cloud/Overview.html \ No newline at end of file From f690c47100f216810133ca308ea147dc6ad93ee1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 15:43:07 +0100 Subject: [PATCH 46/69] feat(installer): make installer use new way to apply mail MTA config Instead of setting a DB setting, we now simply apply system properties. Also, aligned with the way the "from" address is now bound to be the system mail address, this commit removes this subtle difference in the installer as well. --- scripts/installer/as-setup.sh | 16 ++++++++++------ scripts/installer/install.py | 8 -------- scripts/installer/installAppServer.py | 5 +++-- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/scripts/installer/as-setup.sh b/scripts/installer/as-setup.sh index fc5b378cff5..f169dfa5333 100755 --- a/scripts/installer/as-setup.sh +++ b/scripts/installer/as-setup.sh @@ -146,12 +146,10 @@ function final_setup(){ # delete any existing mail/notifyMailSession; configure port, if provided: ./asadmin delete-javamail-resource mail/notifyMailSession - - if [ $SMTP_SERVER_PORT"x" != "x" ] - then - ./asadmin $ASADMIN_OPTS create-javamail-resource --mailhost "$SMTP_SERVER" --mailuser "dataversenotify" --fromaddress "do-not-reply@${HOST_ADDRESS}" --property mail.smtp.port="${SMTP_SERVER_PORT}" mail/notifyMailSession - else - ./asadmin $ASADMIN_OPTS create-javamail-resource --mailhost "$SMTP_SERVER" --mailuser "dataversenotify" --fromaddress "do-not-reply@${HOST_ADDRESS}" mail/notifyMailSession + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.system-email='${ADMIN_EMAIL}'" + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.mta.host='${SMTP_SERVER}'" + if [ "x${SMTP_SERVER_PORT}" != "x" ]; then + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.mta.port='${SMTP_SERVER_PORT}'" fi } @@ -279,6 +277,12 @@ if [ ! -d "$DOMAIN_DIR" ] exit 2 fi +if [ -z "$ADMIN_EMAIL" ] + then + echo "You must specify the system admin email address (ADMIN_EMAIL)." + exit 1 +fi + echo "Setting up your app. server (Payara) to support Dataverse" echo "Payara directory: "$GLASSFISH_ROOT echo "Domain directory: "$DOMAIN_DIR diff --git a/scripts/installer/install.py b/scripts/installer/install.py index 18995695638..2bad29c780e 100644 --- a/scripts/installer/install.py +++ b/scripts/installer/install.py @@ -568,14 +568,6 @@ except: sys.exit("Failure to execute setup-all.sh! aborting.") -# 7b. configure admin email in the application settings -print("configuring system email address...") -returnCode = subprocess.call(["curl", "-X", "PUT", "-d", adminEmail, apiUrl+"/admin/settings/:SystemEmail"]) -if returnCode != 0: - print("\nWARNING: failed to configure the admin email in the Dataverse settings!") -else: - print("\ndone.") - # 8c. configure remote Solr location, if specified if solrLocation != "LOCAL": print("configuring remote Solr location... ("+solrLocation+")") diff --git a/scripts/installer/installAppServer.py b/scripts/installer/installAppServer.py index 698f5ba9a58..7636490c583 100644 --- a/scripts/installer/installAppServer.py +++ b/scripts/installer/installAppServer.py @@ -6,8 +6,9 @@ def runAsadminScript(config): # commands to set up all the app. server (payara6) components for the application. # All the parameters must be passed to that script as environmental # variables: - os.environ['GLASSFISH_DOMAIN'] = "domain1"; - os.environ['ASADMIN_OPTS'] = ""; + os.environ['GLASSFISH_DOMAIN'] = "domain1" + os.environ['ASADMIN_OPTS'] = "" + os.environ['ADMIN_EMAIL'] = config.get('system','ADMIN_EMAIL') os.environ['HOST_ADDRESS'] = config.get('glassfish','HOST_DNS_ADDRESS') os.environ['GLASSFISH_ROOT'] = config.get('glassfish','GLASSFISH_DIRECTORY') From 98244256529959a37b98986226ad71f4cc2b9bcf Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 15:46:12 +0100 Subject: [PATCH 47/69] doc(mail): add mail config paragraphs #7424 --- .../source/installation/config.rst | 154 ++++++++++++++++-- 1 file changed, 142 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 32c61009524..1d23f9a1277 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2520,13 +2520,37 @@ See :ref:`discovery-sign-posting` for details. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_SIGNPOSTING_LEVEL1_ITEM_LIMIT``. +.. _systemEmail: +.. _dataverse.mail.system-email: + +dataverse.mail.system-email ++++++++++++++++++++++++++++ + +This is the email address that "system" emails are sent from such as password reset links, notifications, etc. +It replaces the database setting :ref:`legacySystemEmail` since Dataverse 6.2. + +**WARNING**: Your Dataverse installation will not send mail without this setting in place. + +Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. +If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address. +In case you want your system email address to of no-reply style, have a look at :ref:`dataverse.mail.support-email` setting, too. + +Please note that if you're having any trouble sending email, you can refer to "Troubleshooting" under :doc:`installation-main`. + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SYSTEM_EMAIL``. + +.. _dataverse.mail.support-email: + dataverse.mail.support-email ++++++++++++++++++++++++++++ -This provides an email address distinct from the :ref:`systemEmail` that will be used as the email address for Contact Forms and Feedback API. This address is used as the To address when the Contact form is launched from the Support entry in the top navigation bar and, if configured via :ref:`dataverse.mail.cc-support-on-contact-email`, as a CC address when the form is launched from a Dataverse/Dataset Contact button. -This allows configuration of a no-reply email address for :ref:`systemEmail` while allowing feedback to go to/be cc'd to the support email address, which would normally accept replies. If not set, the :ref:`systemEmail` is used for the feedback API/contact form email. +This provides an email address distinct from the :ref:`systemEmail` that will be used as the email address for Contact Forms and Feedback API. +This address is used as the To address when the Contact form is launched from the Support entry in the top navigation bar and, if configured via :ref:`dataverse.mail.cc-support-on-contact-email`, as a CC address when the form is launched from a Dataverse/Dataset Contact button. +This allows configuration of a no-reply email address for :ref:`systemEmail` while allowing feedback to go to/be cc'd to the support email address, which would normally accept replies. +If not set, the :ref:`systemEmail` is used for the feedback API/contact form email. -Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address. +Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. +If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SUPPORT_EMAIL``. @@ -2535,12 +2559,123 @@ Can also be set via any `supported MicroProfile Config API source`_, e.g. the en dataverse.mail.cc-support-on-contact-email ++++++++++++++++++++++++++++++++++++++++++ -If this setting is true, the contact forms and feedback API will cc the system (:SupportEmail if set, :SystemEmail if not) when sending email to the collection, dataset, or datafile contacts. +If this boolean setting is true, the contact forms and feedback API will cc the system (``dataverse.mail.support-mail`` if set, ``dataverse.mail.system-email`` if not) when sending email to the collection, dataset, or datafile contacts. A CC line is added to the contact form when this setting is true so that users are aware that the cc will occur. The default is false. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_MAIL_CC_SUPPORT_ON_CONTACT_EMAIL``. +dataverse.mail.debug +++++++++++++++++++++ + +When this boolean setting is true, sending an email will generate more verbose logging, enabling you to analyze mail delivery malfunctions. +Defaults to ``false``. + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_MAIL_DEBUG``. + +.. _dataverse.mail.mta: + +dataverse.mail.mta.* +++++++++++++++++++++ + +The following options allow you to configure a target Mail Transfer Agent (MTA) to be used for sending emails to users. +Be advised: as the mail server connection (session) is cached once created, you need to restart Payara when applying configuration changes. + +All can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_MTA_HOST``. +(For environment variables: simply replace "." and "-" with "_" and write as all caps.) + +The following table describes the most important settings commonly used. + +.. list-table:: + :widths: 15 60 25 + :header-rows: 1 + :align: left + + * - Setting Key + - Description + - Default Value + * - ``dataverse.mail.mta.host`` + - The SMTP server to connect to. + - | *No default* + | (``smtp`` in our :ref:`Dataverse container `) + * - ``dataverse.mail.mta.port`` + - The SMTP server port to connect to. + - ``25`` + * - ``dataverse.mail.mta.auth`` + - If ``true``, attempt to authenticate the user using the AUTH command. + - ``false`` + * - ``dataverse.mail.mta.user`` + - The username to use in an AUTH command. + - *No default* + * - ``dataverse.mail.mta.password`` + - The password to use in an AUTH command. (Might be a token when using XOAUTH2 mechanism) + - *No default* + * - ``dataverse.mail.mta.allow-utf8-addresses`` + - If set to ``true``, UTF-8 strings are allowed in message headers, e.g., in addresses. + This should only be set if the mail server also supports UTF-8. + (Quoted from `Jakarta Mail Javadoc `_) + Setting to ``false`` will also make mail address validation in UI/API fail on UTF-8 chars. + - ``true`` + +**WARNING**: +*For security of your password use only safe ways to store and access it.* +*See* :ref:`secure-password-storage` *to learn about your options.* + +Find below a list of even more options you can use to configure sending mails. +Detailed description for every setting can be found in the table included within the `Jakarta Mail Documentation `_. +(Simply replace ``dataverse.mail.mta.`` with ``mail.smtp.``.) + +* Timeouts: + ``dataverse.mail.mta.connectiontimeout``, + ``dataverse.mail.mta.timeout``, + ``dataverse.mail.mta.writetimeout`` +* SSL/TLS: + ``dataverse.mail.mta.starttls.enable``, + ``dataverse.mail.mta.starttls.required``, + ``dataverse.mail.mta.ssl.enable``, + ``dataverse.mail.mta.ssl.checkserveridentity``, + ``dataverse.mail.mta.ssl.trust``, + ``dataverse.mail.mta.ssl.protocols``, + ``dataverse.mail.mta.ssl.ciphersuites`` +* Proxy Connection: + ``dataverse.mail.mta.proxy.host``, + ``dataverse.mail.mta.proxy.port``, + ``dataverse.mail.mta.proxy.user``, + ``dataverse.mail.mta.proxy.password``, + ``dataverse.mail.mta.socks.host``, + ``dataverse.mail.mta.socks.port`` +* SMTP EHLO command details: + ``dataverse.mail.mta.ehlo``, + ``dataverse.mail.mta.localhost``, + ``dataverse.mail.mta.localaddress``, + ``dataverse.mail.mta.localport`` +* Authentication details: + ``dataverse.mail.mta.auth.mechanisms``, + ``dataverse.mail.mta.auth.login.disable``, + ``dataverse.mail.mta.auth.plain.disable``, + ``dataverse.mail.mta.auth.digest-md5.disable``, + ``dataverse.mail.mta.auth.ntlm.disable``, + ``dataverse.mail.mta.auth.xoauth2.disable``, + ``dataverse.mail.mta.auth.ntlm.domain``, + ``dataverse.mail.mta.auth.ntlm.flag``, + ``dataverse.mail.mta.sasl.enable``, + ``dataverse.mail.mta.sasl.usecanonicalhostname``, + ``dataverse.mail.mta.sasl.mechanisms``, + ``dataverse.mail.mta.sasl.authorizationid``, + ``dataverse.mail.mta.sasl.realm`` +* Miscellaneous: + ``dataverse.mail.mta.allow8bitmime``, + ``dataverse.mail.mta.submitter``, + ``dataverse.mail.mta.dsn.notify``, + ``dataverse.mail.mta.dsn.ret``, + ``dataverse.mail.mta.sendpartial``, + ``dataverse.mail.mta.quitwait``, + ``dataverse.mail.mta.quitonsessionreject``, + ``dataverse.mail.mta.userset``, + ``dataverse.mail.mta.noop.strict``, + ``dataverse.mail.mta.mailextension`` + + dataverse.ui.allow-review-for-incomplete ++++++++++++++++++++++++++++++++++++++++ @@ -2763,18 +2898,13 @@ In Dataverse Software 4.7 and lower, the :doc:`/api/search` required an API toke ``curl -X PUT -d true http://localhost:8080/api/admin/settings/:SearchApiRequiresToken`` -.. _systemEmail: +.. _legacySystemEmail: :SystemEmail ++++++++++++ -This is the email address that "system" emails are sent from such as password reset links. Your Dataverse installation will not send mail without this setting in place. - -``curl -X PUT -d 'LibraScholar SWAT Team ' http://localhost:8080/api/admin/settings/:SystemEmail`` - -Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address. - -Please note that if you're having any trouble sending email, you can refer to "Troubleshooting" under :doc:`installation-main`. +Please note that this setting is deprecated since Dataverse 6.2. +It will be picked up for backward compatibility, but please migrate to usage of :ref:`dataverse.mail.system-email`. :HomePageCustomizationFile ++++++++++++++++++++++++++ From 5dcaba9ee23179f43d72b91693f2591ee58a6d17 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 15:47:10 +0100 Subject: [PATCH 48/69] doc(mail): rewrite install docs to match new way of mail config #7424 --- .../source/installation/installation-main.rst | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 46c1b0b0af3..d9ae650e37a 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -157,49 +157,30 @@ If your mail host requires a username/password for access, continue to the next Mail Host Configuration & Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed in the Payara admin console or via command line. +If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed using JVM options group :ref:`dataverse.mail.mta`. -For the Payara console, load a browser with your domain online, navigate to http://localhost:4848 and on the side panel find JavaMail Sessions. By default, the Dataverse Software uses a session named mail/notifyMailSession for routing outgoing emails. Click this mail session in the window to modify it. +To enable authentication with your mail server, simply configure the following options: -When fine tuning your JavaMail Session, there are a number of fields you can edit. The most important are: +- ``dataverse.mail.mta.auth = true`` +- ``dataverse.mail.mta.username = `` +- ``dataverse.mail.mta.password`` -+ **Mail Host:** Desired mail host’s DNS address (e.g. smtp.gmail.com) -+ **Default User:** Username mail host will recognize (e.g. user\@gmail.com) -+ **Default Sender Address:** Email address that your Dataverse installation will send mail from +**WARNING**: +We strongly recommend not using plaintext storage or environment variables, but relying on :ref:`secure-password-storage`. -Depending on the SMTP server you're using, you may need to add additional properties at the bottom of the page (below "Advanced"). +**WARNING**: +It’s recommended to use an *app password* (for smtp.gmail.com users) or utilize a dedicated/non-personal user account with SMTP server auths so that you do not risk compromising your password. -From the "Add Properties" utility at the bottom, use the “Add Property” button for each entry you need, and include the name / corresponding value as needed. Descriptions are optional, but can be used for your own organizational needs. +If your installation’s mail host uses SSL (like smtp.gmail.com) you’ll need to configure these options: -**Note:** These properties are just an example. You may need different/more/fewer properties all depending on the SMTP server you’re using. +- ``dataverse.mail.mta.ssl.enable = true`` +- ``dataverse.mail.mta.port = 587`` -============================== ============================== - Name Value -============================== ============================== -mail.smtp.auth true -mail.smtp.password [Default User password*] -mail.smtp.port [Port number to route through] -============================== ============================== +**NOTE**: Some mail providers might still support using port 465, which formerly was assigned to be SMTP over SSL (SMTPS). +However, this is no longer standardized and the port has been reassigned by the IANA to a different service. +If your provider supports using port 587, be advised to migrate your configuration. -**\*WARNING**: Entering a password here will *not* conceal it on-screen. It’s recommended to use an *app password* (for smtp.gmail.com users) or utilize a dedicated/non-personal user account with SMTP server auths so that you do not risk compromising your password. - -If your installation’s mail host uses SSL (like smtp.gmail.com) you’ll need these name/value pair properties in place: - -====================================== ============================== - Name Value -====================================== ============================== -mail.smtp.socketFactory.port 465 -mail.smtp.port 465 -mail.smtp.socketFactory.fallback false -mail.smtp.socketFactory.class javax.net.ssl.SSLSocketFactory -====================================== ============================== - -The mail session can also be set from command line. To use this method, you will need to delete your notifyMailSession and create a new one. See the below example: - -- Delete: ``./asadmin delete-javamail-resource mail/notifyMailSession`` -- Create (remove brackets and replace the variables inside): ``./asadmin create-javamail-resource --mailhost [smtp.gmail.com] --mailuser [test\@test\.com] --fromaddress [test\@test\.com] --property mail.smtp.auth=[true]:mail.smtp.password=[password]:mail.smtp.port=[465]:mail.smtp.socketFactory.port=[465]:mail.smtp.socketFactory.fallback=[false]:mail.smtp.socketFactory.class=[javax.net.ssl.SSLSocketFactory] mail/notifyMailSession`` - -Be sure you save the changes made here and then restart your Payara server to test it out. +As the mail server connection (session) is cached once created, you need to restart Payara when applying configuration changes. UnknownHostException While Deploying ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From a48e860a511906296a9f43d807adc271e80bfb42 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 16:10:16 +0100 Subject: [PATCH 49/69] fix(ct): migrate compose and configbaker to use new way of mail config --- docker-compose-dev.yml | 2 ++ modules/container-configbaker/scripts/bootstrap/dev/init.sh | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 6eab84092ed..d43fce37bfc 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -14,6 +14,8 @@ services: DATAVERSE_DB_USER: ${DATAVERSE_DB_USER} ENABLE_JDWP: "1" DATAVERSE_FEATURE_API_BEARER_AUTH: "1" + DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost" + DATAVERSE_MAIL_MTA_HOST: "smtp" DATAVERSE_AUTH_OIDC_ENABLED: "1" DATAVERSE_AUTH_OIDC_CLIENT_ID: test DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8 diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh index efdaee3d0c3..f8770436652 100644 --- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh @@ -9,9 +9,6 @@ export DATAVERSE_URL echo "Running base setup-all.sh (INSECURE MODE)..." "${BOOTSTRAP_DIR}"/base/setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out -echo "Setting system mail address..." -curl -X PUT -d "dataverse@localhost" "${DATAVERSE_URL}/api/admin/settings/:SystemEmail" - echo "Setting DOI provider to \"FAKE\"..." curl "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" -X PUT -d FAKE From 6f5cc9f761e49edf2ea67caaeaf67dbc6dbde4df Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 16:13:36 +0100 Subject: [PATCH 50/69] style(mail): update mail config release note --- doc/release-notes/7424-mailsession.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 8a3aa3e956b..67e5684e569 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -1,8 +1,10 @@ ## New way to configure mail transfer agent With this release, we deprecate the usage of `asadmin create-javamail-resource` to configure your MTA. -Instead, we provide the ability to configure your SMTP mail host using JVM options with the flexibility of MicroProfile Config. +Instead, we provide the ability to configure your SMTP mail host using JVM options only, with the flexibility of MicroProfile Config. At this point, no action is required if you want to keep your current configuration. Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. + +For more details on how to configure your the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. \ No newline at end of file From 930fc1b1ddd34d9b30be2826c011f9047a2e0774 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Feb 2024 16:16:16 +0100 Subject: [PATCH 51/69] style(mail): update mail config release note about source of from address --- doc/release-notes/7424-mailsession.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 67e5684e569..25b1d39a471 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -7,4 +7,6 @@ At this point, no action is required if you want to keep your current configurat Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -For more details on how to configure your the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. \ No newline at end of file +For more details on how to configure your the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. + +Please note: as there have been problems with mails delivered to SPAM folders when "From" within mail envelope and mail session configuration mismatched, as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. \ No newline at end of file From db9cd865d9b1796d6379c0133aadc329183d83c3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 25 Mar 2024 08:38:59 +0100 Subject: [PATCH 52/69] docs(mail): apply suggestions from code review Thanks @pdurbin! Co-authored-by: Philip Durbin --- doc/release-notes/7424-mailsession.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 25b1d39a471..43846b0b72d 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -7,6 +7,6 @@ At this point, no action is required if you want to keep your current configurat Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -For more details on how to configure your the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. +For more details on how to configure the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. -Please note: as there have been problems with mails delivered to SPAM folders when "From" within mail envelope and mail session configuration mismatched, as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. \ No newline at end of file +Please note: as there have been problems with email delivered to SPAM folders when the "From" within mail envelope and the mail session configuration didn't match (#4210), as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. \ No newline at end of file From 83d29b122f89e3d902a8dc9b83938c4a702de1c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 25 Mar 2024 09:08:20 +0100 Subject: [PATCH 53/69] feat(ct): add MTA config to demo compose #7424 --- docker/compose/demo/compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 8f1af3e396b..6c2bdcf79a4 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -14,6 +14,8 @@ services: DATAVERSE_DB_PASSWORD: secret DATAVERSE_DB_USER: dataverse DATAVERSE_FEATURE_API_BEARER_AUTH: "1" + DATAVERSE_MAIL_SYSTEM_EMAIL: "Demo Dataverse " + DATAVERSE_MAIL_MTA_HOST: "smtp" JVM_ARGS: -Ddataverse.files.storage-driver-id=file1 -Ddataverse.files.file1.type=file -Ddataverse.files.file1.label=Filesystem From ac74b23a4e316e6f142e82582165d35f720604ed Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Mar 2024 14:03:10 -0400 Subject: [PATCH 54/69] removed outdated "problems sending email" section #9939 --- .../source/installation/installation-main.rst | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 5f44ef1e348..9f935db6510 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -141,19 +141,6 @@ Got ERR_ADDRESS_UNREACHABLE While Navigating on Interface or API Calls If you are receiving an ``ERR_ADDRESS_UNREACHABLE`` while navigating the GUI or making an API call, make sure the ``siteUrl`` JVM option is defined. For details on how to set ``siteUrl``, please refer to :ref:`dataverse.siteUrl` from the :doc:`config` section. For context on why setting this option is necessary, refer to :ref:`dataverse.fqdn` from the :doc:`config` section. -Problems Sending Email -^^^^^^^^^^^^^^^^^^^^^^ - -If your Dataverse installation is not sending system emails, you may need to provide authentication for your mail host. First, double check the SMTP server being used with this Payara asadmin command: - -``./asadmin get server.resources.mail-resource.mail/notifyMailSession.host`` - -This should return the DNS of the mail host you configured during or after installation. mail/notifyMailSession is the JavaMail Session that's used to send emails to users. - -If the command returns a host you don't want to use, you can modify your notifyMailSession with the Payara ``asadmin set`` command with necessary options (`click here for the manual page `_), or via the admin console at http://localhost:4848 with your domain running. - -If your mail host requires a username/password for access, continue to the next section. - Mail Host Configuration & Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From f263a4e698e63fbcc5dc5cf017cbaa4cdf44deb9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Mar 2024 18:00:11 -0400 Subject: [PATCH 55/69] update docs and release note #7424 --- doc/release-notes/7424-mailsession.md | 11 ++++++----- doc/sphinx-guides/source/installation/config.rst | 1 + .../source/installation/installation-main.rst | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 43846b0b72d..faaf618bc17 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -1,12 +1,13 @@ -## New way to configure mail transfer agent +## Simplified SMTP configuration -With this release, we deprecate the usage of `asadmin create-javamail-resource` to configure your MTA. -Instead, we provide the ability to configure your SMTP mail host using JVM options only, with the flexibility of MicroProfile Config. +With this release, we deprecate the usage of `asadmin create-javamail-resource` to configure Dataverse to send mail using your SMTP server and provide a simplified, standard alternative using JVM options or MicroProfile Config. At this point, no action is required if you want to keep your current configuration. Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -For more details on how to configure the connection to your mail provider, please find updated details within the Installation Guide's main installation and configuration section. +Please do take the opportunity to update your SMTP configuration. Details can be found in the [dataverse.mail.mta.*](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-mta) section of the Installation Guide. -Please note: as there have been problems with email delivered to SPAM folders when the "From" within mail envelope and the mail session configuration didn't match (#4210), as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. \ No newline at end of file +Once reconfiguration is complete, you should remove legacy, unused config. First, run `asadmin delete-javamail-resource mail/notifyMailSession` as described in the [6.1 guides](https://guides.dataverse.org/en/6.1/installation/installation-main.html#mail-host-configuration-authentication). Then run `curl -X DELETE http://localhost:8080/api/admin/settings/:SystemEmail` as this database setting has been replace with `dataverse.mail.system-email` as described below. + +Please note: as there have been problems with email delivered to SPAM folders when the "From" within mail envelope and the mail session configuration didn't match (#4210), as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 889cee537d0..30d0567c557 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3025,6 +3025,7 @@ Detailed description for every setting can be found in the table included within ``dataverse.mail.mta.noop.strict``, ``dataverse.mail.mta.mailextension`` +See also :ref:`mail-host-config-auth`. dataverse.ui.allow-review-for-incomplete ++++++++++++++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 9f935db6510..c20b848e1f5 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -141,6 +141,8 @@ Got ERR_ADDRESS_UNREACHABLE While Navigating on Interface or API Calls If you are receiving an ``ERR_ADDRESS_UNREACHABLE`` while navigating the GUI or making an API call, make sure the ``siteUrl`` JVM option is defined. For details on how to set ``siteUrl``, please refer to :ref:`dataverse.siteUrl` from the :doc:`config` section. For context on why setting this option is necessary, refer to :ref:`dataverse.fqdn` from the :doc:`config` section. +.. _mail-host-config-auth: + Mail Host Configuration & Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -149,7 +151,7 @@ If you need to alter your mail host address, user, or provide a password to conn To enable authentication with your mail server, simply configure the following options: - ``dataverse.mail.mta.auth = true`` -- ``dataverse.mail.mta.username = `` +- ``dataverse.mail.mta.user = `` - ``dataverse.mail.mta.password`` **WARNING**: From caf56823905a0e68ddfda2b855abc9e39a6af59e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 25 Mar 2024 18:14:37 -0400 Subject: [PATCH 56/69] link higher up in the guides #7424 --- doc/release-notes/7424-mailsession.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index faaf618bc17..37fede1bb1f 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -6,7 +6,7 @@ At this point, no action is required if you want to keep your current configurat Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -Please do take the opportunity to update your SMTP configuration. Details can be found in the [dataverse.mail.mta.*](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-mta) section of the Installation Guide. +Please do take the opportunity to update your SMTP configuration. Details can be found in section of the Installation Guide starting with the [dataverse.mail.system.email](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-system-email) section of the Installation Guide. Once reconfiguration is complete, you should remove legacy, unused config. First, run `asadmin delete-javamail-resource mail/notifyMailSession` as described in the [6.1 guides](https://guides.dataverse.org/en/6.1/installation/installation-main.html#mail-host-configuration-authentication). Then run `curl -X DELETE http://localhost:8080/api/admin/settings/:SystemEmail` as this database setting has been replace with `dataverse.mail.system-email` as described below. From 362b87e1e7dd079a28c159246daa211525fc0bb3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 13:57:15 +0100 Subject: [PATCH 57/69] fix(mail): remove duplicate JvmSettings.MAIL_MTA_HOST The setting is already covered by the "host" property string in MailSessionProducer. --- .../edu/harvard/iq/dataverse/settings/JvmSettings.java | 1 - .../java/edu/harvard/iq/dataverse/MailServiceBeanIT.java | 2 +- .../harvard/iq/dataverse/util/MailSessionProducerIT.java | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index e71cabceffe..524df1e1ce9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -195,7 +195,6 @@ public enum JvmSettings { MAIL_DEBUG(SCOPE_MAIL, "debug"), // Mail Transfer Agent settings SCOPE_MAIL_MTA(SCOPE_MAIL, "mta"), - MAIL_MTA_HOST(SCOPE_MAIL_MTA, "host"), MAIL_MTA_AUTH(SCOPE_MAIL_MTA, "auth"), MAIL_MTA_USER(SCOPE_MAIL_MTA, "user"), MAIL_MTA_PASSWORD(SCOPE_MAIL_MTA, "password"), diff --git a/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanIT.java b/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanIT.java index 08eed9fe295..17dede5e9f3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/MailServiceBeanIT.java @@ -42,7 +42,7 @@ @Testcontainers(disabledWithoutDocker = true) @ExtendWith(MockitoExtension.class) @LocalJvmSettings -@JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") +@JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpHost", varArgs = "host") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") class MailServiceBeanIT { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java index c4893652153..29b6598b1a9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MailSessionProducerIT.java @@ -71,7 +71,7 @@ static void tearDown() { @Nested @LocalJvmSettings - @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpHost", varArgs = "host") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") class WithoutAuthentication { @Container @@ -121,7 +121,7 @@ void createSession() { @Nested @LocalJvmSettings - @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpHost", varArgs = "host") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, varArgs = "ssl.enable", value = "true") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, varArgs = "ssl.trust", value = "*") @@ -183,7 +183,7 @@ void createSession() { @Nested @LocalJvmSettings - @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, method = "tcSmtpHost") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpHost", varArgs = "host") @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, method = "tcSmtpPort", varArgs = "port") @JvmSetting(key = JvmSettings.MAIL_MTA_AUTH, value = "yes") @JvmSetting(key = JvmSettings.MAIL_MTA_USER, value = username) @@ -252,7 +252,7 @@ void invalidConfigItemsAreIgnoredOnSessionBuild() { } @Test - @JvmSetting(key = JvmSettings.MAIL_MTA_HOST, value = "foobar") + @JvmSetting(key = JvmSettings.MAIL_MTA_SETTING, value = "foobar", varArgs = "host") void invalidHostnameIsFailingWhenSending() { assertDoesNotThrow(() -> new MailSessionProducer().getSession()); From b8ca4a70788943a05188945709a084156eb8f7bb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 14:05:51 +0100 Subject: [PATCH 58/69] fix(mail): do not add a default for SMPT host in ct profile As Payara 6.2023.7 still suffers from the MPCONFIG bug where a profiled setting is not easy to override, lets just remove the default for the container profile and make people add it even for containers. --- doc/sphinx-guides/source/installation/config.rst | 3 +-- src/main/resources/META-INF/microprofile-config.properties | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 30d0567c557..6d061ece384 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2946,8 +2946,7 @@ The following table describes the most important settings commonly used. - Default Value * - ``dataverse.mail.mta.host`` - The SMTP server to connect to. - - | *No default* - | (``smtp`` in our :ref:`Dataverse container `) + - *No default* * - ``dataverse.mail.mta.port`` - The SMTP server port to connect to. - ``25`` diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index 9924d2518ca..517a4e9513b 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -45,8 +45,6 @@ dataverse.rserve.tempdir=/tmp/Rserv # MAIL dataverse.mail.mta.auth=false dataverse.mail.mta.allow-utf8-addresses=true -# In containers, default to hostname smtp, a container on the same network -%ct.dataverse.mail.mta.host=smtp # OAI SERVER dataverse.oai.server.maxidentifiers=100 From d8198b53c3c92af2e91fe6e1df65af791b356b77 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 14:40:11 +0100 Subject: [PATCH 59/69] style(mail): enable more debug output from session producer In case people want to debug Jakarta Mail, they activate dataverse.mail.debug. Let's hook into that and add more verbose output from the session producer, too. That way people can make sure everything is set up as they wish. --- .../dataverse/util/MailSessionProducer.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 13fedb94014..149f92761d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -16,6 +16,7 @@ import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; @ApplicationScoped public class MailSessionProducer { @@ -42,6 +43,12 @@ public class MailSessionProducer { private static final String PREFIX = "mail.smtp."; private static final Logger logger = Logger.getLogger(MailSessionProducer.class.getCanonicalName()); + static { + if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookup(Boolean.class))) { + logger.setLevel(Level.FINE); + } + } + Session systemMailSession; /** @@ -60,7 +67,7 @@ public MailSessionProducer() { } catch (NamingException e) { // This exception simply means the appserver did not provide the legacy mail session. // Debug level output is just fine. - logger.log(Level.FINE, "Error during mail resource lookup", e); + logger.log(Level.FINER, "Error during legacy appserver-level mail resource lookup", e); } } @@ -75,14 +82,21 @@ public Session getSession() { } if (systemMailSession == null) { + logger.fine("Setting up new mail session"); + // Initialize with null (= no authenticator) is a valid argument for the session factory method. Authenticator authenticator = null; // In case we want auth, create an authenticator (default = false from microprofile-config.properties) - if (JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class)) { + if (Boolean.TRUE.equals(JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class))) { + logger.fine("Mail Authentication is enabled, building authenticator"); authenticator = new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { + logger.fine(() -> + String.format("Returning PasswordAuthenticator with username='%s', password='%s'", + JvmSettings.MAIL_MTA_USER.lookup(), + "*".repeat(JvmSettings.MAIL_MTA_PASSWORD.lookup().length()))); return new PasswordAuthentication(JvmSettings.MAIL_MTA_USER.lookup(), JvmSettings.MAIL_MTA_PASSWORD.lookup()); } }; @@ -116,6 +130,10 @@ Properties getMailProperties() { prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(Integer.class, prop).ifPresent( number -> configuration.put(PREFIX + prop, number.toString()))); + logger.fine(() -> "Compiled properties:" + configuration.entrySet().stream() + .map(entry -> "\"" + entry.getKey() + "\": \"" + entry.getValue() + "\"") + .collect(Collectors.joining(",\n"))); + return configuration; } From 2a73426d87c755f97ca9cc9af6c80f2a3f2347be Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 14:58:51 +0100 Subject: [PATCH 60/69] fix(mail): do not fail to deploy when debugging is not configured --- .../java/edu/harvard/iq/dataverse/util/MailSessionProducer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 149f92761d2..202772201de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -44,7 +44,7 @@ public class MailSessionProducer { private static final Logger logger = Logger.getLogger(MailSessionProducer.class.getCanonicalName()); static { - if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookup(Boolean.class))) { + if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookupOptional(Boolean.class).orElse(false))) { logger.setLevel(Level.FINE); } } From 21aa73d31ef8cd96b8b32f3e8de140352c120ef3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 15:00:34 +0100 Subject: [PATCH 61/69] style(mail): applying better fix for default value of mail debugging --- .../java/edu/harvard/iq/dataverse/util/MailSessionProducer.java | 2 +- src/main/resources/META-INF/microprofile-config.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java index 202772201de..149f92761d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -44,7 +44,7 @@ public class MailSessionProducer { private static final Logger logger = Logger.getLogger(MailSessionProducer.class.getCanonicalName()); static { - if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookupOptional(Boolean.class).orElse(false))) { + if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookup(Boolean.class))) { logger.setLevel(Level.FINE); } } diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index 517a4e9513b..b0bc92cf975 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -43,6 +43,7 @@ dataverse.rserve.password=rserve dataverse.rserve.tempdir=/tmp/Rserv # MAIL +dataverse.mail.debug=false dataverse.mail.mta.auth=false dataverse.mail.mta.allow-utf8-addresses=true From 36193714c14de8b13e33cfe0dfeabf8c730e1fe2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 26 Mar 2024 10:37:25 -0400 Subject: [PATCH 62/69] fix dot to dash --- doc/release-notes/7424-mailsession.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 37fede1bb1f..470b78cf2de 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -6,7 +6,7 @@ At this point, no action is required if you want to keep your current configurat Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -Please do take the opportunity to update your SMTP configuration. Details can be found in section of the Installation Guide starting with the [dataverse.mail.system.email](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-system-email) section of the Installation Guide. +Please do take the opportunity to update your SMTP configuration. Details can be found in section of the Installation Guide starting with the [dataverse.mail.system-email](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-system-email) section of the Installation Guide. Once reconfiguration is complete, you should remove legacy, unused config. First, run `asadmin delete-javamail-resource mail/notifyMailSession` as described in the [6.1 guides](https://guides.dataverse.org/en/6.1/installation/installation-main.html#mail-host-configuration-authentication). Then run `curl -X DELETE http://localhost:8080/api/admin/settings/:SystemEmail` as this database setting has been replace with `dataverse.mail.system-email` as described below. From c498cebb31783f242f025d829bfdb11a2c46e79a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 16:07:10 +0100 Subject: [PATCH 63/69] doc(mail): add ssl.enable setting to shortlist Also add notes about common ports in use. --- doc/sphinx-guides/source/installation/config.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 6d061ece384..25afbcc8fff 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2948,8 +2948,11 @@ The following table describes the most important settings commonly used. - The SMTP server to connect to. - *No default* * - ``dataverse.mail.mta.port`` - - The SMTP server port to connect to. + - The SMTP server port to connect to. (Common are ``25`` for plain, ``587`` for SSL, ``465`` for legacy SSL) - ``25`` + * - ``dataverse.mail.mta.ssl.enable`` + - Enable if your mail provider uses SSL. + - ``false`` * - ``dataverse.mail.mta.auth`` - If ``true``, attempt to authenticate the user using the AUTH command. - ``false`` @@ -2981,7 +2984,6 @@ Detailed description for every setting can be found in the table included within * SSL/TLS: ``dataverse.mail.mta.starttls.enable``, ``dataverse.mail.mta.starttls.required``, - ``dataverse.mail.mta.ssl.enable``, ``dataverse.mail.mta.ssl.checkserveridentity``, ``dataverse.mail.mta.ssl.trust``, ``dataverse.mail.mta.ssl.protocols``, From 3e9d992a9abdeae80a796a96ff93246e2817119f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 16:11:24 +0100 Subject: [PATCH 64/69] doc(mail): add newly added settings to release note --- doc/release-notes/7424-mailsession.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index 470b78cf2de..f67dbd6efc5 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -11,3 +11,14 @@ Please do take the opportunity to update your SMTP configuration. Details can be Once reconfiguration is complete, you should remove legacy, unused config. First, run `asadmin delete-javamail-resource mail/notifyMailSession` as described in the [6.1 guides](https://guides.dataverse.org/en/6.1/installation/installation-main.html#mail-host-configuration-authentication). Then run `curl -X DELETE http://localhost:8080/api/admin/settings/:SystemEmail` as this database setting has been replace with `dataverse.mail.system-email` as described below. Please note: as there have been problems with email delivered to SPAM folders when the "From" within mail envelope and the mail session configuration didn't match (#4210), as of this version the sole source for the "From" address is the setting `dataverse.mail.system-email` once you migrate to the new way of configuration. + +List of options added: +- dataverse.mail.system-email +- dataverse.mail.mta.host +- dataverse.mail.mta.port +- dataverse.mail.mta.ssl.enable +- dataverse.mail.mta.auth +- dataverse.mail.mta.user +- dataverse.mail.mta.password +- dataverse.mail.mta.allow-utf8-addresses +- Plus many more for advanced usage and special provider requirements. See [configuration guide for a full list](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-mta). \ No newline at end of file From 785dfc5251d8544ba2681eb4a1d82856baebe1e3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 16:21:09 +0100 Subject: [PATCH 65/69] chore(build): update Maven and test framework dependencies --- modules/dataverse-parent/pom.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index a15575e6e50..db8b3186efc 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -168,11 +168,11 @@ 5.2.0 - 1.19.1 - 3.4.1 - 5.10.0 - 5.6.0 - 0.8.10 + 1.19.7 + 3.7.1 + 5.10.2 + 5.11.0 + 0.8.11 9.3 @@ -182,8 +182,8 @@ 3.3.2 3.5.0 3.1.1 - 3.1.0 - 3.1.0 + 3.2.5 + 3.2.5 3.6.0 3.3.1 3.0.0-M7 @@ -199,7 +199,7 @@ 1.7.0 - 0.43.4 + 0.44.0 From 6b8b90743e2349e90ee9f3ff9f4597dc572fab0f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 16:44:37 +0100 Subject: [PATCH 66/69] chore(build): downgrade DMP to 0.43.4 We need to downgrade to 0.43.4 again because of this regression: fabric8io/docker-maven-plugin#1756 Once they release a new version, try again. --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index db8b3186efc..1a538905a8d 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.44.0 + 0.43.4 From df4838241914dcc5320a4ad81f2798fe094878bb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 26 Mar 2024 12:05:58 -0400 Subject: [PATCH 67/69] simply smtp config docs #7424 --- .../source/installation/config.rst | 25 +++++++++++++++- .../source/installation/installation-main.rst | 30 ------------------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 25afbcc8fff..28b549ec765 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -718,6 +718,19 @@ To enable bearer tokens, you must install and configure Keycloak (for now, see : You can test that bearer tokens are working by following the example under :ref:`bearer-tokens` in the API Guide. +.. _smtp-config: + +SMTP/Email Configuration +------------------------ + +The installer prompts you for some basic options to configure Dataverse to send email using your SMTP server, but in many cases, extra configuration may be necessary. + +Make sure the :ref:`dataverse.mail.support-email` has been set. Email will not be sent without it. + +Then check the list of commonly used settings at the top of :ref:`dataverse.mail.mta`. + +If you have trouble, consider turning on debugging with :ref:`dataverse.mail.debug`. + .. _database-persistence: Database Persistence @@ -2889,6 +2902,8 @@ Please note that if you're having any trouble sending email, you can refer to "T Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SYSTEM_EMAIL``. +See also :ref:`smtp-config`. + .. _dataverse.mail.support-email: dataverse.mail.support-email @@ -2904,6 +2919,8 @@ If you don't include the text, the installation name (see :ref:`Branding Your In Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SUPPORT_EMAIL``. +See also :ref:`smtp-config`. + .. _dataverse.mail.cc-support-on-contact-email: dataverse.mail.cc-support-on-contact-email @@ -2915,6 +2932,10 @@ The default is false. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_MAIL_CC_SUPPORT_ON_CONTACT_EMAIL``. +See also :ref:`smtp-config`. + +.. _dataverse.mail.debug: + dataverse.mail.debug ++++++++++++++++++++ @@ -2923,6 +2944,8 @@ Defaults to ``false``. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_MAIL_DEBUG``. +See also :ref:`smtp-config`. + .. _dataverse.mail.mta: dataverse.mail.mta.* @@ -3026,7 +3049,7 @@ Detailed description for every setting can be found in the table included within ``dataverse.mail.mta.noop.strict``, ``dataverse.mail.mta.mailextension`` -See also :ref:`mail-host-config-auth`. +See also :ref:`smtp-config`. dataverse.ui.allow-review-for-incomplete ++++++++++++++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index c20b848e1f5..3c3376e3c85 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -141,36 +141,6 @@ Got ERR_ADDRESS_UNREACHABLE While Navigating on Interface or API Calls If you are receiving an ``ERR_ADDRESS_UNREACHABLE`` while navigating the GUI or making an API call, make sure the ``siteUrl`` JVM option is defined. For details on how to set ``siteUrl``, please refer to :ref:`dataverse.siteUrl` from the :doc:`config` section. For context on why setting this option is necessary, refer to :ref:`dataverse.fqdn` from the :doc:`config` section. -.. _mail-host-config-auth: - -Mail Host Configuration & Authentication -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed using JVM options group :ref:`dataverse.mail.mta`. - -To enable authentication with your mail server, simply configure the following options: - -- ``dataverse.mail.mta.auth = true`` -- ``dataverse.mail.mta.user = `` -- ``dataverse.mail.mta.password`` - -**WARNING**: -We strongly recommend not using plaintext storage or environment variables, but relying on :ref:`secure-password-storage`. - -**WARNING**: -It’s recommended to use an *app password* (for smtp.gmail.com users) or utilize a dedicated/non-personal user account with SMTP server auths so that you do not risk compromising your password. - -If your installation’s mail host uses SSL (like smtp.gmail.com) you’ll need to configure these options: - -- ``dataverse.mail.mta.ssl.enable = true`` -- ``dataverse.mail.mta.port = 587`` - -**NOTE**: Some mail providers might still support using port 465, which formerly was assigned to be SMTP over SSL (SMTPS). -However, this is no longer standardized and the port has been reassigned by the IANA to a different service. -If your provider supports using port 587, be advised to migrate your configuration. - -As the mail server connection (session) is cached once created, you need to restart Payara when applying configuration changes. - UnknownHostException While Deploying ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From cb144236b5469dcb6ef23f78d411ab5150c64656 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 26 Mar 2024 17:13:51 +0100 Subject: [PATCH 68/69] doc(mail): fix some typos, add hint about support in new SMTP config section --- doc/sphinx-guides/source/installation/config.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 28b549ec765..207b6acb305 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -725,7 +725,8 @@ SMTP/Email Configuration The installer prompts you for some basic options to configure Dataverse to send email using your SMTP server, but in many cases, extra configuration may be necessary. -Make sure the :ref:`dataverse.mail.support-email` has been set. Email will not be sent without it. +Make sure the :ref:`dataverse.mail.system-email` has been set. Email will not be sent without it. A hint will be logged about this fact. +If you want to separate system email from your support team's email, take a look at :ref:`dataverse.mail.support-email`. Then check the list of commonly used settings at the top of :ref:`dataverse.mail.mta`. From e784eb33848085c7e10f736db60e2ae2e8d42541 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 26 Mar 2024 12:18:12 -0400 Subject: [PATCH 69/69] point release note at new SMTP section #7424 --- doc/release-notes/7424-mailsession.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7424-mailsession.md b/doc/release-notes/7424-mailsession.md index f67dbd6efc5..67c876f7ad5 100644 --- a/doc/release-notes/7424-mailsession.md +++ b/doc/release-notes/7424-mailsession.md @@ -6,7 +6,7 @@ At this point, no action is required if you want to keep your current configurat Warnings will show in your server logs to inform and remind you about the deprecation. A future major release of Dataverse may remove this way of configuration. -Please do take the opportunity to update your SMTP configuration. Details can be found in section of the Installation Guide starting with the [dataverse.mail.system-email](https://guides.dataverse.org/en/6.2/installation/config.html#dataverse-mail-system-email) section of the Installation Guide. +Please do take the opportunity to update your SMTP configuration. Details can be found in section of the Installation Guide starting with the [SMTP/Email Configuration](https://guides.dataverse.org/en/6.2/installation/config.html#smtp-email-configuration) section of the Installation Guide. Once reconfiguration is complete, you should remove legacy, unused config. First, run `asadmin delete-javamail-resource mail/notifyMailSession` as described in the [6.1 guides](https://guides.dataverse.org/en/6.1/installation/installation-main.html#mail-host-configuration-authentication). Then run `curl -X DELETE http://localhost:8080/api/admin/settings/:SystemEmail` as this database setting has been replace with `dataverse.mail.system-email` as described below.