PI-2584 Strip HTML from email body + handle duplicated CRNs in subject

projects/justice-email-and-delius/build.gradle.kts

@@ -18,6 +18,7 @@ dependencies {
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
     implementation(libs.azure.identity)
     implementation(libs.microsoft.graph)
+    implementation(libs.html2md)
 
     dev(project(":libs:dev-tools"))
     dev("com.unboundid:unboundid-ldapsdk")

projects/justice-email-and-delius/src/integrationTest/kotlin/uk/gov/justice/digital/hmpps/IntegrationTest.kt

@@ -13,6 +13,7 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.boot.test.mock.mockito.MockBean
 import uk.gov.justice.digital.hmpps.data.generator.Data
+import uk.gov.justice.digital.hmpps.entity.Contact
 import uk.gov.justice.digital.hmpps.entity.ContactRepository
 import uk.gov.justice.digital.hmpps.entity.ContactType.Code.EMAIL_TEXT_FROM_OTHER
 import uk.gov.justice.digital.hmpps.message.Notification
@@ -39,13 +40,9 @@ internal class IntegrationTest {
         handler.handle(notification)
         verify(telemetryService).notificationReceived(notification)
 
-        val contactId = with(argumentCaptor<Map<String, String>>()) {
-            verify(telemetryService).trackEvent(eq("CreatedContact"), capture(), any())
-            firstValue["contactId"]!!.toLong()
-        }
-        val contact = contactRepository.findById(contactId).orElseThrow()
+        val contact = verifyContactCreated()
         assertThat(contact.type.code, equalTo(EMAIL_TEXT_FROM_OTHER.code))
-        assertThat(contact.notes, equalTo("Example message"))
+        assertThat(contact.notes, equalTo("Example message\n"))
         assertThat(
             contact.externalReference,
             equalTo("urn:uk:gov:hmpps:justice-email:00000000-0000-0000-0000-000000000000")
@@ -61,11 +58,7 @@ internal class IntegrationTest {
         handler.handle(notification)
         verify(telemetryService, atLeastOnce()).notificationReceived(notification)
 
-        val contactId = with(argumentCaptor<Map<String, String>>()) {
-            verify(telemetryService).trackEvent(eq("CreatedContact"), capture(), any())
-            firstValue["contactId"]!!.toLong()
-        }
-        val contact = contactRepository.findById(contactId).orElseThrow()
+        val contact = verifyContactCreated()
         assertThat(contact.staffId, equalTo(Data.MANAGER.staffId))
     }
 
@@ -99,4 +92,48 @@ internal class IntegrationTest {
             equalTo("Email address does not end with @justice.gov.uk or @digital.justice.gov.uk")
         )
     }
+
+    @Test
+    fun `converts html to text`() {
+        val notification = Notification(
+            get<EmailMessage>("successful-message").copy(
+                bodyContent = """
+                    <p>Paragraph 1
+                    <p>Paragraph 2 with <strong>bold</strong> text</p>
+                    <ul>
+                        <li>List item 1</li>
+                        <li>List item 2
+                    </ul>
+                    Text<br/>with<br>new lines
+                """.trimIndent()
+            )
+        )
+        handler.handle(notification)
+        val contact = verifyContactCreated()
+        assertThat(
+            contact.notes, equalTo(
+                """
+                Paragraph 1
+
+                Paragraph 2 with **bold** text
+
+                * List item 1
+                * List item 2
+
+                Text  
+                with  
+                new lines
+
+            """.trimIndent()
+            )
+        )
+    }
+
+    private fun verifyContactCreated(): Contact {
+        val contactId = with(argumentCaptor<Map<String, String>>()) {
+            verify(telemetryService).trackEvent(eq("CreatedContact"), capture(), any())
+            firstValue["contactId"]!!.toLong()
+        }
+        return contactRepository.findById(contactId).orElseThrow()
+    }
 }

projects/justice-email-and-delius/src/main/kotlin/uk/gov/justice/digital/hmpps/config/HtmlToMarkdownConfig.kt

@@ -0,0 +1,11 @@
+package uk.gov.justice.digital.hmpps.config
+
+import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+class HtmlToMarkdownConfig {
+    @Bean
+    fun htmlToMarkdownConverter() = FlexmarkHtmlConverter.builder().build()
+}

projects/justice-email-and-delius/src/main/kotlin/uk/gov/justice/digital/hmpps/messaging/Handler.kt

@@ -1,5 +1,6 @@
 package uk.gov.justice.digital.hmpps.messaging
 
+import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter
 import org.openfolder.kotlinasyncapi.annotation.Schema
 import org.openfolder.kotlinasyncapi.annotation.channel.Channel
 import org.openfolder.kotlinasyncapi.annotation.channel.Message
@@ -25,6 +26,7 @@ import uk.gov.justice.digital.hmpps.telemetry.TelemetryService
 class Handler(
     auditedInteractionService: AuditedInteractionService,
     override val converter: NotificationConverter<EmailMessage>,
+    private val htmlToMarkdownConverter: FlexmarkHtmlConverter,
     private val telemetryService: TelemetryService,
     private val contactRepository: ContactRepository,
     private val contactTypeRepository: ContactTypeRepository,
@@ -52,7 +54,7 @@ class Handler(
                 type = contactTypeRepository.getByCode(EMAIL_TEXT_FROM_OTHER),
                 date = message.receivedDateTime,
                 startTime = message.receivedDateTime,
-                notes = message.bodyContent,
+                notes = htmlToMarkdownConverter.convert(message.bodyContent),
                 staffId = staffId,
                 teamId = manager.teamId,
                 providerId = manager.providerId,
@@ -71,9 +73,9 @@ class Handler(
     }
 
     private fun EmailMessage.extractCrn(): String {
-        val crns = Regex("[A-Za-z][0-9]{6}").findAll(subject)
+        val crns = Regex("[A-Za-z][0-9]{6}").findAll(subject).map { it.value }.distinct()
         return when (crns.count()) {
-            1 -> crns.single().value.uppercase()
+            1 -> crns.single().uppercase()
             0 -> throw IllegalArgumentException("No CRN in message subject")
             else -> throw IllegalArgumentException("Multiple CRNs in message subject")
         }

projects/justice-email-and-delius/src/main/kotlin/uk/gov/justice/digital/hmpps/service/MailboxService.kt

@@ -6,6 +6,7 @@ import io.opentelemetry.api.trace.SpanKind
 import io.opentelemetry.instrumentation.annotations.WithSpan
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Service
+import uk.gov.justice.digital.hmpps.message.MessageAttributes
 import uk.gov.justice.digital.hmpps.message.Notification
 import uk.gov.justice.digital.hmpps.messaging.EmailMessage
 import uk.gov.justice.digital.hmpps.publisher.NotificationPublisher
@@ -54,6 +55,7 @@ class MailboxService(
             bodyContent = body.content,
             fromEmailAddress = from.emailAddress.address,
             receivedDateTime = receivedDateTime.toZonedDateTime(),
-        )
+        ),
+        attributes = MessageAttributes("email.message.received")
     )
 }

settings.gradle.kts

@@ -81,6 +81,7 @@ dependencyResolutionManagement {
             library("azure-app-insights", "com.microsoft.azure:applicationinsights-web:3.6.2")
             library("azure-identity", "com.azure:azure-identity:1.13.3")
             library("flipt", "io.flipt:flipt-java:1.1.1")
+            library("html2md", "com.vladsch.flexmark:flexmark-html2md-converter:0.64.8")
             library("microsoft-graph", "com.microsoft.graph:microsoft-graph:6.16.0")
             library("mockito-inline", "org.mockito:mockito-inline:5.2.0")
             library("mockito-kotlin", "org.mockito.kotlin:mockito-kotlin:5.4.0")