Merge branch 'main' into feat/support-json-ld-ontology-api-3

.git-blame-ignore-revs

@@ -6,3 +6,6 @@
 
 # Scala Steward: Reformat with scalafmt 3.8.3
 a5523a0b097ee63e511bff6abf018d5ac6740288
+
+# Scala Steward: Reformat with scalafmt 3.8.4
+fb8d6ba614e93997749443355c7e59cdb12edf58

.github/workflows/dependency-graph.yml

@@ -11,6 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: sbt/setup-sbt@v1
       - uses: scalacenter/sbt-dependency-submission@v2
         with:
           ## Optional: Define the working directory of your build.

.scalafix.conf

@@ -15,6 +15,6 @@ OrganizeImports {
   importSelectorsOrder = Ascii
   importsOrder = Ascii
   preset = DEFAULT
-  removeUnused = false
+  removeUnused = true
   targetDialect = Scala3
 }

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.8.3"
+version = "3.8.4"
 runner.dialect = scala3
 maxColumn = 120
 align.preset = most

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## [31.2.1](https://github.com/dasch-swiss/dsp-api/compare/v31.2.0...v31.2.1) (2025-01-20)
+
+
+### Maintenances
+
+* Add missing sbt setup (DEV-4494) ([#3463](https://github.com/dasch-swiss/dsp-api/issues/3463)) ([e73b358](https://github.com/dasch-swiss/dsp-api/commit/e73b358b735a7cafd011fe3861d82634e8ada192))
+* Dependency updates ([#3458](https://github.com/dasch-swiss/dsp-api/issues/3458)) ([f96d579](https://github.com/dasch-swiss/dsp-api/commit/f96d5798929501ba06e16cdb1d7e49579fb492de))
+* Extract TestDspIngestClient ([#3462](https://github.com/dasch-swiss/dsp-api/issues/3462)) ([05c58a7](https://github.com/dasch-swiss/dsp-api/commit/05c58a70cbec1ea79663ea9641ae42bed978f9c4))
+* Minor dependency updates ([#3457](https://github.com/dasch-swiss/dsp-api/issues/3457)) ([cf84f2d](https://github.com/dasch-swiss/dsp-api/commit/cf84f2de7936f76b7697903780f50ebce4687146))
+* Patch dependency updates ([#3456](https://github.com/dasch-swiss/dsp-api/issues/3456)) ([2340dc1](https://github.com/dasch-swiss/dsp-api/commit/2340dc1d456f60bd75ad7ccd569693fb0aeddbbe))
+* Remove unused twirl templates ([#3465](https://github.com/dasch-swiss/dsp-api/issues/3465)) ([ba04c8b](https://github.com/dasch-swiss/dsp-api/commit/ba04c8b437c673c7876a75d728bd8a98c9782364))
+* Replace query twirl template 'getProjectAdminData' with SparqlQueryBuilder ([#3466](https://github.com/dasch-swiss/dsp-api/issues/3466)) ([10ebd2f](https://github.com/dasch-swiss/dsp-api/commit/10ebd2f085c18b9366de1d655a59621b77de67dd))
+* Update dsp-app to v11.22.5 ([#3453](https://github.com/dasch-swiss/dsp-api/issues/3453)) ([c984b8c](https://github.com/dasch-swiss/dsp-api/commit/c984b8c09be500ed5b63a440ac08b26c9e536864))
+* Update scalafix to 0.14.0 and enable remove unused imports (DEFV-4296) ([#3464](https://github.com/dasch-swiss/dsp-api/issues/3464)) ([f7cf483](https://github.com/dasch-swiss/dsp-api/commit/f7cf483bc60f1ca7bb34d293e502dd81bc078d54))
+
+
+### Bug Fixes
+
+* Add rdfs:label to knora-base resources to ensure this is present for  isSegmentOfValue properties (DEV-4505) ([#3455](https://github.com/dasch-swiss/dsp-api/issues/3455)) ([bc06166](https://github.com/dasch-swiss/dsp-api/commit/bc06166d6fba1f07ec168246a5f8bbe1aa1f16af))
+* Find the right doap when looking for by ForWhat ([#3460](https://github.com/dasch-swiss/dsp-api/issues/3460)) ([265cb99](https://github.com/dasch-swiss/dsp-api/commit/265cb998a93a6ea465200c12e1eb27440458156b))
+
+## [31.2.0](https://github.com/dasch-swiss/dsp-api/compare/v31.1.0...v31.2.0) (2025-01-08)
+
+
+### Maintenances
+
+* Minor dependency updates ([#3438](https://github.com/dasch-swiss/dsp-api/issues/3438)) ([d31a55f](https://github.com/dasch-swiss/dsp-api/commit/d31a55fc9d67052119879ad9807e2f0a9528fc0e))
+* Remove unused user argument from reloading ontology cache command ([#3447](https://github.com/dasch-swiss/dsp-api/issues/3447)) ([54033bd](https://github.com/dasch-swiss/dsp-api/commit/54033bdbfe62fd95a208052335e63c0dcddc3889))
+* Update copyright ([#3450](https://github.com/dasch-swiss/dsp-api/issues/3450)) ([23c3c6e](https://github.com/dasch-swiss/dsp-api/commit/23c3c6e7cc0b868e6bffbc9e0ab800f7b13506b1))
+
+
+### Documentation
+
+* Remove outdated statement on TEI in Standoff support ([#3452](https://github.com/dasch-swiss/dsp-api/issues/3452)) ([76f9077](https://github.com/dasch-swiss/dsp-api/commit/76f90770f214d48389bf8ba0faf7cc70aefbc16a))
+
+
+### Enhancements
+
+* Add StandoffFootnoteTag (DEV-4306, DEV-4491) ([#3443](https://github.com/dasch-swiss/dsp-api/issues/3443)) ([ed78393](https://github.com/dasch-swiss/dsp-api/commit/ed783934720bcf6e8a333e73a469551cbcbd3837))
+* Remove license and copyright from Project (DEV-4479) ([#3445](https://github.com/dasch-swiss/dsp-api/issues/3445)) ([c87233b](https://github.com/dasch-swiss/dsp-api/commit/c87233b382035a34704cc4bb9c16f59601dbe525))
+* Support JSON-LD ontology v2 change requests ([#3451](https://github.com/dasch-swiss/dsp-api/issues/3451)) ([a8b4bab](https://github.com/dasch-swiss/dsp-api/commit/a8b4bab882ba36809169c76ff83060032b53aa1c))
+
+
+### Tests
+
+* Migrate InputOntologyV2Spec to become a simple unit zio-test in the test source set ([#3449](https://github.com/dasch-swiss/dsp-api/issues/3449)) ([63aed0f](https://github.com/dasch-swiss/dsp-api/commit/63aed0f3b324bf0d43c8d772358ec3e06e8f22bd))
+
 ## [31.1.0](https://github.com/dasch-swiss/dsp-api/compare/v31.0.0...v31.1.0) (2024-12-16)
 
 

build.sbt

@@ -150,14 +150,9 @@ val customScalacOptions = Seq(
   "-deprecation",
   "-Yresolve-term-conflict:package",
   "-Wconf:src=target/.*:s", // silence TWIRL templates unused imports warnings
-  "-Wunused:imports",
-  "-Wunused:privates",
-  "-Wunused:locals",
-  "-Wunused:explicits",
-  "-Wunused:implicits",
-  "-Wunused:params",
   "-Wvalue-discard",
   "-Xmax-inlines:64",
+  "-Wunused:all",
 //  "-Xfatal-warnings",
 )
 

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   app:
-    image: daschswiss/dsp-app:v11.22.1
+    image: daschswiss/dsp-app:v11.22.5
     ports:
       - "4200:4200"
     networks:

docs/02-dsp-ontologies/knora-base.md

@@ -695,7 +695,7 @@ end positions of a substring in the text that has a particular attribute. The OW
   represents the end of a virtual (non syntactical) hierarchy.
 
 The `StandoffTag` class is not used directly in RDF data; instead, its subclasses are used. A few subclasses are
-currently provided in `standoff-onto.ttl`, and more will be added to support TEI semantics.
+currently provided in `standoff-onto.ttl`.
 Projects are able to define their own custom standoff tag classes (direct subclasses of `StandoffTag`
 or one of the standoff data type classes or subclasses of one of the standoff classes defined in `standoff-onto.ttl`).
 

docs/03-endpoints/api-v2/text/overview.md

@@ -35,6 +35,7 @@ This allows for the following markup:
     - horizontal rules
     - code blocks
     - block quotes
+    - footnotes
 - typographical markup
     - italics
     - bold

docs/03-endpoints/api-v2/text/standard-standoff.md

@@ -41,10 +41,21 @@ that are mapped to standoff classes and properties defined in the ontology:
 - `<cite>` → `standoff:StandoffCiteTag`
 - `<blockquote>` → `standoff:StandoffBlockquoteTag`
 - `<code>` → `standoff:StandoffCodeTag`
+- `<footnote content="footnote text">` → `standoff:StandoffFootnoteTag`
 
 The HTML produced by CKEditor is wrapped in an XML doctype and a pair of root tags `<text>...</text>` 
 and then sent to the DSP-API.
 The XML sent to the GUI by the DSP-API is unwrapped accordingly.
 Although the GUI supports HTML5, it is treated as if it was XHTML in strict XML notation.
 
 Text with standard standoff markup can be transformed to TEI XML as described [here](tei-xml.md).
+
+## Footnotes
+
+Footnote support is currently experimental and has some limitations:
+
+- CKE does not support footnotes out of the box. DSP-APP uses a custom build of CKE that supports footnotes.
+- The content of footnotes is not covered by the full text search.
+- The content of footnotes may contain further markup, but this will internally not be converted to standoff. 
+  For that reason, markup in footnotes can not be searched for through gravsearch
+  and hence outgoing and incoming links will not be displayed in DSP-APP.

integration/src/test/scala/org/knora/webapi/E2ESpec.scala

@@ -32,13 +32,11 @@ import org.knora.webapi.config.AppConfig
 import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.TestStartupUtils
-import org.knora.webapi.messages.store.sipimessages.SipiUploadResponse
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtocol
 import org.knora.webapi.messages.util.rdf.*
 import org.knora.webapi.routing.PekkoRoutesData
 import org.knora.webapi.routing.UnsafeZioRun
-import org.knora.webapi.testservices.FileToUpload
 import org.knora.webapi.testservices.TestClientService
 import org.knora.webapi.util.FileUtil
 import org.knora.webapi.util.LogAspect
@@ -151,9 +149,6 @@ abstract class E2ESpec
   protected def getResponseAsJsonLD(request: HttpRequest): JsonLDDocument =
     UnsafeZioRun.runOrThrow(ZIO.serviceWithZIO[TestClientService](_.getResponseJsonLD(request)))
 
-  protected def uploadToIngest(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse =
-    UnsafeZioRun.runOrThrow(ZIO.serviceWithZIO[TestClientService](_.uploadToIngest(loginToken, filesToUpload)))
-
   protected def responseToJsonLDDocument(httpResponse: HttpResponse): JsonLDDocument = {
     val responseBodyFuture: Future[String] =
       httpResponse.entity.toStrict(FiniteDuration(10L, TimeUnit.SECONDS)).map(_.data.decodeString("UTF-8"))

integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -28,14 +28,14 @@ import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
 import org.knora.webapi.core.LayersTest.DefaultTestEnvironmentWithSipi
 import org.knora.webapi.core.TestStartupUtils
-import org.knora.webapi.messages.store.sipimessages.*
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtocol
 import org.knora.webapi.messages.util.rdf.JsonLDDocument
 import org.knora.webapi.messages.util.rdf.JsonLDUtil
 import org.knora.webapi.routing.UnsafeZioRun
-import org.knora.webapi.testservices.FileToUpload
 import org.knora.webapi.testservices.TestClientService
+import org.knora.webapi.testservices.TestDspIngestClient
+import org.knora.webapi.testservices.TestDspIngestClient.UploadedFile
 import org.knora.webapi.util.LogAspect
 
 /**
@@ -171,17 +171,8 @@ abstract class ITKnoraLiveSpec
         .getOrThrowFiberFailure()
     }
 
-  protected def uploadToIngest(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse =
-    Unsafe.unsafe { implicit u =>
-      runtime.unsafe
-        .run(
-          for {
-            testClient <- ZIO.service[TestClientService]
-            result     <- testClient.uploadToIngest(loginToken, filesToUpload)
-          } yield result,
-        )
-        .getOrThrow()
-    }
+  protected def uploadToIngest(fileToUpload: java.nio.file.Path): UploadedFile =
+    UnsafeZioRun.runOrThrow(ZIO.serviceWithZIO[TestDspIngestClient](_.uploadFile(fileToUpload)))
 
   protected def responseToString(httpResponse: HttpResponse): String = {
     val responseBodyFuture: Future[String] =

integration/src/test/scala/org/knora/webapi/core/LayersTest.scala

@@ -86,14 +86,15 @@ import org.knora.webapi.testcontainers.FusekiTestContainer
 import org.knora.webapi.testcontainers.SharedVolumes
 import org.knora.webapi.testcontainers.SipiTestContainer
 import org.knora.webapi.testservices.TestClientService
+import org.knora.webapi.testservices.TestDspIngestClient
 
 object LayersTest {
 
   /**
    * The `Environment`s that we require for the tests to run - with or without Sipi
    */
   type DefaultTestEnvironmentWithoutSipi =
-    LayersLive.DspEnvironmentLive & FusekiTestContainer & TestClientService
+    LayersLive.DspEnvironmentLive & FusekiTestContainer & TestClientService & TestDspIngestClient
 
   type DefaultTestEnvironmentWithSipi =
     DefaultTestEnvironmentWithoutSipi & SipiTestContainer & DspIngestTestContainer & SharedVolumes.Volumes
@@ -153,6 +154,7 @@ object LayersTest {
     State &
     StringFormatter &
     TestClientService &
+    TestDspIngestClient &
     TriplestoreService &
     UserRestService &
     ValuesResponderV2
@@ -224,6 +226,7 @@ object LayersTest {
       StringFormatter.live,
       TapirToPekkoInterpreter.layer,
       TestClientService.layer,
+      TestDspIngestClient.layer,
       TriplestoreServiceLive.layer,
       ValuesResponderV2.layer,
     )

integration/src/test/scala/org/knora/webapi/core/TestStartupUtils.scala

@@ -9,7 +9,6 @@ import com.typesafe.scalalogging.LazyLogging
 import zio.*
 
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
-import org.knora.webapi.messages.util.KnoraSystemInstances
 import org.knora.webapi.slice.ontology.repo.service.OntologyCache
 import org.knora.webapi.store.triplestore.api.TriplestoreService
 
@@ -32,7 +31,7 @@ trait TestStartupUtils extends LazyLogging {
       tss <- ZIO.service[TriplestoreService]
       _   <- tss.resetTripleStoreContent(rdfDataObjects).timeout(480.seconds)
       _   <- ZIO.logInfo("... loading test data done.")
-      _   <- ZIO.serviceWithZIO[OntologyCache](_.loadOntologies(KnoraSystemInstances.Users.SystemUser)).orDie
+      _   <- ZIO.serviceWithZIO[OntologyCache](_.loadOntologies()).orDie
     } yield ()
 
 }

integration/src/test/scala/org/knora/webapi/e2e/v2/ValuesRouteV2E2ESpec.scala

@@ -19,6 +19,7 @@ import java.time.Instant
 import java.util.UUID
 import scala.concurrent.Await
 import scala.concurrent.duration.*
+import scala.xml.XML
 
 import dsp.errors.AssertionException
 import dsp.errors.BadRequestException
@@ -1557,6 +1558,72 @@ class ValuesRouteV2E2ESpec extends E2ESpec {
       assert(savedTextValueAsXml.contains(expectedText))
     }
 
+    "create a text value with standoff containing a footnote" in {
+      val resourceIri: IRI = AThing.iri
+      val textValueAsXml: String =
+        """|<?xml version="1.0" encoding="UTF-8"?>
+           |<text>This text has a footnote<footnote content="this is footnote content"/> containing text.</text>
+           |""".stripMargin
+
+      val propertyIri: SmartIri                     = "http://0.0.0.0:3333/ontology/0001/anything/v2#hasText".toSmartIri
+      val maybeResourceLastModDate: Option[Instant] = getResourceLastModificationDate(resourceIri, anythingUserEmail)
+
+      val jsonLDEntity = createTextValueWithStandoffRequest(
+        resourceIri = resourceIri,
+        textValueAsXml = textValueAsXml,
+        mappingIri = standardMappingIri,
+      )
+
+      val request = Post(
+        baseApiUrl + "/v2/values",
+        HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLDEntity),
+      ) ~> addCredentials(BasicHttpCredentials(anythingUserEmail, password))
+      val response: HttpResponse = singleAwaitingRequest(request)
+      assert(response.status == StatusCodes.OK, responseToString(response))
+      val responseJsonDoc: JsonLDDocument = responseToJsonLDDocument(response)
+      val valueIri: IRI                   = responseJsonDoc.body.requireStringWithValidation(JsonLDKeywords.ID, validationFun)
+      val valueType: SmartIri =
+        responseJsonDoc.body.requireStringWithValidation(JsonLDKeywords.TYPE, stringFormatter.toSmartIriWithErr)
+      valueType should ===(KnoraApiV2Complex.TextValue.toSmartIri)
+
+      val savedValue: JsonLDObject = getValue(
+        resourceIri = resourceIri,
+        maybePreviousLastModDate = maybeResourceLastModDate,
+        propertyIriForGravsearch = propertyIri,
+        propertyIriInResult = propertyIri,
+        expectedValueIri = valueIri,
+        userEmail = anythingUserEmail,
+      )
+
+      val savedTextValueAsXml = savedValue
+        .getRequiredString(KnoraApiV2Complex.TextValueAsXml)
+        .fold(msg => throw BadRequestException(msg), XML.loadString)
+      assert(savedTextValueAsXml == XML.loadString(textValueAsXml))
+    }
+
+    "not create a text value with standoff containing a footnote without content" in {
+      val resourceIri: IRI = AThing.iri
+      val textValueAsXml: String =
+        """|<?xml version="1.0" encoding="UTF-8"?>
+           |<text>
+           |   This text has a footnote<footnote /> without content.
+           |</text>
+           |""".stripMargin
+
+      val jsonLDEntity = createTextValueWithStandoffRequest(
+        resourceIri = resourceIri,
+        textValueAsXml = textValueAsXml,
+        mappingIri = standardMappingIri,
+      )
+
+      val request = Post(
+        baseApiUrl + "/v2/values",
+        HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLDEntity),
+      ) ~> addCredentials(BasicHttpCredentials(anythingUserEmail, password))
+      val response: HttpResponse = singleAwaitingRequest(request)
+      assert(response.status == StatusCodes.BadRequest)
+    }
+
     "create a TextValue from XML representing HTML with an attribute containing escaped quotes" in {
       // Create the mapping.