Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Properly handle copyrighted data #574

Merged
merged 12 commits into from
Feb 21, 2025
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import no.ndla.audioapi.model.{api, domain}
import no.ndla.common.CirceUtil
import no.ndla.common.implicits.*
import no.ndla.language.Language.AllLanguages
import no.ndla.mapping.License
import no.ndla.search.Elastic4sClient

import scala.concurrent.ExecutionContext.Implicits.global
Expand Down Expand Up @@ -82,8 +83,9 @@ trait AudioSearchService {
): Try[domain.SearchResult[api.AudioSummaryDTO]] = {

val licenseFilter = settings.license match {
case None => Some(boolQuery().not(termQuery("license", "copyrighted")))
case Some(lic) => Some(termQuery("license", lic))
case None => Some(boolQuery().not(termQuery("license", License.Copyrighted.toString)))
case Some("all") => None
case Some(lic) => Some(termQuery("license", lic))
}

val seriesEpisodeFilter = settings.seriesFilter match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import no.ndla.audioapi.{TestEnvironment, UnitSuite}
import no.ndla.common.model.NDLADate
import no.ndla.common.model.domain.article.Copyright
import no.ndla.common.model.domain.{Author, Tag, Title}
import no.ndla.mapping.License
import no.ndla.tapirtesting.TapirControllerTest
import org.mockito.Mockito.when
import sttp.client3.quick.*
Expand All @@ -31,7 +32,16 @@ class HealthControllerTest extends UnitSuite with TestEnvironment with TapirCont
val created: NDLADate = NDLADate.of(2017, 3, 1, 12, 15, 32)

val copyrighted: Copyright =
Copyright("copyrighted", Some("New York"), Seq(Author("Forfatter", "Clark Kent")), Seq(), Seq(), None, None, false)
Copyright(
License.Copyrighted.toString,
Some("New York"),
Seq(Author("Forfatter", "Clark Kent")),
Seq(),
Seq(),
None,
None,
false
)

val audioMeta: AudioMetaInformation = domain.AudioMetaInformation(
Some(1),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@

package no.ndla.audioapi.service

import no.ndla.audioapi.model.domain._
import no.ndla.audioapi.model.domain.*
import no.ndla.audioapi.model.{api, domain}
import no.ndla.audioapi.{TestEnvironment, UnitSuite}
import no.ndla.common.model.domain.article.Copyright
import no.ndla.common.model.domain.{Author, Tag, Title}
import no.ndla.common.model.{NDLADate, domain => common, api => commonApi}
import no.ndla.common.model.{NDLADate, api as commonApi, domain as common}
import no.ndla.mapping.License
import no.ndla.mapping.License.CC_BY_SA

import scala.util.Success
Expand All @@ -25,7 +26,16 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment {
val created: NDLADate = NDLADate.of(2017, 3, 1, 12, 15, 32)

val copyrighted: Copyright =
Copyright("copyrighted", Some("New York"), Seq(Author("Forfatter", "Clark Kent")), Seq(), Seq(), None, None, false)
Copyright(
License.Copyrighted.toString,
Some("New York"),
Seq(Author("Forfatter", "Clark Kent")),
Seq(),
Seq(),
None,
None,
false
)

val audioMeta: AudioMetaInformation = domain.AudioMetaInformation(
Some(1),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import no.ndla.audioapi.{TestData, TestEnvironment, UnitSuite}
import no.ndla.common.model.NDLADate
import no.ndla.common.model.domain.article.Copyright
import no.ndla.common.model.domain.{Author, Tag, Title}
import no.ndla.mapping.License
import no.ndla.scalatestsuite.IntegrationSuite
import org.mockito.ArgumentMatchers.any
import org.mockito.Mockito.when
Expand Down Expand Up @@ -53,7 +54,16 @@ class AudioSearchServiceTest
)

val copyrighted: Copyright =
Copyright("copyrighted", Some("New York"), List(Author("Forfatter", "Clark Kent")), Seq(), Seq(), None, None, false)
Copyright(
License.Copyrighted.toString,
Some("New York"),
List(Author("Forfatter", "Clark Kent")),
Seq(),
Seq(),
None,
None,
false
)

val updated1: NDLADate = NDLADate.of(2017, 4, 1, 12, 15, 32)
val updated2: NDLADate = NDLADate.of(2017, 5, 1, 12, 15, 32)
Expand Down Expand Up @@ -339,7 +349,7 @@ class AudioSearchServiceTest
searchSettings.copy(
query = Some("batmen"),
language = Some("nb"),
license = Some("copyrighted")
license = Some(License.Copyrighted.toString)
)
)
results.totalCount should be(1)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import no.ndla.common.model.domain.concept.{
WordClass
}
import no.ndla.conceptapi.integration.model.TaxonomyData
import no.ndla.mapping.License
import org.mockito.Mockito.when

import java.util.UUID
Expand Down Expand Up @@ -73,7 +74,7 @@ class DraftConceptSearchServiceTest extends IntegrationSuite(EnableElasticsearch
)

val copyrighted: DraftCopyright = DraftCopyright(
Some("copyrighted"),
Some(License.Copyrighted.toString),
Some("New York"),
List(Author("Forfatter", "Clark Kent")),
List(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import no.ndla.common.model.domain.concept.{
WordClass
}
import no.ndla.conceptapi.integration.model.TaxonomyData
import no.ndla.mapping.License
import no.ndla.search.model.domain.{Bucket, TermAggregation}
import org.mockito.Mockito.when

Expand Down Expand Up @@ -69,7 +70,7 @@ class PublishedConceptSearchServiceTest
)

val copyrighted: DraftCopyright = DraftCopyright(
Some("copyrighted"),
Some(License.Copyrighted.toString),
Some("New York"),
List(Author("Forfatter", "Clark Kent")),
List(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@

package no.ndla.draftapi.service.search

import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.ElasticDsl.*
import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery
import com.typesafe.scalalogging.StrictLogging
import no.ndla.draftapi.Props
import no.ndla.draftapi.model.api
import no.ndla.draftapi.model.api.ErrorHandling
import no.ndla.draftapi.model.domain._
import no.ndla.draftapi.model.domain.*
import no.ndla.language.Language
import no.ndla.mapping.License
import no.ndla.search.Elastic4sClient

import java.util.concurrent.Executors
Expand All @@ -34,7 +35,7 @@ trait ArticleSearchService {
class ArticleSearchService extends StrictLogging with SearchService[api.ArticleSummaryDTO] {
import props.{ElasticSearchIndexMaxResultWindow, ElasticSearchScrollKeepAlive}

private val noCopyright = boolQuery().not(termQuery("license", "copyrighted"))
private val noCopyright = boolQuery().not(termQuery("license", License.Copyrighted.toString))

override val searchIndex: String = props.DraftSearchIndex

Expand Down Expand Up @@ -79,8 +80,9 @@ trait ArticleSearchService {
else None

val licenseFilter = settings.license match {
case None => Some(noCopyright)
case Some(lic) => Some(termQuery("license", lic))
case Some("all") => None
case Some(lic) => Some(termQuery("license", lic))
case None => Some(noCopyright)
}

val idFilter = if (settings.withIdIn.isEmpty) None else Some(idsQuery(settings.withIdIn))
Expand Down
3 changes: 2 additions & 1 deletion draft-api/src/test/scala/no/ndla/draftapi/TestData.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import no.ndla.common.model.{NDLADate, api as commonApi, domain as common}
import no.ndla.draftapi.integration.{LearningPath, Title}
import no.ndla.draftapi.model.api.*
import no.ndla.draftapi.model.{api, domain}
import no.ndla.mapping.License
import no.ndla.mapping.License.{CC_BY, CC_BY_NC_SA}
import no.ndla.network.tapir.auth.Permission.{DRAFT_API_ADMIN, DRAFT_API_PUBLISH, DRAFT_API_WRITE}
import no.ndla.network.tapir.auth.TokenUser
Expand Down Expand Up @@ -54,7 +55,7 @@ object TestData {
false
)
private val copyrighted = common.draft.DraftCopyright(
Some("copyrighted"),
Some(License.Copyrighted.toString),
Some("New York"),
List(common.Author("Forfatter", "Clark Kent")),
List(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
package no.ndla.draftapi.service.search

import no.ndla.common.model.NDLADate
import no.ndla.common.model.domain._
import no.ndla.common.model.domain.draft._
import no.ndla.common.model.domain.*
import no.ndla.common.model.domain.draft.*
import no.ndla.draftapi.TestData.searchSettings
import no.ndla.draftapi._
import no.ndla.draftapi.model.domain._
import no.ndla.draftapi.*
import no.ndla.draftapi.model.domain.*
import no.ndla.language.Language
import no.ndla.mapping.License
import no.ndla.scalatestsuite.IntegrationSuite

import scala.util.Success
Expand Down Expand Up @@ -53,7 +54,7 @@ class ArticleSearchServiceTest extends IntegrationSuite(EnableElasticsearchConta
)

val copyrighted: DraftCopyright = DraftCopyright(
Some("copyrighted"),
Some(License.Copyrighted.toString),
Some("New York"),
List(Author("Forfatter", "Clark Kent")),
List(),
Expand Down Expand Up @@ -494,7 +495,7 @@ class ArticleSearchServiceTest extends IntegrationSuite(EnableElasticsearchConta
val Success(results) = articleSearchService.matchingQuery(
searchSettings.copy(
query = Some("supermann"),
license = Some("copyrighted"),
license = Some(License.Copyrighted.toString),
sort = Sort.ByTitleAsc
)
)
Expand Down Expand Up @@ -592,7 +593,7 @@ class ArticleSearchServiceTest extends IntegrationSuite(EnableElasticsearchConta
val Success(search) = articleSearchService.matchingQuery(
searchSettings.copy(
searchLanguage = Language.AllLanguages,
license = Some("copyrighted"),
license = Some(License.Copyrighted.toString),
sort = Sort.ByTitleAsc,
pageSize = 100
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ trait BaseImageController {
val includeCopyrighted: EndpointInput.Query[Boolean] =
query[Boolean]("includeCopyrighted")
.description("Return copyrighted images. May be omitted.")
.deprecated()
.default(false)
val sort: EndpointInput.Query[Option[String]] = query[Option[String]]("sort")
.description(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ trait ImageControllerV2 {
pageSize: Option[Int],
page: Option[Int],
podcastFriendly: Option[Boolean],
includeCopyrighted: Boolean,
shouldScroll: Boolean,
modelReleasedStatus: Seq[ModelReleasedStatus.Value],
user: Option[TokenUser]
Expand All @@ -102,7 +101,6 @@ trait ImageControllerV2 {
page = page,
pageSize = pageSize,
podcastFriendly = podcastFriendly,
includeCopyrighted = includeCopyrighted,
shouldScroll = shouldScroll,
modelReleased = modelReleasedStatus
)
Expand All @@ -117,7 +115,6 @@ trait ImageControllerV2 {
page = page,
pageSize = pageSize,
podcastFriendly = podcastFriendly,
includeCopyrighted = includeCopyrighted,
shouldScroll = shouldScroll,
modelReleased = modelReleasedStatus
)
Expand All @@ -140,7 +137,6 @@ trait ImageControllerV2 {
.in(language)
.in(fallback)
.in(license)
.in(includeCopyrighted)
.in(sort)
.in(pageNo)
.in(pageSize)
Expand All @@ -159,7 +155,6 @@ trait ImageControllerV2 {
language,
fallback,
license,
includeCopyrighted,
sortStr,
pageNo,
pageSize,
Expand All @@ -182,7 +177,6 @@ trait ImageControllerV2 {
pageSize,
pageNo,
podcastFriendly,
includeCopyrighted,
shouldScroll,
modelReleasedStatus,
user
Expand All @@ -204,15 +198,14 @@ trait ImageControllerV2 {
val language = searchParams.language.getOrElse(Language.AllLanguages)
val fallback = searchParams.fallback.getOrElse(false)
scrollSearchOr(searchParams.scrollId, language, user) {
val minimumSize = searchParams.minimumSize
val query = searchParams.query
val license = searchParams.license
val pageSize = searchParams.pageSize
val page = searchParams.page
val podcastFriendly = searchParams.podcastFriendly
val sort = searchParams.sort
val includeCopyrighted = searchParams.includeCopyrighted.getOrElse(false)
val shouldScroll = searchParams.scrollId.exists(InitialScrollContextKeywords.contains)
val minimumSize = searchParams.minimumSize
val query = searchParams.query
val license = searchParams.license.orElse(Option.when(searchParams.includeCopyrighted.contains(true))("all"))
val pageSize = searchParams.pageSize
val page = searchParams.page
val podcastFriendly = searchParams.podcastFriendly
val sort = searchParams.sort
val shouldScroll = searchParams.scrollId.exists(InitialScrollContextKeywords.contains)
val modelReleasedStatus =
searchParams.modelReleased.getOrElse(Seq.empty).flatMap(ModelReleasedStatus.valueOf)

Expand All @@ -226,7 +219,6 @@ trait ImageControllerV2 {
pageSize,
page,
podcastFriendly,
includeCopyrighted,
shouldScroll,
modelReleasedStatus,
user
Expand Down
Loading
Loading