Skip to content

Commit

Permalink
concept-api: Add subject tag migrations for published table
Browse files Browse the repository at this point in the history
Whops forgot these
  • Loading branch information
jnatten committed Feb 6, 2025
1 parent c8cbfcb commit e6ca382
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import no.ndla.network.NdlaClient
import no.ndla.search.{BaseIndexService, Elastic4sClient}
import no.ndla.common.Clock
import no.ndla.common.configuration.BaseComponentRegistry
import no.ndla.conceptapi.db.migrationwithdependencies.V23__SubjectNameAsTags
import no.ndla.conceptapi.db.migrationwithdependencies.{V23__SubjectNameAsTags, V25__SubjectNameAsTagsPublished}
import no.ndla.database.{DBMigrator, DataSource}
import no.ndla.network.tapir.TapirApplication

Expand Down Expand Up @@ -62,7 +62,8 @@ class ComponentRegistry(properties: ConceptApiProperties)
with ConceptControllerHelpers {
override val props: ConceptApiProperties = properties
override val migrator: DBMigrator = DBMigrator(
new V23__SubjectNameAsTags(props)
new V23__SubjectNameAsTags(props),
new V25__SubjectNameAsTagsPublished(props)
)

override val dataSource: HikariDataSource = DataSource.getHikariDataSource
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Part of NDLA concept-api
* Copyright (C) 2025 NDLA
*
* See LICENSE
*
*/

package no.ndla.conceptapi.db.migration

import io.circe.parser
import io.circe.syntax.EncoderOps
import io.circe.generic.auto.*
import no.ndla.database.DocumentMigration

class V24__SplitTagsPublished extends DocumentMigration {
override val columnName: String = "document"
override val tableName: String = "publishedconceptdata"

private def convertTags(tags: List[TagsObject]): List[TagsObject] = tags.map { to =>
val splitTags = to.tags.flatMap(_.split(":")).filterNot(_.isEmpty)
to.copy(tags = splitTags)
}

override def convertColumn(document: String): String = {
val oldDocument = parser.parse(document).toTry.get
oldDocument.hcursor.downField("tags").as[Option[List[TagsObject]]].toTry.get match {
case None => document
case Some(tags) =>
val convertedTags = convertTags(tags).asJson
val newDocument = oldDocument.mapObject(_.remove("tags").add("tags", convertedTags))
newDocument.noSpaces
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Part of NDLA concept-api
* Copyright (C) 2025 NDLA
*
* See LICENSE
*
*/

package no.ndla.conceptapi.db.migrationwithdependencies

import com.typesafe.scalalogging.StrictLogging
import io.circe.{Json, parser}
import no.ndla.common.CirceUtil
import no.ndla.conceptapi.ConceptApiProperties
import no.ndla.database.DocumentMigration
import sttp.client3.quick.*
import io.circe.generic.auto.*
import io.circe.syntax.EncoderOps

class V25__SubjectNameAsTagsPublished(
properties: ConceptApiProperties,
prefetchedSubjects: Option[List[TaxonomySubject]] = None
) extends DocumentMigration
with StrictLogging {
override val columnName: String = "document"
override val tableName: String = "publishedconceptdata"

def toMap(subject: TaxonomySubject): Map[String, String] =
subject.translations
.map(t => t.language -> t.name)
.toMap
.withDefaultValue(subject.name)

lazy val subjects: List[TaxonomySubject] = prefetchedSubjects match {
case Some(value) => value
case None =>
val request = quickRequest.get(uri"${properties.TaxonomyUrl}/v1/nodes?nodeType=SUBJECT")
val response = simpleHttpClient.send(request)
CirceUtil.unsafeParseAs[List[TaxonomySubject]](response.body)
}

lazy val subjectIdToTranslationsMap: Map[String, Map[String, String]] = {
subjects.map { subject => subject.id -> toMap(subject) }.toMap
}

private def getLanguagesOfField(fieldName: String, json: Json): List[String] = {
json.hcursor.downField(fieldName).as[Option[List[LanguageObject]]].toTry.get match {
case Some(languageObjects) if languageObjects.nonEmpty => languageObjects.map(_.language)
case _ => List.empty
}
}

def getLanguages(json: Json): List[String] = {
val fields = List("title", "content", "tags", "visualElement", "metaImage")
fields.flatMap(field => getLanguagesOfField(field, json)).distinct
}

def getTags(json: Json): List[TagsObject] = {
json.hcursor.downField("tags").as[Option[List[TagsObject]]].toTry.get.getOrElse(List.empty)
}

override def convertColumn(document: String): String = {
val oldDocument = parser.parse(document).toTry.get
val languages = getLanguages(oldDocument)
val existingTags = getTags(oldDocument)
oldDocument.hcursor.downField("subjectIds").as[Option[List[String]]].toTry.get match {
case Some(subjectIds) if subjectIds.nonEmpty =>
val newTags = subjectIds.foldLeft(existingTags) { case (accTags, sid) =>
if (subjectIdToTranslationsMap.contains(sid)) {
val sidTranslations = subjectIdToTranslationsMap(sid)
languages.map { lang =>
val tr = sidTranslations(lang)
val t = accTags.find(_.language == lang).getOrElse(TagsObject(List.empty, lang))
t.copy(tags = t.tags :+ tr)
}
} else {
logger.error(s"Subject with id '$sid' not found when running '${getClass.getSimpleName}'")
accTags
}
}

oldDocument.mapObject {
case o if !o.contains("tags") => o
case o => o.remove("tags").add("tags", newTags.asJson)
}.noSpaces

case _ => document
}
}
}

0 comments on commit e6ca382

Please sign in to comment.