search-api: Include more data in the grep search

To allow us to completely skip calling the udir grep api from the
frontend/graphql we need more data! This patch includes everything(?) we
need to stop calling the grep api directly from elsewhere.

common/src/main/scala/no/ndla/common/CirceUtil.scala

@@ -16,17 +16,20 @@ import scala.util.{Failure, Try}
 object CirceUtil {
   // NOTE: Circe's `DecodingFailure` does not include a stack trace, so we wrap it in our own exception
   //       to make it more like other failures.
-  case class CirceFailure(message: String) extends RuntimeException(message)
+  case class CirceFailure(message: String, jsonString: String) extends RuntimeException(message)
   object CirceFailure {
-    def apply(reason: Throwable): Throwable = new CirceFailure(reason.getMessage).initCause(reason)
+    def apply(jsonString: String, reason: Throwable): Throwable = {
+      val message = s"${reason.getMessage}\n$jsonString"
+      new CirceFailure(message, jsonString).initCause(reason)
+    }
   }
 
   def tryParseAs[T](str: String)(implicit d: Decoder[T]): Try[T] = {
     parser
       .parse(str)
       .toTry
       .flatMap(_.as[T].toTry)
-      .recoverWith { ex => Failure(CirceFailure(ex)) }
+      .recoverWith { ex => Failure(CirceFailure(str, ex)) }
   }
 
   /** This might throw an exception! Use with care, probably only use this in tests */

search-api/src/main/scala/no/ndla/searchapi/integration/ZipUtil.scala

@@ -25,8 +25,13 @@ object ZipUtil {
         .continually(zis.read(buffer))
         .takeWhile(_ != -1)
         .foreach(fout.write(buffer, 0, _))
+
+      fout.close()
     }
 
+    zis.close()
+    fis.close()
+
     if (deleteArchive) zipFile.delete()
 
     targetDir

search-api/src/main/scala/no/ndla/searchapi/model/api/DescriptionDTO.scala

@@ -0,0 +1,27 @@
+/*
+ * Part of NDLA search-api
+ * Copyright (C) 2018 NDLA
+ *
+ * See LICENSE
+ */
+
+package no.ndla.searchapi.model.api
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+import no.ndla.language.model.LanguageField
+import sttp.tapir.Schema.annotations.description
+
+@description("Title of resource")
+case class DescriptionDTO(
+    @description("The freetext description of the resource") description: String,
+    @description("ISO 639-1 code that represents the language used in title") language: String
+) extends LanguageField[String] {
+  override def value: String    = description
+  override def isEmpty: Boolean = description.isEmpty
+}
+
+object DescriptionDTO {
+  implicit val encoder: Encoder[DescriptionDTO] = deriveEncoder
+  implicit val decoder: Decoder[DescriptionDTO] = deriveDecoder
+}

search-api/src/main/scala/no/ndla/searchapi/model/api/grep/GrepResultDTO.scala

@@ -8,19 +8,170 @@
 
 package no.ndla.searchapi.model.api.grep
 
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import io.circe.{Decoder, Encoder}
-import no.ndla.searchapi.model.api.TitleDTO
+import cats.implicits.*
+import io.circe.generic.auto.*
+import sttp.tapir.generic.auto.*
+import io.circe.syntax.*
+import io.circe.{Decoder, Encoder, Json}
+import no.ndla.language.Language
+import no.ndla.language.Language.findByLanguageOrBestEffort
+import no.ndla.search.model.LanguageValue
+import no.ndla.searchapi.model.api.{DescriptionDTO, TitleDTO}
+import no.ndla.searchapi.model.grep.{
+  GrepKjerneelement,
+  GrepKompetansemaal,
+  GrepKompetansemaalSett,
+  GrepLaererplan,
+  GrepTitle,
+  GrepTverrfagligTema
+}
+import no.ndla.searchapi.model.search.SearchableGrepElement
+import sttp.tapir.Schema
 import sttp.tapir.Schema.annotations.description
 
+import scala.util.{Success, Try}
+
 @description("Information about a single grep search result entry")
-case class GrepResultDTO(
-    @description("The grep code") code: String,
-    @description("The greps title") title: TitleDTO,
-    @description("The grep laereplan") laereplanCode: Option[String]
-)
+sealed trait GrepResultDTO {
+  @description("The grep code") val code: String
+  @description("The greps title") val title: TitleDTO
+}
 
 object GrepResultDTO {
-  implicit val encoder: Encoder[GrepResultDTO] = deriveEncoder
-  implicit val decoder: Decoder[GrepResultDTO] = deriveDecoder
+  implicit val encoder: Encoder[GrepResultDTO] = Encoder.instance[GrepResultDTO] { result =>
+    val json = result match {
+      case x: GrepKjerneelementDTO      => x.asJson
+      case x: GrepKompetansemaalDTO     => x.asJson
+      case x: GrepKompetansemaalSettDTO => x.asJson
+      case x: GrepLaererplanDTO         => x.asJson
+      case x: GrepTverrfagligTemaDTO    => x.asJson
+    }
+    // NOTE: Adding the discriminator field that scala-tsi generates in the typescript type.
+    //       Useful for guarding the type of the object in the frontend.
+    json.mapObject(_.add("type", Json.fromString(result.getClass.getSimpleName)))
+  }
+
+  implicit val decoder: Decoder[GrepResultDTO] = List[Decoder[GrepResultDTO]](
+    Decoder[GrepKjerneelementDTO].widen,
+    Decoder[GrepKompetansemaalDTO].widen,
+    Decoder[GrepKompetansemaalSettDTO].widen,
+    Decoder[GrepLaererplanDTO].widen,
+    Decoder[GrepTverrfagligTemaDTO].widen
+  ).reduceLeft(_ or _)
+
+  implicit val s: Schema[GrepResultDTO] = Schema.oneOfWrapped[GrepResultDTO]
+  def fromSearchable(searchable: SearchableGrepElement, language: String): Try[GrepResultDTO] = {
+    val titleLv = findByLanguageOrBestEffort(searchable.title.languageValues, language)
+      .getOrElse(LanguageValue(Language.DefaultLanguage, ""))
+    val title = TitleDTO(title = titleLv.value, language = titleLv.language)
+
+    searchable.domainObject match {
+      case core: GrepKjerneelement =>
+        val descriptionLvs = GrepTitle.convertTitles(core.beskrivelse.tekst.toSeq)
+        val descriptionLv: LanguageValue[String] =
+          findByLanguageOrBestEffort(descriptionLvs, language)
+            .getOrElse(LanguageValue(Language.DefaultLanguage, ""))
+        val description = DescriptionDTO(description = descriptionLv.value, language = descriptionLv.language)
+
+        Success(
+          GrepKjerneelementDTO(
+            code = core.kode,
+            title = title,
+            description = description,
+            laereplan = GrepLaererplanDTO(
+              code = core.`tilhoerer-laereplan`.kode,
+              title = TitleDTO(core.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage)
+            )
+          )
+        )
+      case goal: GrepKompetansemaal =>
+        Success(
+          GrepKompetansemaalDTO(
+            code = goal.kode,
+            title = title,
+            laereplan = GrepLaererplanDTO(
+              code = goal.`tilhoerer-laereplan`.kode,
+              title = TitleDTO(goal.`tilhoerer-laereplan`.tittel, Language.DefaultLanguage)
+            ),
+            kompetansemaalSett = GrepReferencedKompetansemaalSettDTO(
+              code = goal.`tilhoerer-kompetansemaalsett`.kode,
+              title = goal.`tilhoerer-kompetansemaalsett`.tittel
+            ),
+            tverrfagligeTemaer = goal.`tilknyttede-tverrfaglige-temaer`.map { crossTopic =>
+              GrepTverrfagligTemaDTO(
+                code = crossTopic.referanse.kode,
+                title = TitleDTO(crossTopic.referanse.tittel, Language.DefaultLanguage)
+              )
+            },
+            kjerneelementer = goal.`tilknyttede-kjerneelementer`.map { core =>
+              GrepReferencedKjerneelementDTO(
+                code = core.referanse.kode,
+                title = core.referanse.tittel
+              )
+            }
+          )
+        )
+      case goalSet: GrepKompetansemaalSett =>
+        Success(
+          GrepKompetansemaalSettDTO(
+            code = goalSet.kode,
+            title = title,
+            kompetansemaal = goalSet.kompetansemaal.map { goal =>
+              GrepReferencedKompetansemaalDTO(
+                code = goal.kode,
+                title = goal.tittel
+              )
+            }
+          )
+        )
+      case curriculum: GrepLaererplan =>
+        Success(
+          GrepLaererplanDTO(
+            code = curriculum.kode,
+            title = title
+          )
+        )
+      case crossTopic: GrepTverrfagligTema =>
+        Success(
+          GrepTverrfagligTemaDTO(
+            code = crossTopic.kode,
+            title = title
+          )
+        )
+    }
+  }
 }
+
+case class GrepReferencedKjerneelementDTO(code: String, title: String)
+case class GrepReferencedKompetansemaalDTO(code: String, title: String)
+case class GrepKjerneelementDTO(
+    code: String,
+    title: TitleDTO,
+    description: DescriptionDTO,
+    laereplan: GrepLaererplanDTO
+) extends GrepResultDTO
+case class GrepKompetansemaalDTO(
+    code: String,
+    title: TitleDTO,
+    laereplan: GrepLaererplanDTO,
+    kompetansemaalSett: GrepReferencedKompetansemaalSettDTO,
+    tverrfagligeTemaer: List[GrepTverrfagligTemaDTO],
+    kjerneelementer: List[GrepReferencedKjerneelementDTO]
+) extends GrepResultDTO
+case class GrepReferencedKompetansemaalSettDTO(
+    code: String,
+    title: String
+)
+case class GrepKompetansemaalSettDTO(
+    code: String,
+    title: TitleDTO,
+    kompetansemaal: List[GrepReferencedKompetansemaalDTO]
+) extends GrepResultDTO
+case class GrepLaererplanDTO(
+    code: String,
+    title: TitleDTO
+) extends GrepResultDTO
+case class GrepTverrfagligTemaDTO(
+    code: String,
+    title: TitleDTO
+) extends GrepResultDTO

search-api/src/main/scala/no/ndla/searchapi/model/grep/GrepElement.scala

@@ -7,26 +7,50 @@
 
 package no.ndla.searchapi.model.grep
 
+import cats.implicits.*
 import io.circe.{Decoder, Encoder}
 import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.syntax.EncoderOps
 
 sealed trait GrepElement {
   val kode: String
   def getTitle: Seq[GrepTitle]
 }
+object GrepElement {
+  implicit val decoder: Decoder[GrepElement] =
+    List[Decoder[GrepElement]](
+      Decoder[GrepKjerneelement].widen,
+      Decoder[GrepKompetansemaal].widen,
+      Decoder[GrepKompetansemaalSett].widen,
+      Decoder[GrepLaererplan].widen,
+      Decoder[GrepTverrfagligTema].widen
+    ).reduceLeft(_ or _)
+
+  implicit val encoder: Encoder[GrepElement] = Encoder.instance {
+    case x: GrepKjerneelement      => x.asJson
+    case x: GrepKompetansemaal     => x.asJson
+    case x: GrepKompetansemaalSett => x.asJson
+    case x: GrepLaererplan         => x.asJson
+    case x: GrepTverrfagligTema    => x.asJson
+  }
+}
 
 sealed trait BelongsToLaerePlan {
   val `tilhoerer-laereplan`: BelongsToObj
 }
 
-case class TitleObj(tekst: List[GrepTitle])
-object TitleObj {
-  implicit val encoder: Encoder[TitleObj] = deriveEncoder
-  implicit val decoder: Decoder[TitleObj] = deriveDecoder
+case class GrepTextObj(tekst: List[GrepTitle])
+object GrepTextObj {
+  implicit val encoder: Encoder[GrepTextObj] = deriveEncoder
+  implicit val decoder: Decoder[GrepTextObj] = deriveDecoder
 }
 
-case class GrepKjerneelement(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj)
-    extends GrepElement
+case class GrepKjerneelement(
+    kode: String,
+    tittel: GrepTextObj,
+    beskrivelse: GrepTextObj,
+    `tilhoerer-laereplan`: BelongsToObj
+) extends GrepElement
     with BelongsToLaerePlan {
   override def getTitle: Seq[GrepTitle] = tittel.tekst
 }
@@ -35,14 +59,38 @@ object GrepKjerneelement {
   implicit val decoder: Decoder[GrepKjerneelement] = deriveDecoder
 }
 
-case class BelongsToObj(kode: String)
+case class BelongsToObj(
+    kode: String,
+    tittel: String
+)
 object BelongsToObj {
   implicit val encoder: Encoder[BelongsToObj] = deriveEncoder
   implicit val decoder: Decoder[BelongsToObj] = deriveDecoder
 }
 
-case class GrepKompetansemaal(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj)
-    extends GrepElement
+case class ReferenceObj(
+    kode: String,
+    tittel: String
+)
+object ReferenceObj {
+  implicit val encoder: Encoder[ReferenceObj] = deriveEncoder
+  implicit val decoder: Decoder[ReferenceObj] = deriveDecoder
+}
+
+case class ReferenceWrapperObj(referanse: ReferenceObj)
+object ReferenceWrapperObj {
+  implicit val encoder: Encoder[ReferenceWrapperObj] = deriveEncoder
+  implicit val decoder: Decoder[ReferenceWrapperObj] = deriveDecoder
+}
+
+case class GrepKompetansemaal(
+    kode: String,
+    tittel: GrepTextObj,
+    `tilhoerer-laereplan`: BelongsToObj,
+    `tilhoerer-kompetansemaalsett`: BelongsToObj,
+    `tilknyttede-tverrfaglige-temaer`: List[ReferenceWrapperObj],
+    `tilknyttede-kjerneelementer`: List[ReferenceWrapperObj]
+) extends GrepElement
     with BelongsToLaerePlan {
   override def getTitle: Seq[GrepTitle] = tittel.tekst
 }
@@ -51,8 +99,12 @@ object GrepKompetansemaal {
   implicit val decoder: Decoder[GrepKompetansemaal] = deriveDecoder
 }
 
-case class GrepKompetansemaalSett(kode: String, tittel: TitleObj, `tilhoerer-laereplan`: BelongsToObj)
-    extends GrepElement
+case class GrepKompetansemaalSett(
+    kode: String,
+    tittel: GrepTextObj,
+    `tilhoerer-laereplan`: BelongsToObj,
+    kompetansemaal: List[ReferenceObj]
+) extends GrepElement
     with BelongsToLaerePlan {
   override def getTitle: Seq[GrepTitle] = tittel.tekst
 }
@@ -61,15 +113,21 @@ object GrepKompetansemaalSett {
   implicit val decoder: Decoder[GrepKompetansemaalSett] = deriveDecoder
 }
 
-case class GrepLaererplan(kode: String, tittel: TitleObj) extends GrepElement {
+case class GrepLaererplan(
+    kode: String,
+    tittel: GrepTextObj
+) extends GrepElement {
   override def getTitle: Seq[GrepTitle] = tittel.tekst
 }
 object GrepLaererplan {
   implicit val encoder: Encoder[GrepLaererplan] = deriveEncoder
   implicit val decoder: Decoder[GrepLaererplan] = deriveDecoder
 }
 
-case class GrepTverrfagligTema(kode: String, tittel: Seq[GrepTitle]) extends GrepElement {
+case class GrepTverrfagligTema(
+    kode: String,
+    tittel: Seq[GrepTitle]
+) extends GrepElement {
   override def getTitle: Seq[GrepTitle] = tittel
 }
 object GrepTverrfagligTema {