diff --git a/database/src/main/scala/no/ndla/database/TableMigration.scala b/database/src/main/scala/no/ndla/database/TableMigration.scala index d62f1d65a..748390212 100644 --- a/database/src/main/scala/no/ndla/database/TableMigration.scala +++ b/database/src/main/scala/no/ndla/database/TableMigration.scala @@ -9,11 +9,12 @@ package no.ndla.database import org.flywaydb.core.api.migration.{BaseJavaMigration, Context} -import scalikejdbc.{DB, DBSession, *} +import scalikejdbc.* abstract class TableMigration[ROW_DATA] extends BaseJavaMigration { val tableName: String val whereClause: SQLSyntax + val chunkSize: Int = 1000 def extractRowData(rs: WrappedResultSet): ROW_DATA def updateRow(rowData: ROW_DATA)(implicit session: DBSession): Int lazy val tableNameSQL: SQLSyntax = SQLSyntax.createUnsafely(tableName) @@ -25,7 +26,7 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration { } private def allRows(offset: Long)(implicit session: DBSession): Seq[ROW_DATA] = { - sql"select * from $tableNameSQL where $whereClause order by id limit 1000 offset $offset" + sql"select * from $tableNameSQL where $whereClause order by id limit $chunkSize offset $offset" .map(rs => extractRowData(rs)) .list() } @@ -36,11 +37,11 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration { private def migrateRows(implicit session: DBSession): Unit = { val count = countAllRows.get - var numPagesLeft = (count / 1000) + 1 + var numPagesLeft = (count / chunkSize) + 1 var offset = 0L while (numPagesLeft > 0) { - allRows(offset * 1000).map { rowData => updateRow(rowData) }: Unit + allRows(offset * chunkSize).map { rowData => updateRow(rowData) }: Unit numPagesLeft -= 1 offset += 1 } diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala index be99ee007..34aeb92da 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala @@ -21,7 +21,8 @@ import no.ndla.myndlaapi.controller.{ SwaggerDocControllerConfig, UserController } -import no.ndla.myndlaapi.integration.SearchApiClient +import no.ndla.myndlaapi.db.migrationwithdependencies.V16__MigrateResourcePaths +import no.ndla.myndlaapi.integration.{SearchApiClient, TaxonomyApiClient} import no.ndla.myndlaapi.integration.nodebb.NodeBBClient import no.ndla.myndlaapi.repository.{ArenaRepository, ConfigRepository, FolderRepository, UserRepository} import no.ndla.myndlaapi.service.{ @@ -70,37 +71,41 @@ class ComponentRegistry(properties: MyNdlaApiProperties) with NodeBBClient with InternController with SearchApiClient + with TaxonomyApiClient + with V16__MigrateResourcePaths with NdlaClient { override val props: MyNdlaApiProperties = properties - lazy val healthController: TapirHealthController = new TapirHealthController - lazy val clock: SystemClock = new SystemClock - lazy val migrator: DBMigrator = DBMigrator() - lazy val folderController: FolderController = new FolderController - lazy val feideApiClient: FeideApiClient = new FeideApiClient - lazy val redisClient = new RedisClient(props.RedisHost, props.RedisPort) - lazy val folderRepository: FolderRepository = new FolderRepository - lazy val folderConverterService: FolderConverterService = new FolderConverterService - lazy val folderReadService: FolderReadService = new FolderReadService - lazy val folderWriteService: FolderWriteService = new FolderWriteService - lazy val userRepository: UserRepository = new UserRepository - lazy val userService: UserService = new UserService - lazy val userController: UserController = new UserController - lazy val configRepository: ConfigRepository = new ConfigRepository - lazy val configService: ConfigService = new ConfigService - lazy val configController: ConfigController = new ConfigController - lazy val statsController: StatsController = new StatsController - lazy val arenaRepository: ArenaRepository = new ArenaRepository - lazy val arenaReadService: ArenaReadService = new ArenaReadService - lazy val arenaController: ArenaController = new ArenaController - lazy val converterService: ConverterService = new ConverterService - lazy val importService: ImportService = new ImportService - lazy val nodebb: NodeBBClient = new NodeBBClient - lazy val internController: InternController = new InternController - lazy val searchApiClient: SearchApiClient = new SearchApiClient - lazy val ndlaClient: NdlaClient = new NdlaClient - lazy val myndlaApiClient: MyNDLAApiClient = new MyNDLAApiClient + lazy val healthController: TapirHealthController = new TapirHealthController + lazy val clock: SystemClock = new SystemClock + lazy val folderController: FolderController = new FolderController + lazy val feideApiClient: FeideApiClient = new FeideApiClient + lazy val redisClient = new RedisClient(props.RedisHost, props.RedisPort) + lazy val folderRepository: FolderRepository = new FolderRepository + lazy val folderConverterService: FolderConverterService = new FolderConverterService + lazy val folderReadService: FolderReadService = new FolderReadService + lazy val folderWriteService: FolderWriteService = new FolderWriteService + lazy val userRepository: UserRepository = new UserRepository + lazy val userService: UserService = new UserService + lazy val userController: UserController = new UserController + lazy val configRepository: ConfigRepository = new ConfigRepository + lazy val configService: ConfigService = new ConfigService + lazy val configController: ConfigController = new ConfigController + lazy val statsController: StatsController = new StatsController + lazy val arenaRepository: ArenaRepository = new ArenaRepository + lazy val arenaReadService: ArenaReadService = new ArenaReadService + lazy val arenaController: ArenaController = new ArenaController + lazy val converterService: ConverterService = new ConverterService + lazy val importService: ImportService = new ImportService + lazy val nodebb: NodeBBClient = new NodeBBClient + lazy val internController: InternController = new InternController + lazy val searchApiClient: SearchApiClient = new SearchApiClient + lazy val taxonomyApiClient: TaxonomyApiClient = new TaxonomyApiClient + lazy val ndlaClient: NdlaClient = new NdlaClient + lazy val myndlaApiClient: MyNDLAApiClient = new MyNDLAApiClient + lazy val v16__MigrateResourcePaths: V16__MigrateResourcePaths = new V16__MigrateResourcePaths + override val migrator: DBMigrator = DBMigrator(v16__MigrateResourcePaths) override val dataSource: HikariDataSource = DataSource.getHikariDataSource DataSource.connectToDatabase() diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala new file mode 100644 index 000000000..d99805384 --- /dev/null +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala @@ -0,0 +1,42 @@ +/* + * Part of NDLA myndla-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + */ + +package no.ndla.myndlaapi.db.migrationwithdependencies + +import no.ndla.database.TableMigration +import no.ndla.myndlaapi.integration.TaxonomyApiClient +import no.ndla.network.NdlaClient +import scalikejdbc.{DBSession, WrappedResultSet, scalikejdbcSQLInterpolationImplicitDef} + +import java.util.UUID + +trait V16__MigrateResourcePaths { + this: TaxonomyApiClient & NdlaClient => + + class V16__MigrateResourcePaths extends TableMigration[ResourceRow] { + override val tableName: String = "resources" + override val whereClause: scalikejdbc.SQLSyntax = sqls"path is not null" + override val chunkSize: Int = 1000 + override def extractRowData(rs: WrappedResultSet): ResourceRow = ResourceRow( + UUID.fromString(rs.string("id")), + rs.string("resource_type"), + rs.string("path") + ) + override def updateRow(rowData: ResourceRow)(implicit session: DBSession): Int = { + rowData.resourceType match { + case "article" | "learningpath" | "multidisciplinary" | "topic" => + taxonomyApiClient + .resolveUrl(rowData.path) + .map { path => sql"update resources set path=$path where id = ${rowData.id}".update() } + .get + case _ => 0 + } + } + } +} + +case class ResourceRow(id: UUID, resourceType: String, path: String) diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala index e07c122de..ef459b061 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/SearchApiClient.scala @@ -17,7 +17,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} trait SearchApiClient { - this: NdlaClient with Props => + this: NdlaClient & Props => val searchApiClient: SearchApiClient diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala new file mode 100644 index 000000000..1451398fc --- /dev/null +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala @@ -0,0 +1,38 @@ +/* + * Part of NDLA myndla-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + */ + +package no.ndla.myndlaapi.integration + +import com.typesafe.scalalogging.StrictLogging +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import no.ndla.myndlaapi.Props +import no.ndla.network.NdlaClient +import sttp.client3.quick.* + +import scala.util.{Success, Try} + +trait TaxonomyApiClient { + this: NdlaClient & Props => + + val taxonomyApiClient: TaxonomyApiClient + + class TaxonomyApiClient extends StrictLogging { + private val resolveEndpoint = s"${props.TaxonomyUrl}/v1/url/resolve" + + def resolveUrl(path: String): Try[String] = { + val req = quickRequest.get(uri"$resolveEndpoint?path=$path") + ndlaClient.fetch[ResolvePathResponse](req).map(resolved => resolved.url).orElse(Success(path)) + } + } +} + +case class ResolvePathResponse(url: String) +object ResolvePathResponse { + implicit def decoder: Decoder[ResolvePathResponse] = deriveDecoder + implicit def encoder: Encoder[ResolvePathResponse] = deriveEncoder +} diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala index 40a7adb17..618397a64 100644 --- a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala +++ b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala @@ -116,7 +116,13 @@ class FolderRepositoryTest val resource2 = repository.insertResource("feide", "/path2", ResourceType.Topic, created, TestData.baseResourceDocument) val resource3 = - repository.insertResource("feide", "/path3", ResourceType.Multidisciplinary, created, TestData.baseResourceDocument) + repository.insertResource( + "feide", + "/path3", + ResourceType.Multidisciplinary, + created, + TestData.baseResourceDocument + ) val resource4 = repository.insertResource("feide", "/path4", ResourceType.Image, created, TestData.baseResourceDocument) val resource5 =