Add service to run maintenance jobs

core/build.gradle.kts

@@ -59,6 +59,7 @@ dependencies {
     implementation(projects.services.authorizationService)
     implementation(projects.services.hierarchyService)
     implementation(projects.services.infrastructureService)
+    implementation(projects.services.maintenanceService)
     implementation(projects.services.reportStorageService)
     implementation(projects.services.secretService)
     implementation(projects.storage.storageSpi)

core/src/main/kotlin/di/Module.kt

@@ -71,6 +71,7 @@ import org.eclipse.apoapsis.ortserver.services.ReportStorageService
 import org.eclipse.apoapsis.ortserver.services.RepositoryService
 import org.eclipse.apoapsis.ortserver.services.SecretService
 import org.eclipse.apoapsis.ortserver.services.VulnerabilityService
+import org.eclipse.apoapsis.ortserver.services.maintenance.MaintenanceService
 import org.eclipse.apoapsis.ortserver.storage.Storage
 
 import org.koin.core.module.dsl.singleOf
@@ -117,6 +118,8 @@ fun ortServerModule(config: ApplicationConfig) = module {
         val keycloakGroupPrefix = get<ApplicationConfig>().tryGetString("keycloak.groupPrefix").orEmpty()
         DefaultAuthorizationService(get(), get(), get(), get(), get(), keycloakGroupPrefix)
     }
+
+    singleOf(::MaintenanceService)
     single { OrchestratorService(get(), get(), get()) }
     single { OrganizationService(get(), get(), get(), get()) }
     single { ProductService(get(), get(), get(), get()) }

core/src/main/kotlin/plugins/Lifecycle.kt

@@ -27,9 +27,12 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 
 import org.eclipse.apoapsis.ortserver.services.AuthorizationService
+import org.eclipse.apoapsis.ortserver.services.maintenance.MaintenanceService
+import org.eclipse.apoapsis.ortserver.services.maintenance.jobs.DeduplicatePackagesJob
 import org.eclipse.apoapsis.ortserver.utils.logging.runBlocking
 import org.eclipse.apoapsis.ortserver.utils.logging.withMdcContext
 
+import org.koin.ktor.ext.get
 import org.koin.ktor.ext.inject
 
 import org.slf4j.MDC
@@ -41,6 +44,7 @@ import org.slf4j.MDC
 fun Application.configureLifecycle() {
     environment.monitor.subscribe(DatabaseReady) {
         val authorizationService by inject<AuthorizationService>()
+        val maintenanceService by inject<MaintenanceService>()
 
         val mdcContext = MDC.getCopyOfContextMap()
 
@@ -50,6 +54,14 @@ fun Application.configureLifecycle() {
                 syncRoles(authorizationService)
             }
         }
+
+        thread {
+            MDC.setContextMap(mdcContext)
+            runBlocking(Dispatchers.IO) {
+                maintenanceService.addJob(DeduplicatePackagesJob(get()))
+                maintenanceService.run()
+            }
+        }
     }
 }
 

dao/src/main/kotlin/tables/MaintenanceJobsTable.kt

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2024 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.eclipse.apoapsis.ortserver.dao.tables
+
+import kotlinx.serialization.json.JsonObject
+
+import org.eclipse.apoapsis.ortserver.dao.utils.jsonb
+import org.eclipse.apoapsis.ortserver.dao.utils.toDatabasePrecision
+import org.eclipse.apoapsis.ortserver.model.MaintenanceJobData
+import org.eclipse.apoapsis.ortserver.model.MaintenanceJobStatus
+
+import org.jetbrains.exposed.dao.LongEntity
+import org.jetbrains.exposed.dao.LongEntityClass
+import org.jetbrains.exposed.dao.id.EntityID
+import org.jetbrains.exposed.dao.id.LongIdTable
+import org.jetbrains.exposed.sql.kotlin.datetime.timestamp
+
+/**
+ * A table that stores the state of maintenance jobs.
+ */
+object MaintenanceJobsTable : LongIdTable("maintenance_jobs") {
+    val name = text("name")
+    val status = enumerationByName<MaintenanceJobStatus>("status", 128)
+    val startedAt = timestamp("started_at").nullable()
+    val updatedAt = timestamp("updated_at").nullable()
+    val finishedAt = timestamp("finished_at").nullable()
+    val data = jsonb<JsonObject>("data").nullable()
+}
+
+class MaintenanceJobDao(id: EntityID<Long>) : LongEntity(id) {
+    companion object : LongEntityClass<MaintenanceJobDao>(MaintenanceJobsTable)
+
+    var name by MaintenanceJobsTable.name
+    var status by MaintenanceJobsTable.status
+    var startedAt by MaintenanceJobsTable.startedAt.transform({ it?.toDatabasePrecision() }, { it })
+    var updatedAt by MaintenanceJobsTable.updatedAt.transform({ it?.toDatabasePrecision() }, { it })
+    var finishedAt by MaintenanceJobsTable.finishedAt.transform({ it?.toDatabasePrecision() }, { it })
+    var data by MaintenanceJobsTable.data
+
+    fun mapToModel() = MaintenanceJobData(
+        id = id.value,
+        name = name,
+        status = status,
+        startedAt = startedAt,
+        updatedAt = updatedAt,
+        finishedAt = finishedAt,
+        data = data
+    )
+}

dao/src/main/kotlin/tables/runs/analyzer/PackagesTable.kt

@@ -33,6 +33,7 @@ import org.jetbrains.exposed.dao.LongEntity
 import org.jetbrains.exposed.dao.LongEntityClass
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.dao.id.LongIdTable
+import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.and
 
 /**
@@ -64,7 +65,7 @@ class PackageDao(id: EntityID<Long>) : LongEntity(id) {
                         (PackagesTable.homepageUrl eq pkg.homepageUrl) and
                         (PackagesTable.isMetadataOnly eq pkg.isMetadataOnly) and
                         (PackagesTable.isModified eq pkg.isModified)
-            }.firstOrNull {
+            }.orderBy(PackagesTable.id to SortOrder.ASC).firstOrNull {
                 it.identifier.mapToModel() == pkg.identifier &&
                         mapAndCompare(it.authors, pkg.authors, AuthorDao::name) &&
                         mapAndCompare(it.declaredLicenses, pkg.declaredLicenses, DeclaredLicenseDao::name) &&

dao/src/main/resources/db/migration/V72__createIndexPackagesAuthorsPackageId.sql

@@ -0,0 +1 @@
+CREATE INDEX packages_authors_package_id ON packages_authors(package_id);

dao/src/main/resources/db/migration/V73__maintenanceJobs.sql

@@ -0,0 +1,10 @@
+CREATE TABLE maintenance_jobs
+(
+    id          bigserial PRIMARY KEY,
+    name        text NOT NULL,
+    status      text NOT NULL,
+    started_at  timestamp,
+    updated_at  timestamp,
+    finished_at timestamp,
+    data        jsonb
+);

model/src/commonMain/kotlin/MaintenanceJobData.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2024 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.eclipse.apoapsis.ortserver.model
+
+import kotlinx.datetime.Instant
+import kotlinx.serialization.json.JsonObject
+
+/**
+ * The data of a maintenance job, as stored in the database.
+ */
+data class MaintenanceJobData(
+    /** The unique identifier. */
+    val id: Long,
+
+    /** The name of the job. */
+    val name: String,
+
+    /** The status of the job. */
+    val status: MaintenanceJobStatus,
+
+    /** The time the job was started. */
+    val startedAt: Instant?,
+
+    /** The time the job was last updated. */
+    val updatedAt: Instant?,
+
+    /** The time the job finished. */
+    val finishedAt: Instant?,
+
+    /** The job data. */
+    val data: JsonObject?
+)

model/src/commonMain/kotlin/MaintenanceJobStatus.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.eclipse.apoapsis.ortserver.model
+
+/**
+ * An enum representing the status of a maintenance job.
+ */
+enum class MaintenanceJobStatus(val completed: Boolean) {
+    /** The job is currently running. */
+    STARTED(completed = false),
+
+    /** The job has finished successfully. */
+    FINISHED(completed = true),
+
+    /** The job has failed. */
+    FAILED(completed = true);
+
+    companion object {
+        val uncompletedStates = entries.filter { !it.completed }
+    }
+}

services/maintenance/build.gradle.kts

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+plugins {
+    // Apply precompiled plugins.
+    id("ort-server-kotlin-jvm-conventions")
+    id("ort-server-publication-conventions")
+
+    // Apply third-party plugins.
+    alias(libs.plugins.kotlinSerialization)
+}
+
+group = "org.eclipse.apoapsis.ortserver.services"
+
+dependencies {
+    api(projects.model)
+
+    implementation(projects.dao)
+    implementation(projects.utils.logging)
+
+    implementation(libs.kotlinxSerializationJson)
+
+    runtimeOnly(libs.logback)
+
+    testImplementation(testFixtures(projects.dao))
+    testImplementation(projects.utils.test)
+
+    testImplementation(libs.kotestAssertionsCore)
+    testImplementation(libs.kotestRunnerJunit5)
+    testImplementation(libs.mockk)
+}

services/maintenance/src/main/kotlin/MaintenanceJob.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.eclipse.apoapsis.ortserver.services.maintenance
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+import org.eclipse.apoapsis.ortserver.model.MaintenanceJobData
+import org.eclipse.apoapsis.ortserver.utils.logging.withMdcContext
+
+abstract class MaintenanceJob {
+    abstract val name: String
+
+    private val _active = AtomicBoolean(false)
+
+    val active get() = _active.get()
+
+    suspend fun start(service: MaintenanceService, jobData: MaintenanceJobData) {
+        _active.set(true)
+
+        try {
+            withMdcContext("maintenanceJob" to name) {
+                execute(service, jobData)
+            }
+        } finally {
+            _active.set(false)
+        }
+    }
+
+    abstract suspend fun execute(service: MaintenanceService, jobData: MaintenanceJobData)
+}