Kubernetes scheduling first draft

gradle/libs.versions.toml

@@ -10,6 +10,7 @@ coroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3"
 # com.expediagroup:graphql-kotlin-ktor-server:7.0.2
 graphql-server = { module = "com.expediagroup:graphql-kotlin-ktor-server", version.ref = "graphql" }
 graphql-client = { module = "com.expediagroup:graphql-kotlin-ktor-client", version.ref = "graphql" }
+koin = "io.insert-koin:koin-ktor:3.5.3"
 ktor-client-cio = { module = "io.ktor:ktor-client-cio" }
 ktor-client-content-negotiation = { module = "io.ktor:ktor-client-content-negotiation" }
 ktor-client-logging = { module = "io.ktor:ktor-client-logging" }
@@ -26,7 +27,7 @@ ktor-serialisation-kotlinx-json = { module = "io.ktor:ktor-serialization-kotlinx
 kubernetes-client = "io.fabric8:kubernetes-client:6.10.0"
 logging-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j" }
 logging-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" }
-logging-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" }
+logging-slf4jimpl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" }
 serialization = "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0"
 
 exposed-core = { module = "org.jetbrains.exposed:exposed-core", version.ref = "exposedVersion" }

hub/build.gradle.kts

@@ -18,7 +18,7 @@ dependencies {
     implementation(libs.ktor.serialisation.kotlinx.json)
     implementation(libs.logging.api)
     implementation(libs.logging.core)
-    implementation(libs.logging.impl)
+    implementation(libs.logging.slf4jimpl)
     implementation(libs.exposed.core)
     implementation(libs.exposed.crypt)
     implementation(libs.exposed.dao)

hub/src/main/kotlin/org/sourcegrade/lab/hub/Main.kt

@@ -1,5 +1,10 @@
 package org.sourcegrade.lab.hub
 
-import io.ktor.server.netty.EngineMain
+import io.ktor.server.application.Application
+import io.ktor.server.engine.embeddedServer
+import io.ktor.server.netty.Netty
 
-fun main(args: Array<String>) = EngineMain.main(args)
+fun main() {
+    embeddedServer(Netty, module = Application::module)
+        .start(wait = true)
+}

manifests/README.md

@@ -0,0 +1,13 @@
+# Kubernetes Manifests
+
+## Base
+
+This directory contains the base Kubernetes manifests for the application.
+These should usually not differ between environments.
+
+## Example
+
+This directory contains an example of how to use the base manifests to deploy
+the application to a specific environment.
+
+While the defaults should work for most environments, you may need to modify these manifests to suit your needs.

manifests/base/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sourcegrade-lab

manifests/base/operator-rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: operator
+  namespace: sourcegrade-lab
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: access-pods
+  namespace: sourcegrade-lab
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: operator
+  namespace: sourcegrade-lab
+subjects:
+  - kind: ServiceAccount
+    name: operator
+    apiGroup: ""
+roleRef:
+  kind: Role
+  name: access-pods
+  apiGroup: rbac.authorization.k8s.io

manifests/example/operator/deploy.yaml

@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: operator
+  namespace: sourcegrade-lab
+  labels:
+    app: operator
+spec:
+  selector:
+    matchLabels:
+      app: operator
+  strategy:
+    type: Recreate
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: operator
+    spec:
+      serviceAccountName: operator
+      containers:
+        - name: operator
+          image: images.sourcegrade.org/sourcegrade/lab-operator:latest
+          ports:
+            - containerPort: 8080
+              hostPort: 80

manifests/example/operator/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: operator
+  namespace: sourcegrade-lab
+spec:
+  type: ClusterIP
+  selector:
+    app: operator
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80

operator/build.gradle.kts

@@ -9,8 +9,11 @@ repositories {
 
 dependencies {
     implementation(libs.graphql.server)
+    implementation(libs.koin)
     implementation(libs.ktor.server.netty)
     implementation(libs.kubernetes.client)
+    implementation(libs.logging.core)
+    runtimeOnly(libs.logging.slf4jimpl)
 }
 
 application {

operator/src/main/kotlin/org/sourcegrade/lab/operator/Foo.kt

@@ -0,0 +1,10 @@
+package org.sourcegrade.lab.operator
+
+import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder
+
+fun foo() {
+    JobBuilder()
+        .withNewSpec()
+        .withNewTemplate()
+        .
+}

operator/src/main/kotlin/org/sourcegrade/lab/operator/KoinModule.kt

@@ -0,0 +1,11 @@
+package org.sourcegrade.lab.operator
+
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import org.koin.core.module.dsl.singleOf
+import org.koin.dsl.module
+
+internal val koinModule = module {
+    single<Logger> { LogManager.getLogger("AutoGD Operator") }
+    singleOf(::Operator)
+}

operator/src/main/kotlin/org/sourcegrade/lab/operator/Main.kt

@@ -1,5 +1,10 @@
 package org.sourcegrade.lab.operator
 
-import io.ktor.server.netty.EngineMain
+import io.ktor.server.application.Application
+import io.ktor.server.engine.embeddedServer
+import io.ktor.server.netty.Netty
 
-fun main(args: Array<String>) = EngineMain.main(args)
+fun main(args: Array<String>) {
+    embeddedServer(Netty, module = Application::module)
+        .start(wait = true)
+}

operator/src/main/kotlin/org/sourcegrade/lab/operator/Module.kt

@@ -4,14 +4,25 @@ import com.expediagroup.graphql.server.ktor.GraphQL
 import com.expediagroup.graphql.server.ktor.graphQLPostRoute
 import com.expediagroup.graphql.server.operations.Query
 import io.ktor.server.application.Application
+import io.ktor.server.application.ApplicationStarted
 import io.ktor.server.application.install
 import io.ktor.server.routing.Routing
+import kotlinx.coroutines.runBlocking
+import org.apache.logging.log4j.Logger
+import org.koin.java.KoinJavaComponent.inject
+import org.koin.ktor.ext.inject
+import org.koin.ktor.plugin.Koin
 
 class HelloWorldQuery : Query {
     fun hello(): String = "Hello World!"
 }
 
 fun Application.module() {
+
+    install(Koin) {
+        modules(koinModule)
+    }
+
     install(GraphQL) {
         schema {
             packages = listOf("org.sourcegrade.lab.operator")
@@ -23,4 +34,15 @@ fun Application.module() {
     install(Routing) {
         graphQLPostRoute()
     }
+
+    configureRouting()
+
+    environment.monitor.subscribe(ApplicationStarted) {
+        val logger by inject<Logger>()
+        logger.info("Starting operator...")
+        val operator by inject<Operator>()
+        runBlocking {
+            operator.initialize()
+        }
+    }
 }

operator/src/main/kotlin/org/sourcegrade/lab/operator/Operator.kt

@@ -0,0 +1,60 @@
+package org.sourcegrade.lab.operator
+
+import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder
+import io.fabric8.kubernetes.api.model.batch.v1.JobTemplateSpecBuilder
+import io.fabric8.kubernetes.client.KubernetesClient
+import io.fabric8.kubernetes.client.KubernetesClientBuilder
+import kotlinx.coroutines.Job
+import org.apache.logging.log4j.Logger
+import java.util.Collections
+
+class Operator(
+    private val logger: Logger,
+) {
+
+    private lateinit var client: KubernetesClient
+
+    fun initialize() {
+        logger.info("Initializing operator...")
+        client = KubernetesClientBuilder().build()
+        logger.info("Connected to Kubernetes API server ${client.masterUrl} with API version ${client.apiVersion}.")
+        logger.info("In namespace '${client.namespace}'.")
+        logger.info("Getting pods")
+
+        JobBuilder()
+            .withNewSpec()
+            .withNewTemplate()
+            .withNewSpec()
+            .addNewContainer()
+
+        try {
+            logger.info("Pods: ${client.pods().inNamespace(client.namespace).list()}")
+        } catch (e: Exception) {
+            logger.error("Failed to get pods", e)
+        }
+
+        JobBuilder()
+            .withApiVersion("batch/v1")
+            .withNewMetadata()
+            .withName("pi")
+//            .withLabels(Collections.singletonMap("label1", "maximum-length-of-63-characters"))
+//            .withAnnotations(Collections.singletonMap("annotation1", "some-very-long-annotation"))
+            .endMetadata()
+            .withNewSpec()
+            .withNewTemplate()
+            .withNewSpec()
+            .addNewContainer()
+            .withName("pi")
+            .withImage("perl")
+            .withArgs("perl", "-Mbignum=bpi", "-wle", "print bpi(2000)")
+            .endContainer()
+            .withRestartPolicy("Never")
+            .endSpec()
+            .endTemplate()
+            .endSpec()
+            .build()
+
+
+        logger.info("Got pods")
+    }
+}

operator/src/main/resources/log4j2.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="Console">
+      <PatternLayout disableAnsi="false">
+        <pattern>%highlight{[%d{yy-MMM-dd HH:mm:ss}] [%p] - %m%n%throwable}{INFO=white}</pattern>
+      </PatternLayout>
+    </Console>
+    <RollingRandomAccessFile name="File" fileName="logs/latest.log"
+                             filePattern="logs/%d{yyyy-MM-dd}-%i.log.gz"
+                             immediateFlush="false">
+      <PatternLayout>
+        <pattern>[%d{yy-MMM-dd HH:mm:ss}] [%p] - %m%n%throwable</pattern>
+      </PatternLayout>
+      <Policies>
+        <TimeBasedTriggeringPolicy/>
+        <OnStartupTriggeringPolicy/>
+      </Policies>
+    </RollingRandomAccessFile>
+  </Appenders>
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="Console"/>
+      <AppenderRef ref="File"/>
+    </Root>
+  </Loggers>
+</Configuration>