Merge pull request #131 from mirkoDeer/main

Add gzip upload & download of container images

Author: cmdjulian

.gitattributes

@@ -0,0 +1 @@
+*.tar filter=lfs diff=lfs merge=lfs -text

.idea/inspectionProfiles/ktlint.xml

@@ -23,7 +23,7 @@ instance of `RegistryClientException` for all errors. These errors are than divi
 2. network related errors (`NetworkError`) like HostNotFound or SSL related errors
 3. unexpected errors (`UnknownError`)
 
-As authentication schema JWT auth and BasicAuth are supported. Currently, there are no plans to implement certificate
+As an authentication schema, JWT auth and BasicAuth are supported. Currently, there are no plans to implement certificate
 based authentication.
 
 The Registry communication can be done using either `HTTP` or `HTTPS`. The library is also able to use a proxy for the

README.md

@@ -7,9 +7,8 @@ dependencies {
     api(project(":kirc-core"))
     api(project(":kirc-image"))
     implementation(project(":kirc-suspending"))
-
-    implementation(platform(coroutines.bom))
-    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core")
+    implementation(libs.coroutines)
+    implementation(libs.kotlinx.io)
 }
 
 tasks.jar {

kirc-blocking/build.gradle.kts

@@ -2,10 +2,13 @@ package de.cmdjulian.kirc.client
 
 import de.cmdjulian.kirc.image.ContainerImageName
 import kotlinx.coroutines.runBlocking
+import kotlinx.io.files.SystemTemporaryDirectory
 import java.net.Proxy
 import java.net.URI
+import java.nio.file.Path
 import java.security.KeyStore
 import java.time.Duration
+import kotlin.io.path.Path
 
 object BlockingContainerImageClientFactory {
 
@@ -25,9 +28,12 @@ object BlockingContainerImageClientFactory {
         skipTlsVerify: Boolean = false,
         keystore: KeyStore? = null,
         timeout: Duration = Duration.ofSeconds(5),
-    ): BlockingContainerImageRegistryClient =
-        SuspendingContainerImageClientFactory.create(url, credentials, proxy, skipTlsVerify, keystore, timeout)
+        tmpPath: Path = Path(SystemTemporaryDirectory.toString()),
+    ): BlockingContainerImageRegistryClient {
+        return SuspendingContainerImageClientFactory
+            .create(url, credentials, proxy, skipTlsVerify, keystore, timeout, tmpPath)
             .toBlockingClient()
+    }
 
     @JvmStatic
     @JvmOverloads
@@ -39,9 +45,10 @@ object BlockingContainerImageClientFactory {
         skipTlsVerify: Boolean = false,
         keystore: KeyStore? = null,
         timeout: Duration = Duration.ofSeconds(5),
+        tmpPath: Path = Path(SystemTemporaryDirectory.toString()),
     ): BlockingContainerImageClient = runBlocking {
         SuspendingContainerImageClientFactory
-            .create(image, credentials, proxy, insecure, skipTlsVerify, keystore, timeout)
+            .create(image, credentials, proxy, insecure, skipTlsVerify, keystore, timeout, tmpPath)
             .toBlockingClient()
     }
 }

kirc-blocking/src/main/kotlin/de/cmdjulian/kirc/client/BlockingContainerImageClientFactory.kt

@@ -6,8 +6,15 @@ import de.cmdjulian.kirc.image.Repository
 import de.cmdjulian.kirc.image.Tag
 import de.cmdjulian.kirc.spec.image.ImageConfig
 import de.cmdjulian.kirc.spec.manifest.Manifest
+import de.cmdjulian.kirc.spec.manifest.ManifestList
 import de.cmdjulian.kirc.spec.manifest.ManifestSingle
 import kotlinx.coroutines.runBlocking
+import kotlinx.io.asInputStream
+import kotlinx.io.asSink
+import kotlinx.io.asSource
+import kotlinx.io.buffered
+import java.io.InputStream
+import java.io.OutputStream
 
 interface BlockingContainerImageRegistryClient {
     /**
@@ -57,7 +64,7 @@ interface BlockingContainerImageRegistryClient {
      * If the [reference] points to a ManifestList, the behaviour is up to the registry. Usually the first entry of the
      * list is returned.
      *
-     * To be safe, it's better to use [config] instead.
+     * To be safe, it's better to use [Digest] or config([Repository], [ManifestSingle]) instead.
      */
     fun config(repository: Repository, reference: Reference): ImageConfig
 
@@ -74,6 +81,22 @@ interface BlockingContainerImageRegistryClient {
         reference: Reference,
         manifest: ManifestSingle? = null,
     ): BlockingContainerImageClient
+
+    /**
+     * Uploads [tar] image archive to container registry at [repository] with [reference]
+     *
+     * @return the digest of uploaded image
+     */
+    fun upload(repository: Repository, reference: Reference, tar: InputStream): Digest
+
+    /**
+     * Downloads a docker image for certain [reference].
+     *
+     * For [reference] we download everything to what [reference] directs to (either [ManifestSingle] or [ManifestList])
+     */
+    fun download(repository: Repository, reference: Reference): InputStream
+
+    fun download(repository: Repository, reference: Reference, destination: OutputStream)
 }
 
 fun SuspendingContainerImageRegistryClient.toBlockingClient() = object : BlockingContainerImageRegistryClient {
@@ -119,4 +142,14 @@ fun SuspendingContainerImageRegistryClient.toBlockingClient() = object : Blockin
 
         return client.toBlockingClient()
     }
+
+    override fun upload(repository: Repository, reference: Reference, tar: InputStream): Digest =
+        runBlocking { this@toBlockingClient.upload(repository, reference, tar.asSource().buffered()) }
+
+    override fun download(repository: Repository, reference: Reference): InputStream =
+        runBlocking { this@toBlockingClient.download(repository, reference).asInputStream() }
+
+    override fun download(repository: Repository, reference: Reference, destination: OutputStream) = runBlocking {
+        this@toBlockingClient.download(repository, reference, destination.asSink().buffered())
+    }
 }

kirc-blocking/src/main/kotlin/de/cmdjulian/kirc/client/BlockingContainerImageRegistryClient.kt

@@ -22,6 +22,16 @@ sealed class RegistryClientException(
         cause: Throwable,
     ) : RegistryClientException(url, repository, reference, message, cause) {
 
+        class BadRequestException(
+            url: URL,
+            repository: Repository?,
+            reference: Reference?,
+            error: ErrorResponse?,
+            cause: Throwable,
+        ) : ClientException(url, repository, reference, "bad request", error, cause) {
+            override fun toString(): String = "DistributionClientException.BadRequestError -> $message"
+        }
+
         class AuthenticationException(
             url: URL,
             repository: Repository?,
@@ -62,6 +72,26 @@ sealed class RegistryClientException(
             override fun toString() = "ClientException.MethodNotAllowed (is registry delete enabled?) -> $message"
         }
 
+        class RangeNotSatisfiable(
+            url: URL,
+            repository: Repository?,
+            reference: Reference?,
+            error: ErrorResponse?,
+            cause: Throwable,
+        ) : ClientException(url, repository, reference, "range not satisfiable", error, cause) {
+            override fun toString() = "ClientException.RangeNotSatisfiable -> $message"
+        }
+
+        class TooManyRequests(
+            url: URL,
+            repository: Repository?,
+            reference: Reference?,
+            error: ErrorResponse?,
+            cause: Throwable,
+        ) : ClientException(url, repository, reference, "too many requests", error, cause) {
+            override fun toString() = "ClientException.TooManyRequests -> $message"
+        }
+
         class UnexpectedErrorException(
             url: URL,
             repository: Repository?,

kirc-core/src/main/kotlin/de/cmdjulian/kirc/exception/RegistryClientException.kt

@@ -1,8 +1,17 @@
 package de.cmdjulian.kirc.spec
 
+import de.cmdjulian.kirc.image.Digest
 import de.cmdjulian.kirc.spec.image.ImageConfig
 import de.cmdjulian.kirc.spec.manifest.ManifestSingle
 import io.goodforgod.graalvm.hint.annotation.ReflectionHint
 
+/**
+ * Represents a Docker Container Image
+ */
 @ReflectionHint
-data class ContainerImage(val manifest: ManifestSingle, val config: ImageConfig, val blobs: List<LayerBlob>)
+data class ContainerImage(
+    val manifest: ManifestSingle,
+    val digest: Digest,
+    val config: ImageConfig,
+    val blobs: List<LayerBlob>,
+)

kirc-core/src/main/kotlin/de/cmdjulian/kirc/spec/ContainerImage.kt

@@ -0,0 +1,26 @@
+package de.cmdjulian.kirc.spec
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.databind.PropertyNamingStrategies
+import com.fasterxml.jackson.databind.annotation.JsonNaming
+import de.cmdjulian.kirc.image.Digest
+import de.cmdjulian.kirc.spec.manifest.LayerReference
+import io.goodforgod.graalvm.hint.annotation.ReflectionHint
+
+typealias ManifestJson = List<ManifestJsonEntry>
+
+/**
+ * Represents the `manifest.json` file inside a docker archive.
+ *
+ * NOT PART OF the OFFICIAL OCI STANDARD!
+ */
+@ReflectionHint
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class)
+@ReflectionHint(types = [PropertyNamingStrategies.UpperCamelCaseStrategy::class])
+data class ManifestJsonEntry(
+    val config: String,
+    val repoTags: List<String>,
+    val layers: List<String>,
+    val layerSources: Map<Digest, LayerReference>,
+)

kirc-core/src/main/kotlin/de/cmdjulian/kirc/spec/ManifestJson.kt

@@ -0,0 +1,6 @@
+package de.cmdjulian.kirc.spec
+
+import io.goodforgod.graalvm.hint.annotation.ReflectionHint
+
+@ReflectionHint
+data class OciLayout(val imageLayoutVersion: String = "1.0.0")