chore: add oci references, merge templating changes

CHANGELOG.md

@@ -11,8 +11,13 @@ All notable changes to this project will be documented in this file.
 - Run a `containerdebug` process in the background of each Spark container to collect debugging information ([#508]).
 - Aggregate emitted Kubernetes events on the CustomResources ([#515]).
 
+### Changed
+
+- Default to OCI for image metadata and product image selection ([#514]).
+
 [#501]: https://github.com/stackabletech/spark-k8s-operator/pull/501
 [#508]: https://github.com/stackabletech/spark-k8s-operator/pull/508
+[#514]: https://github.com/stackabletech/spark-k8s-operator/pull/514
 [#515]: https://github.com/stackabletech/spark-k8s-operator/pull/515
 
 ## [24.11.1] - 2025-01-10

Cargo.toml

@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.84.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.39", features = ["full"] }
 tracing = "0.1"

deploy/helm/spark-k8s-operator/crds/crds.yaml

@@ -917,7 +917,7 @@ spec:
                     Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details.
                   properties:
                     custom:
-                      description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0`
+                      description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0`
                       type: string
                     productVersion:
                       description: Version of the product, e.g. `1.4.1`.
@@ -944,7 +944,7 @@ spec:
                       nullable: true
                       type: array
                     repo:
-                      description: Name of the docker repo, e.g. `docker.stackable.tech/stackable`
+                      description: Name of the docker repo, e.g. `oci.stackable.tech/sdp`
                       nullable: true
                       type: string
                     stackableVersion:
@@ -1047,7 +1047,7 @@ spec:
                     Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details.
                   properties:
                     custom:
-                      description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0`
+                      description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0`
                       type: string
                     productVersion:
                       description: Version of the product, e.g. `1.4.1`.
@@ -1074,7 +1074,7 @@ spec:
                       nullable: true
                       type: array
                     repo:
-                      description: Name of the docker repo, e.g. `docker.stackable.tech/stackable`
+                      description: Name of the docker repo, e.g. `oci.stackable.tech/sdp`
                       nullable: true
                       type: string
                     stackableVersion:

deploy/helm/spark-k8s-operator/values.yaml

@@ -1,7 +1,7 @@
 # Default values for spark-k8s-operator.
 ---
 image:
-  repository: docker.stackable.tech/stackable/spark-k8s-operator
+  repository: oci.stackable.tech/sdp/spark-k8s-operator
   pullPolicy: IfNotPresent
   pullSecrets: []
 

docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

@@ -5,7 +5,7 @@ metadata:
   name: example-sparkapp-image
   namespace: default
 spec:
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
+  image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
   sparkImage:
     productVersion: 3.5.2
   mode: cluster

docs/modules/spark-k8s/pages/reference/environment-variables.adoc

@@ -30,7 +30,7 @@ docker run \
 --env KUBECONFIG=/home/stackable/.kube/config \
 --env KUBERNETES_CLUSTER_DOMAIN=mycluster.local \
 --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \
-docker.stackable.tech/stackable/spark-k8s-operator:latest
+oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev
 ----
 
 == WATCH_NAMESPACE
@@ -59,5 +59,5 @@ docker run \
 --env KUBECONFIG=/home/stackable/.kube/config \
 --env WATCH_NAMESPACE=test \
 --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \
-docker.stackable.tech/stackable/spark-k8s-operator:latest
+oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev
 ----

docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc

@@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver:
 
 [source, Dockerfile]
 ----
-FROM docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable24.11.0 # <1>
+FROM oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable24.11.0 # <1>
 
 RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar"
 ----
@@ -75,7 +75,7 @@ metadata:
   name: spark-jdbc
 spec:
   sparkImage:
-    custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1>
+    custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1>
     productVersion: "3.5.2" # <2>
     pullPolicy: IfNotPresent # <3>
 ...

examples/README-examples.md

@@ -16,7 +16,7 @@ stackablectl operator install spark-k8s commons secret
 Load the `ny-tlc-report` image to the cluster:
 
 ````text
-kind load docker-image docker.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform
+kind load docker-image oci.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform
 ````
 
 ## Set up the `PersistentVolumeClaim`

examples/ny-tlc-report-image.yaml

@@ -6,8 +6,8 @@ metadata:
   namespace: default
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable0.0.0-dev
+  image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0
+  sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable0.0.0-dev
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py

kind/assert-pvc-jars.yaml

@@ -13,7 +13,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: assert-pvc-jars
-          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"

kind/kind-pvc.yaml

@@ -24,7 +24,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: aws-deps
-          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"

tests/templates/kuttl/custom-log-directory/20-test-logs.yaml

@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",

tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2

@@ -12,7 +12,7 @@ spec:
     productVersion: "{{ test_scenario['values']['spark'] }}"
 {% endif %}
     pullPolicy: IfNotPresent
-  image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
+  image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS

tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2

@@ -29,7 +29,7 @@ spec:
     productVersion: "{{ test_scenario['values']['spark'] }}"
 {% endif %}
     pullPolicy: IfNotPresent
-  image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
+  image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS

tests/templates/kuttl/logging/10-install-test-runner.yaml

@@ -17,6 +17,6 @@ spec:
     spec:
       containers:
         - name: test-runner
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           stdin: true
           tty: true

tests/templates/kuttl/overrides/20-test-logs.yaml

@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",

tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2

@@ -5,7 +5,7 @@ metadata:
   name: pyspark-ny-public-s3-image
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
-  image: "docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}"
+  image: "oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}"
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}

tests/templates/kuttl/smoke/60-test-logs.yaml

@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",

tests/templates/kuttl/spark-history-server/20-test-logs.yaml

@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",

tests/test-definition.yaml

@@ -9,11 +9,11 @@ dimensions:
       - 3.5.2
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
+      # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: spark-delta-lake
     values:
       - 3.5.2
-      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
+      # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: hbase
     values:
       - 2.6.0