first commit

.dockerignore

@@ -0,0 +1,5 @@
+/.git
+/.github
+/helm
+/LICENSE
+/README.md

.github/workflows/ci.yaml

@@ -0,0 +1,51 @@
+name: Create and publish a Docker image
+on:
+  push:
+    branches: ['main']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+
+        - name: Log in to the Container registry
+          uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+          with:
+            registry: ${{ env.REGISTRY }}
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+          with:
+            images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+        - name: Build and push Docker image
+          id: push
+          uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+          with:
+            context: .
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+        - name: Generate artifact attestation
+          uses: actions/attest-build-provenance@v1
+          with:
+            subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            subject-digest: ${{ steps.push.outputs.digest }}
+            push-to-registry: true

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9-slim
+
+WORKDIR /usr/src/app
+
+COPY src/ .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 8080
+
+CMD ["python", "app.py"]

README.md

@@ -1,2 +1,79 @@
 # kubernetes-jobsequence
 A Kubernetes Job that spawns Jobs in a sequence, and waits for there completion
+
+# Example
+
+```
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: job-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: job-cluster-admin-binding
+subjects:
+- kind: ServiceAccount
+  name: job-serviceaccount
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: do-subjobs
+  namespace: default
+data:
+  job1.yaml: |
+    apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: sleep-job1
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sleep-container
+            image: busybox
+            command: ["sleep", "10"]
+          restartPolicy: Never
+  job2.yaml: |
+    apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: sleep-job2
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sleep-container
+            image: busybox
+            command: ["sleep", "20"]
+          restartPolicy: Never
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: do-something
+  namespace: default
+spec:
+  template:
+    spec:
+      serviceAccountName: job-serviceaccount
+      volumes:
+        - name: jobs
+          configMap:
+            name: do-subjobs
+      containers:
+        - name: runner
+          image: ghcr.io/deltachaos/kubernetes-jobsequence
+          volumeMounts:
+            - name: jobs
+              mountPath: /jobs
+      restartPolicy: Never
+```

src/app.py

@@ -0,0 +1,112 @@
+import os
+import yaml
+import random
+import string
+import time
+import logging
+from kubernetes import client, config
+
+def generate_random_suffix(length=6):
+    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
+def create_configmap(namespace, name, data):
+    v1 = client.CoreV1Api()
+    metadata = client.V1ObjectMeta(name=name)
+    configmap = client.V1ConfigMap(metadata=metadata, data=data)
+    return v1.create_namespaced_config_map(namespace=namespace, body=configmap)
+
+def delete_configmap(namespace, name):
+    v1 = client.CoreV1Api()
+    return v1.delete_namespaced_config_map(name=name, namespace=namespace)
+
+def create_job(namespace, job_definition):
+    batch_v1 = client.BatchV1Api()
+    return batch_v1.create_namespaced_job(namespace=namespace, body=job_definition)
+
+def wait_for_job_completion(namespace, job_name):
+    batch_v1 = client.BatchV1Api()
+    while True:
+        job = batch_v1.read_namespaced_job(name=job_name, namespace=namespace)
+        job_status = job.status
+        if job_status.succeeded:
+            logging.info(f"Job {job_name} succeeded.")
+            return True
+        if job_status.failed:
+            logging.error(f"Job {job_name} failed.")
+            return False
+        logging.info(f"Job {job_name} status: {job_status.active} active pods.")
+        time.sleep(5)
+
+def read_configmap(namespace, name):
+    v1 = client.CoreV1Api()
+    return v1.read_namespaced_config_map(name=name, namespace=namespace).data
+
+def read_job_files_from_directory(directory):
+    job_files = []
+    for filename in os.listdir(directory):
+        if filename.endswith('.yaml') or filename.endswith('.yml'):
+            with open(os.path.join(directory, filename), 'r') as file:
+                job_files.append(file.read())
+    return job_files
+
+def main():
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    logging.info("Starting the job sequence script.")
+
+    config.load_incluster_config()
+    namespace = os.getenv('NAMESPACE', 'default')
+    configmap_name = os.getenv('JOB_CONFIGMAP')
+    job_name_env = os.getenv('JOB_NAME')
+
+    v1 = client.CoreV1Api()
+
+    if configmap_name:
+        logging.info(f"Reading initial configmap: {configmap_name}")
+        configmap = v1.read_namespaced_config_map(name=configmap_name, namespace=namespace)
+        queue = list(configmap.data.values())
+    else:
+        logging.info("JOB_CONFIGMAP environment variable is not set. Reading job files from /jobs directory.")
+        queue = read_job_files_from_directory('/jobs')
+
+    while queue:
+        yaml_definition = queue.pop(0)
+        job_definition = yaml.safe_load(yaml_definition)
+
+        # Create a configmap for job results
+        result_configmap_name = f"{job_name_env}-{generate_random_suffix()}"
+        logging.info(f"Creating result configmap: {result_configmap_name}")
+        create_configmap(namespace, result_configmap_name, data={})
+
+        # Modify job definition to include the result configmap
+        job_definition['spec']['template']['spec']['containers'][0]['env'].append(
+            client.V1EnvVar(name='JOBSEQUENCE_RESULT_CONFIGMAP', value=result_configmap_name)
+        )
+
+        # Create the job
+        logging.info(f"Creating job: {job_definition['metadata']['name']}")
+        job = create_job(namespace, job_definition)
+        job_name = job.metadata.name
+
+        # Wait for the job to complete
+        logging.info(f"Waiting for job {job_name} to complete.")
+        job_succeeded = wait_for_job_completion(namespace, job_name)
+
+        # Read the result configmap
+        logging.info(f"Reading result configmap: {result_configmap_name}")
+        result_configmap_data = read_configmap(namespace, result_configmap_name)
+
+        # Delete the result configmap
+        logging.info(f"Deleting result configmap: {result_configmap_name}")
+        delete_configmap(namespace, result_configmap_name)
+
+        if not job_succeeded:
+            logging.error("Exiting due to job failure.")
+            exit(1)
+
+        # Add items to the queue
+        queue.extend(result_configmap_data.values())
+
+    logging.info("Job sequence script completed.")
+
+if __name__ == '__main__':
+    main()

src/requirements.txt

@@ -0,0 +1,2 @@
+PyYAML==6.0.1
+kubernetes==30.1.0