image for k8s operator

.github/workflows/build_workpool_operator.yml

@@ -0,0 +1,41 @@
+name: geoBoundaries Base Image Build
+run-name: gB k8s Workpool Operator
+on: push
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Check out the repository
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Define lowercase repository owner
+      id: repo
+      run: echo "REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ./geoBoundaryBuilder/images/k8s_worker.Dockerfile
+        push: true
+        tags: ghcr.io/${{ env.REPO_OWNER }}/gb-workpool-operator:latest
+        build-args: BUILDKIT_STEP_LOG_MAX_SIZE=10485760 BUILDKIT_STEP_LOG_MAX_SPEED=100000000
+      env:
+        # Enable Docker CLI debug output
+        DOCKER_BUILDKIT: 1
+        DOCKER_CLI_EXPERIMENTAL: enabled
+        DEBUG: 1

geoBoundaryBuilder/images/k8s_worker.Dockerfile

@@ -0,0 +1,40 @@
+# Start from the Prefect image
+FROM prefecthq/prefect:3.1.6.dev6-python3.12-kubernetes
+
+# Install Docker
+USER root
+
+# Install system dependencies and Docker
+RUN apt-get update && \
+    apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+    sudo \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | tee /etc/apt/trusted.gpg.d/docker.asc \
+    && echo "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list \
+    && apt-get update && apt-get install -y \
+    docker-ce-cli \
+    docker-ce \
+    docker-compose \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up Docker group
+RUN groupadd docker && usermod -aG docker $USER
+
+# Configure the user to have permission to run Docker commands
+RUN mkdir /etc/sudoers.d && \
+    echo "$USER ALL=(ALL) NOPASSWD: /usr/bin/docker" > /etc/sudoers.d/docker
+
+# Set the working directory and set the default command
+WORKDIR /workspace
+
+# Revert back to a non-root user (assuming $USER is a non-root user in the Prefect image)
+USER $USER
+
+# Optionally, expose Docker socket if needed (to share Docker daemon)
+VOLUME ["/var/run/docker.sock:/var/run/docker.sock"]
+

geoBoundaryBuilder/k8s_manifests/C_prefect_workpool.yml

@@ -16,7 +16,7 @@ spec:
   restartPolicy: Always
   containers:
     - name: prefect-container
-      image: "prefecthq/prefect:3.1.6.dev6-python3.12-kubernetes"
+      image: "gb-workpool-operator:latest"
       env:
         - name: PREFECT_API_URL
           value: "http://prefect-server-service.geoboundaries.svc.cluster.local:4200/api"

geoBoundaryBuilder/k8s_manifests/dev.yml

@@ -10,6 +10,7 @@ spec:
   containers:
     - name: gb-dev-container
       image: ghcr.io/wmgeolab/gb-base:latest # Replace with your custom image name
+      imagePullPolicy: Always # Always pull the latest image
       command: ["/bin/sh", "-c"] # Keep the container running
       args:
         - |

geoBoundaryBuilder/test.py

@@ -1,42 +1,43 @@
 import os
 import subprocess
 
-# Configure Prefect home directory and API URL
+# Step 1: Configure Prefect home directory and API URL
 os.environ["PREFECT_HOME"] = "/tmp/.prefect"  # Ensure this path is writable
 PREFECT_API_URL = "http://prefect-server-service.geoboundaries.svc.cluster.local:4200/api"
 os.environ["PREFECT_API_URL"] = PREFECT_API_URL  # Directly set in environment
-print(f"Using PREFECT_API_URL: {PREFECT_API_URL}")
+print(f"Using PREFECT_API_URL: {PREFECT_API_URL}")  # Debug output
 subprocess.run(["prefect", "config", "set", f"PREFECT_API_URL={PREFECT_API_URL}"], check=True)
 
+from prefect import flow
+from prefect.context import get_run_context
 
-from prefect import flow, task
-
-
-# Define a task to process numbers
-@task
-def multiply_by_20(number):
-    return number * 20
-
-# Define the main flow
 @flow
-def process_numbers_flow(numbers):
-    results = []
-    for number in numbers:
-        result = multiply_by_20.submit(number)  # Submit tasks asynchronously
-        results.append(result)
-    return [r.result() for r in results]  # Gather the results
+def my_flow():
+    # Your flow logic here
+    print("Running my flow on Kubernetes")
 
 if __name__ == "__main__":
-    # Deploy the flow
-    print("Deploying the flow...")
-    process_numbers_flow.deploy(
-        name="process-numbers-deployment",
+    # Get the current script path
+    script_path = __file__
+
+    # Define dynamic parameters
+    image = "ghcr.io/wmgeolab/gb-base:latest"
+
+    # Deploy the flow with dynamic configurations
+    deployment = my_flow.deploy(
+        name="dynamic-k8s-flow",
         work_pool_name="k8s-gB",
-        image="ghcr.io/wmgeolab/gb-base:latest"  # Ensure this matches your Kubernetes work pool
+        image=image,
+        job_variables={
+            "env": {"EXTRA_PIP_PACKAGES": "your-required-packages"},
+            "image_pull_policy": "Always",
+            "command": [
+                "bash",
+                "-c",
+                f"pip install -r requirements.txt && python {script_path}"
+            ]
+        }
     )
-
-    # Run the flow locally for testing
-    print("Running the flow locally...")
-    numbers = list(range(1, 11))
-    results = process_numbers_flow(numbers)
-    print("Results:", results)
+
+    # Optionally, run the deployment immediately
+    deployment.run()