Improved Armada Airflow Operator (#3672)

* Syncing out Armada Airflow Operator (#164)

* changes

* Fixing tests

* Formatting

* Making integration tests work

* Formatting files

* Formatting, ensuring Armada trigger can cancel

* Fixing docs and adding back examples

* Fixing docs

* Addressing comments, supporting more python versions

* Updating docs

* Removing airflow operator build script from github workflow check

---------

Co-authored-by: Mustafa Ilyas <[email protected]>

.github/workflows/airflow-operator.yml

@@ -16,7 +16,6 @@ on:
       - 'internal/jobservice/*'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
-      - 'scripts/build-airflow-operator.sh'
       - 'scripts/build-python-client.sh'
       - 'third_party/airflow/**'
       - './magefiles/tests.go'
@@ -37,7 +36,6 @@ on:
       - 'internal/jobservice/*'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
-      - 'scripts/build-airflow-operator.sh'
       - 'scripts/build-python-client.sh'
       - 'third_party/airflow/**'
 

build/airflow-operator/Dockerfile

@@ -1,5 +1,5 @@
 ARG PLATFORM=x86_64
-ARG BASE_IMAGE=python:3.8.18-bookworm
+ARG BASE_IMAGE=python:3.10.14-bookworm
 FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto

developer/env/docker/server.env

@@ -1,3 +1,3 @@
 ARMADA_QUEUECACHEREFRESHPERIOD="1s"
 ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://example.com:10000"
-
+ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST=postgres

docker-compose.yaml

@@ -61,6 +61,7 @@ services:
     depends_on:
       - lookoutv2-migration
       - eventingester
+      - lookoutingesterv2
     working_dir: /app
     env_file:
       - developer/env/docker/server.env