Generate cache and pass valid fernet keys in 2.10.1 and 2.10.3 (#196)

images/airflow/2.10.1/docker-compose.yaml

@@ -19,7 +19,7 @@ x-airflow-common: &airflow-common
     MWAA__CORE__AUTH_TYPE: "testing"
     # Additional Airflow configuration can be passed here in JSON form.
     MWAA__CORE__CUSTOM_AIRFLOW_CONFIGS: "{}"
-    MWAA__CORE__FERNET_KEY: '{"FernetKey": "fake-key-nNge+lks3RBeGVrnZ1Dq5GjKerbZKmb7dXNnsNsGy3E="}'
+    MWAA__CORE__FERNET_KEY: ${FERNET_KEY}
     MWAA__WEBSERVER__SECRET: '{"secret_key": "fake-key-aYDdF6d+Fjznai5yBW63CUAi0IipJqDHlNSWIun6y8o="}'
     # Use this enviornment variable to enable encryption with KMS.
     MWAA__CORE__KMS_KEY_ARN: ${MWAA__CORE__KMS_KEY_ARN}

images/airflow/2.10.1/generate_fernet_key.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""
+This Module generates Fernet keys, which are used by Airflow for connection encryption
+"""
+
+from cryptography.fernet import Fernet
+import json
+
+def generate_fernet_key():
+    """
+    Generate a Fernet key and return it as a JSON string.
+
+    :returns A JSON string containing the generated Fernet key in the format {"FernetKey": "<key>"}
+    """
+    key = Fernet.generate_key().decode()
+    return json.dumps({"FernetKey": key})
+
+if __name__ == "__main__":
+    print(generate_fernet_key())

images/airflow/2.10.1/run.sh

@@ -10,6 +10,37 @@ else
     CONTAINER_RUNTIME="docker"
 fi
 
+# Generate valid Fernet key as json
+generate_fernet_key() {
+
+    # Install cryptography package quietly
+    chmod +x temporary-pip-install generate_fernet_key.py
+    ./temporary-pip-install cryptography >/dev/null 2>&1
+
+    # Generate the key and format as JSON
+    KEY=$(python3 generate_fernet_key.py)
+
+    # Uninstall cryptography package quietly
+    python3 -m pip uninstall -y cryptography cryptography-vectors &>/dev/null 2>&1
+
+    echo "$KEY"
+}
+
+# Set up cache directory ; generate if it dosen't exist
+CACHE_DIR="${HOME}/.cache/mwaa-local"
+FERNET_KEY_FILE="${CACHE_DIR}/fernet.key"
+mkdir -p "${CACHE_DIR}"
+
+# Check if we have a cached Fernet key, if not generate and cache it
+if [ ! -f "${FERNET_KEY_FILE}" ]; then
+    generate_fernet_key > "${FERNET_KEY_FILE}"
+    chmod 600 "${FERNET_KEY_FILE}"
+fi
+
+# Read the Fernet key from cache
+FERNET_KEY=$(cat "${FERNET_KEY_FILE}")
+export FERNET_KEY
+
 # Build the Docker image
 ./build.sh $CONTAINER_RUNTIME
 

images/airflow/2.10.1/temporary-pip-install

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# This script is specifically designed for temporarily installing packages needed ONLY before bootstrap steps.
+# It intentionally bypasses constraint checks, since it is intended that the packages will be used for setup/configuration 
+# and then UNINSTALLED before the bootstrap steps, during local setup.
+#
+# NOTE: This script should NOT be used for installing production Airflow/MWAA dependencies.
+# For those, use 'safe-pip-install' which properly handles Airflow/MWAA constraints.
+
+pip3 install "$@"

images/airflow/2.10.3/docker-compose.yaml

@@ -19,7 +19,7 @@ x-airflow-common: &airflow-common
     MWAA__CORE__AUTH_TYPE: "testing"
     # Additional Airflow configuration can be passed here in JSON form.
     MWAA__CORE__CUSTOM_AIRFLOW_CONFIGS: "{}"
-    MWAA__CORE__FERNET_KEY: '{"FernetKey": "fake-key-nNge+lks3RBeGVrnZ1Dq5GjKerbZKmb7dXNnsNsGy3E="}'
+    MWAA__CORE__FERNET_KEY: ${FERNET_KEY}
     MWAA__WEBSERVER__SECRET: '{"secret_key": "fake-key-aYDdF6d+Fjznai5yBW63CUAi0IipJqDHlNSWIun6y8o="}'
     # Use this enviornment variable to enable encryption with KMS.
     MWAA__CORE__KMS_KEY_ARN: ${MWAA__CORE__KMS_KEY_ARN}

images/airflow/2.10.3/generate_fernet_key.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""
+This Module generates Fernet keys, which are used by Airflow for connection encryption
+"""
+
+from cryptography.fernet import Fernet
+import json
+
+def generate_fernet_key():
+    """
+    Generate a Fernet key and return it as a JSON string.
+
+    :returns A JSON string containing the generated Fernet key in the format {"FernetKey": "<key>"}
+    """
+    key = Fernet.generate_key().decode()
+    return json.dumps({"FernetKey": key})
+
+if __name__ == "__main__":
+    print(generate_fernet_key())

images/airflow/2.10.3/run.sh

@@ -10,6 +10,37 @@ else
     CONTAINER_RUNTIME="docker"
 fi
 
+# Generate valid Fernet key as json
+generate_fernet_key() {
+
+    # Install cryptography package quietly
+    chmod +x temporary-pip-install generate_fernet_key.py
+    ./temporary-pip-install cryptography >/dev/null 2>&1
+
+    # Generate the key and format as JSON
+    KEY=$(python3 generate_fernet_key.py)
+
+    # Uninstall cryptography package quietly
+    python3 -m pip uninstall -y cryptography cryptography-vectors &>/dev/null 2>&1
+
+    echo "$KEY"
+}
+
+# Set up cache directory ; generate if it dosen't exist
+CACHE_DIR="${HOME}/.cache/mwaa-local"
+FERNET_KEY_FILE="${CACHE_DIR}/fernet.key"
+mkdir -p "${CACHE_DIR}"
+
+# Check if we have a cached Fernet key, if not generate and cache it
+if [ ! -f "${FERNET_KEY_FILE}" ]; then
+    generate_fernet_key > "${FERNET_KEY_FILE}"
+    chmod 600 "${FERNET_KEY_FILE}"
+fi
+
+# Read the Fernet key from cache
+FERNET_KEY=$(cat "${FERNET_KEY_FILE}")
+export FERNET_KEY
+
 # Build the Docker image
 ./build.sh $CONTAINER_RUNTIME
 

images/airflow/2.10.3/temporary-pip-install

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# This script is specifically designed for temporarily installing packages needed ONLY before bootstrap steps.
+# It intentionally bypasses constraint checks, since it is intended that the packages will be used for setup/configuration 
+# and then UNINSTALLED before the bootstrap steps, during local setup.
+#
+# NOTE: This script should NOT be used for installing production Airflow/MWAA dependencies.
+# For those, use 'safe-pip-install' which properly handles Airflow/MWAA constraints.
+
+pip3 install "$@"