diff --git a/images/airflow/2.9.2/docker-compose.yaml b/images/airflow/2.9.2/docker-compose.yaml index 7196ee4..6b7b3e5 100644 --- a/images/airflow/2.9.2/docker-compose.yaml +++ b/images/airflow/2.9.2/docker-compose.yaml @@ -20,7 +20,7 @@ x-airflow-common: &airflow-common # Additional Airflow configuration can be passed here in JSON form. MWAA__CORE__CREATED_AT: "Tue Sep 18 23:05:58 UTC 2024" MWAA__CORE__CUSTOM_AIRFLOW_CONFIGS: "{}" - MWAA__CORE__FERNET_KEY: '{"FernetKey": "fake-key-nNge+lks3RBeGVrnZ1Dq5GjKerbZKmb7dXNnsNsGy3E="}' + MWAA__CORE__FERNET_KEY: ${FERNET_KEY} MWAA__WEBSERVER__SECRET: '{"secret_key": "fake-key-aYDdF6d+Fjznai5yBW63CUAi0IipJqDHlNSWIun6y8o="}' # Use this enviornment variable to enable encryption with KMS. MWAA__CORE__KMS_KEY_ARN: ${MWAA__CORE__KMS_KEY_ARN} diff --git a/images/airflow/2.9.2/generate_fernet_key.py b/images/airflow/2.9.2/generate_fernet_key.py new file mode 100755 index 0000000..96b79d6 --- /dev/null +++ b/images/airflow/2.9.2/generate_fernet_key.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +""" +This Module generates Fernet keys, which are used by Airflow for connection encryption +""" + +from cryptography.fernet import Fernet +import json + +def generate_fernet_key(): + """ + Generate a Fernet key and return it as a JSON string. + + :returns A JSON string containing the generated Fernet key in the format {"FernetKey": ""} + """ + key = Fernet.generate_key().decode() + return json.dumps({"FernetKey": key}) + +if __name__ == "__main__": + print(generate_fernet_key()) diff --git a/images/airflow/2.9.2/run.sh b/images/airflow/2.9.2/run.sh index 619e46b..2f35cbe 100755 --- a/images/airflow/2.9.2/run.sh +++ b/images/airflow/2.9.2/run.sh @@ -10,6 +10,37 @@ else CONTAINER_RUNTIME="docker" fi +# Generate valid Fernet key as json +generate_fernet_key() { + + # Install cryptography package quietly + chmod +x temporary-pip-install generate_fernet_key.py + ./temporary-pip-install cryptography >/dev/null 2>&1 + + # Generate the key and format as JSON + KEY=$(python3 generate_fernet_key.py) + + # Uninstall cryptography package quietly + python3 -m pip uninstall -y cryptography cryptography-vectors &>/dev/null 2>&1 + + echo "$KEY" +} + +# Set up cache directory ; generate if it dosen't exist +CACHE_DIR="${HOME}/.cache/mwaa-local" +FERNET_KEY_FILE="${CACHE_DIR}/fernet.key" +mkdir -p "${CACHE_DIR}" + +# Check if we have a cached Fernet key, if not generate and cache it +if [ ! -f "${FERNET_KEY_FILE}" ]; then + generate_fernet_key > "${FERNET_KEY_FILE}" + chmod 600 "${FERNET_KEY_FILE}" +fi + +# Read the Fernet key from cache +FERNET_KEY=$(cat "${FERNET_KEY_FILE}") +export FERNET_KEY + # Build the Docker image ./build.sh $CONTAINER_RUNTIME diff --git a/images/airflow/2.9.2/temporary-pip-install b/images/airflow/2.9.2/temporary-pip-install new file mode 100755 index 0000000..a8fe793 --- /dev/null +++ b/images/airflow/2.9.2/temporary-pip-install @@ -0,0 +1,10 @@ +#!/bin/bash + +# This script is specifically designed for temporarily installing packages needed ONLY before bootstrap steps. +# It intentionally bypasses constraint checks, since it is intended that the packages will be used for setup/configuration +# and then UNINSTALLED before the bootstrap steps, during local setup. +# +# NOTE: This script should NOT be used for installing production Airflow/MWAA dependencies. +# For those, use 'safe-pip-install' which properly handles Airflow/MWAA constraints. + +pip3 install "$@" \ No newline at end of file diff --git a/quality-checks/lint_bash.sh b/quality-checks/lint_bash.sh index 75b0278..af24666 100755 --- a/quality-checks/lint_bash.sh +++ b/quality-checks/lint_bash.sh @@ -8,10 +8,9 @@ if [[ "$PWD" != "$REPO_ROOT" ]]; then echo "The script must be run from the repo root. Please cd into the repo root directory and type: ./quality-checks/${SCRIPT_NAME}" exit 1 fi - -# Lint all Bash files +# Lint all Bash files, excluding .venv directory echo "Running ShellCheck on Bash scripts..." -if ! find . -type f -name "*.sh" -exec shellcheck {} +; then +if ! find . -type f -name "*.sh" -not -path "./.venv/*" -exec shellcheck {} +; then echo "ShellCheck linting failed." exit 1 else diff --git a/quality-checks/pip_install_check.py b/quality-checks/pip_install_check.py index 34a1eca..27a263a 100755 --- a/quality-checks/pip_install_check.py +++ b/quality-checks/pip_install_check.py @@ -1,4 +1,4 @@ -#!/bin/python3 +#!/usr/bin/env python3 """ This module verifies there are no direct use of "pip install" in the code. diff --git a/quality-checks/run_all.py b/quality-checks/run_all.py index be2f5e6..838d8b3 100755 --- a/quality-checks/run_all.py +++ b/quality-checks/run_all.py @@ -1,4 +1,4 @@ -#!/bin/python3 +#!/usr/bin/env python3 """Run all quality check scripts under the quality-checks/ folder.""" import os