Temporary change to CI so we can see the changes in the current branc… #5919

Workflow file for this run

.github/workflows/ci-python-sdk.yaml at a5471f0

	name: Build and test astro Python SDK
	on:
	push:
	branches: [ 'main', 'release-**', 'fix-ci' ]
	paths:
	- 'python-sdk/**'
	- '.github/workflows/ci-python-sdk.yaml'
	# Run on PRs from forks
	pull_request_target:
	branches: [ 'main' ]
	types: ['labeled']
	paths:
	- 'python-sdk/**'
	- '.github/workflows/ci-python-sdk.yaml'
	- '*'
	release:
	types: [ 'created' ]
	defaults:
	run:
	working-directory: python-sdk

	# This allows a subsequently queued workflow run to interrupt and cancel previous runs
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	# This list should only have non-sensitive env vars
	# Env vars with secrets should be in the specific jobs
	env:
	SETUPTOOLS_USE_DISTUTILS: stdlib
	POSTGRES_HOST: postgres
	POSTGRES_PORT: 5432
	AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci
	REDSHIFT_DATABASE: dev
	REDSHIFT_HOST: utkarsh-cluster.cdru7mxqmtyx.us-east-2.redshift.amazonaws.com
	SNOWFLAKE_SCHEMA: ASTROFLOW_CI
	SNOWFLAKE_DATABASE: SANDBOX
	SNOWFLAKE_WAREHOUSE: DEMO
	SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com
	SNOWFLAKE_ACCOUNT: gp21411
	SNOWFLAKE_REGION: us-east-1
	SNOWFLAKE_ROLE: AIRFLOW_TEST_USER
	SFTP_HOSTNAME: ${{ secrets.SFTP_HOSTNAME }}
	SFTP_USERNAME: ${{ secrets.SFTP_USERNAME }}
	SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }}
	AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True
	AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
	AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }}
	AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }}
	AIRFLOW_VAR_FOO: templated_file_name
	AWS_BUCKET: tmp9
	GOOGLE_BUCKET: dag-authoring
	FORCE_COLOR: "true"
	MSSQL_DB: ${{ secrets.MSSQL_DB }}
	MSSQL_HOST: ${{ secrets.MSSQL_HOST }}
	MSSQL_LOGIN: ${{ secrets.MSSQL_LOGIN }}
	MSSQL_PASSWORD: ${{ secrets.MSSQL_PASSWORD }}
	MYSQL_DB: ${{ secrets.MYSQL_DB }}
	MYSQL_HOST: ${{ secrets.MYSQL_HOST }}
	MYSQL_LOGIN: ${{ secrets.MYSQL_LOGIN }}
	MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}

	jobs:
	Markdown-link-check:
	if: github.event.action != 'labeled'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: gaurav-nelson/github-action-markdown-link-check@v1
	with:
	config-file: '.github/workflows/mlc_config.json'

	Type-Check:
	if: github.event.action != 'labeled'
	runs-on: ubuntu-latest
	env:
	MYPY_FORCE_COLOR: 1
	TERM: xterm-color
	SETUPTOOLS_USE_DISTUTILS: stdlib
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }}
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s type_check

	Build-Docs:
	if: github.event.action != 'labeled'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }}
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s build_docs

	Run-Optional-Packages-tests-python-sdk:
	if: >-
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.fork == false
	) \|\|
	(
	github.event_name == 'pull_request_target' &&
	contains(github.event.pull_request.labels.*.name, 'safe to test')
	) \|\|
	(
	github.event_name == 'release'
	)
	runs-on: ubuntu-latest
	services:
	postgres:
	image: dimberman/pagila-test
	env:
	POSTGRES_PASSWORD: postgres
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--name postgres
	ports:
	- 5432:5432
	sftp:
	image: ghcr.io/astronomer/astro-sdk/sftp_docker
	ports:
	- 2222:22
	ftp:
	image: ghcr.io/astronomer/astro-sdk/ftp_docker
	ports:
	- 21:21
	- 30000-30009:30000-30009
	env:
	FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }}
	FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }}
	FTP_USER_HOME: /home/foo
	PUBLICHOST: "localhost"
	steps:
	- uses: actions/checkout@v3
	if: github.event_name != 'pull_request_target'

	- name: Checkout pull/${{ github.event.number }}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	if: github.event_name == 'pull_request_target'

	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml
	- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
	- run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s test_examples_by_dependency -- --cov=src --cov-report=xml --cov-branch
	- name: Upload coverage
	uses: actions/upload-artifact@v4
	with:
	name: coverage${{ matrix.group }}
	path: ./python-sdk/.coverage
	include-hidden-files: true
	env:
	GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
	GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }}
	REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
	REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
	SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
	SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
	AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }}
	AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }}

	Run-Unit-tests-Airflow-2-8:
	strategy:
	matrix:
	version: [ '3.8', '3.9', '3.10', '3.11' ]
	if: >-
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.fork == false
	) \|\|
	(
	github.event_name == 'pull_request_target' &&
	contains(github.event.pull_request.labels.*.name, 'safe to test')
	) \|\|
	(
	github.event_name == 'release'
	)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	if: github.event_name != 'pull_request_target'

	- name: Checkout pull/${{ github.event.number }}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	if: github.event_name == 'pull_request_target'
	- uses: actions/setup-python@v3
	with:
	python-version: ${{ matrix.version }}
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s "test-${{ matrix.version }}(airflow='2.8')" -- tests/ --cov=src --cov-report=xml --cov-branch
	- name: Upload coverage
	uses: actions/upload-artifact@v4
	with:
	name: coverage-unit-test
	path: ./python-sdk/.coverage
	include-hidden-files: true

	Run-load-file-Integration-Airflow-2-8:
	if: >-
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.fork == false
	) \|\|
	(
	github.event_name == 'pull_request_target' &&
	contains(github.event.pull_request.labels.*.name, 'safe to test')
	) \|\|
	(
	github.event_name == 'release'
	)
	strategy:
	fail-fast: false
	matrix:
	group: [ 1, 2, 3]
	runs-on: ubuntu-latest
	services:
	postgres:
	# Docker Hub image
	image: dimberman/pagila-test
	env:
	POSTGRES_PASSWORD: postgres
	# Set health checks to wait until postgres has started
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432
	sftp:
	image: ghcr.io/astronomer/astro-sdk/sftp_docker
	ports:
	- 2222:22
	ftp:
	image: ghcr.io/astronomer/astro-sdk/ftp_docker
	ports:
	- 21:21
	- 30000-30009:30000-30009
	env:
	FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }}
	FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }}
	FTP_USER_HOME: /home/foo
	PUBLICHOST: "localhost"
	steps:
	- uses: actions/checkout@v3
	if: github.event_name != 'pull_request_target'

	- name: Checkout pull/${{ github.event.number }}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	if: github.event_name == 'pull_request_target'
	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-2.5-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml
	- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
	- run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "test_load_file.py and not redshift" --splits 3 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch
	- run: cat /tmp/durations-${{ matrix.group }}
	- name: Upload coverage
	uses: actions/upload-artifact@v4
	with:
	name: coverage-${{ matrix.group }}-integration-tests
	path: ./python-sdk/.coverage
	include-hidden-files: true
	- name: Collect pytest durations
	uses: actions/upload-artifact@v4
	with:
	name: pytest_durations_load_file_${{ matrix.group }}
	path: /tmp/durations-${{ matrix.group }}
	include-hidden-files: true
	env:
	GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
	GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }}
	REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
	REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
	SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
	SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
	AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }}

	Run-example-dag-Integration-Airflow-2-8:
	if: >-
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.fork == false
	) \|\|
	(
	github.event_name == 'pull_request_target' &&
	contains(github.event.pull_request.labels.*.name, 'safe to test')
	) \|\|
	(
	github.event_name == 'release'
	)
	strategy:
	fail-fast: false
	matrix:
	group: [ 1, 2, 3]
	runs-on: ubuntu-latest
	services:
	postgres:
	# Docker Hub image
	image: dimberman/pagila-test
	env:
	POSTGRES_PASSWORD: postgres
	# Set health checks to wait until postgres has started
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432
	sftp:
	image: ghcr.io/astronomer/astro-sdk/sftp_docker
	ports:
	- 2222:22
	ftp:
	image: ghcr.io/astronomer/astro-sdk/ftp_docker
	ports:
	- 21:21
	- 30000-30009:30000-30009
	env:
	FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }}
	FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }}
	FTP_USER_HOME: /home/foo
	PUBLICHOST: "localhost"
	steps:
	- uses: actions/checkout@v3
	if: github.event_name != 'pull_request_target'

	- name: Checkout pull/${{ github.event.number }}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	if: github.event_name == 'pull_request_target'
	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml
	- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
	- run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "test_example_dags.py and not redshift" --splits 3 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch
	- run: cat /tmp/durations-${{ matrix.group }}
	- name: Upload coverage
	uses: actions/upload-artifact@v4
	with:
	name: coverage-${{ matrix.group }}-integration-tests
	path: ./python-sdk/.coverage
	include-hidden-files: true
	- name: Collect pytest durations
	uses: actions/upload-artifact@v4
	with:
	name: pytest_durations_example_dags_${{ matrix.group }}
	path: /tmp/durations-${{ matrix.group }}
	include-hidden-files: true
	env:
	GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
	GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }}
	REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
	REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
	SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
	SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
	AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }}

	Run-Integration-tests-Airflow-2-8:
	if: >-
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.fork == false
	) \|\|
	(
	github.event_name == 'pull_request_target' &&
	contains(github.event.pull_request.labels.*.name, 'safe to test')
	) \|\|
	(
	github.event_name == 'release'
	)
	strategy:
	fail-fast: false
	matrix:
	group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ]
	runs-on: ubuntu-latest
	services:
	postgres:
	# Docker Hub image
	image: dimberman/pagila-test
	env:
	POSTGRES_PASSWORD: postgres
	# Set health checks to wait until postgres has started
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432
	sftp:
	image: ghcr.io/astronomer/astro-sdk/sftp_docker
	ports:
	- 2222:22
	ftp:
	image: ghcr.io/astronomer/astro-sdk/ftp_docker
	ports:
	- 21:21
	- 30000-30009:30000-30009
	env:
	FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }}
	FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }}
	FTP_USER_HOME: /home/foo
	PUBLICHOST: "localhost"
	steps:
	- uses: actions/checkout@v3
	if: github.event_name != 'pull_request_target'

	- name: Checkout pull/${{ github.event.number }}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	if: github.event_name == 'pull_request_target'
	- uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v3
	with:
	path: \|
	~/.cache/pip
	.nox
	key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml
	- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
	- run: sqlite3 /tmp/sqlite_default.db "VACUUM;"
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "not test_load_file.py and not test_example_dags.py and not redshift" --splits 11 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch
	- run: cat /tmp/durations-${{ matrix.group }}
	- name: Upload coverage
	uses: actions/upload-artifact@v4
	with:
	name: coverage-${{ matrix.group }}-integration-tests
	path: ./python-sdk/.coverage
	- name: Collect pytest durations
	uses: actions/upload-artifact@v4
	with:
	name: pytest_durations_integration_tests_${{ matrix.group }}
	path: /tmp/durations-${{ matrix.group }}
	include-hidden-files: true
	env:
	GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }}
	GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }}
	REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }}
	REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }}
	SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }}
	SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
	DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
	DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
	DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
	AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
	AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }}
	AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }}


	Generate-Constraints:
	if: (github.event_name == 'release' \|\| github.event_name == 'push')
	strategy:
	fail-fast: false
	matrix:
	python: [ '3.8', '3.9', '3.10', '3.11' ]
	airflow: [ '2.7', '2.8']
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v3
	with:
	python-version: '${{ matrix.python }}'
	architecture: 'x64'
	- name: Get pip cache dir
	id: pip-cache
	run: \|
	echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_OUTPUT
	- uses: actions/cache@v3
	with:
	path: ${{ steps.pip-cache.outputs.pip_cache_dir }}
	key: constraints-${{ matrix.python }}-${{ matrix.airflow }}-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }}
	- run: pip3 install -U pip wheel build nox
	- run: nox -s build
	- run: pip3 install 'apache-airflow~=${{ matrix.airflow }}' "$(ls dist/*.whl)[all]"
	- run: pip3 list --format=freeze > constraints-${{ matrix.python }}-${{ matrix.airflow }}
	- run: cat constraints-${{ matrix.python }}-${{ matrix.airflow }}
	- name: Upload constraints
	uses: actions/upload-artifact@v3
	with:
	name: constraints-${{ matrix.python }}-${{ matrix.airflow }}
	path: ./python-sdk/constraints-${{ matrix.python }}-${{ matrix.airflow }}
	if-no-files-found: error
	include-hidden-files: true

	Code-Coverage:
	if: github.event.action != 'labeled'
	needs:
	- Run-Unit-tests-Airflow-2-8
	- Run-Integration-tests-Airflow-2-8
	- Run-load-file-Integration-Airflow-2-8
	- Run-example-dag-Integration-Airflow-2-8
	- Run-Optional-Packages-tests-python-sdk
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Set up Python 3.10
	uses: actions/setup-python@v3
	with:
	python-version: '3.10'
	- name: Install coverage
	run: \|
	pip3 install coverage
	- name: Download all artifacts
	uses: actions/download-artifact@v2
	with:
	path: ./python-sdk/coverage
	- name: Run coverage
	run: \|
	coverage combine ./coverage/coverage*/.coverage
	coverage report
	coverage xml
	- name: Upload coverage
	uses: codecov/codecov-action@v2
	with:
	fail_ci_if_error: true
	token: ${{ secrets.CODECOV_TOKEN }}
	flags: PythonSDK
	files: ./python-sdk/coverage.xml

	collect-durations:
	runs-on: ubuntu-latest
	needs:
	- Run-Unit-tests-Airflow-2-8
	- Run-Integration-tests-Airflow-2-8
	- Run-load-file-Integration-Airflow-2-8
	steps:
	- uses: actions/checkout@v3
	- name: Cache
	uses: actions/cache@v3
	with:
	path: .test_durations
	key: test_durations-0-${{ github.run_id }}
	- name: Collect
	uses: actions/download-artifact@v3
	with:
	path: ./python-sdk/artifacts
	- name: Consolidate and print durations
	run: \|
	jq -n -S \
	'reduce (inputs \| to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \
	artifacts/pytest_durations_/ > .test_durations_print && \
	cat .test_durations_print
	- name: Consolidate
	run: \|
	jq -n -S \
	'reduce (inputs \| to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \
	artifacts/pytest_durations_/ > .test_durations

	build-n-publish:
	if: github.event_name == 'release'
	name: Build and publish Python 🐍 distributions 📦 to PyPI
	needs:
	- Run-Unit-tests-Airflow-2-8
	- Run-Integration-tests-Airflow-2-8
	- Run-load-file-Integration-Airflow-2-8
	- Run-example-dag-Integration-Airflow-2-8
	- Run-Optional-Packages-tests-python-sdk
	- Generate-Constraints
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- uses: actions/setup-python@v2
	with:
	python-version: '3.10'
	architecture: 'x64'
	- uses: actions/cache@v2
	with:
	path: ~/.cache/pip
	key: ${{ hashFiles('python-sdk/pyproject.toml') }}
	- run: \|
	sudo apt update
	sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl
	- run: pip3 install nox
	- run: nox -s build
	- run: nox -s release -- dist/*
	env:
	TWINE_USERNAME: __token__
	TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Temporary change to CI so we can see the changes in the current branc… #5919

Workflow file

Temporary change to CI so we can see the changes in the current branc… #5919

Jobs

Run details

Workflow file for this run