Merge branch 'main' into issue-43043

.asf.yaml

@@ -122,15 +122,14 @@ github:
     # Max 10 collaborators allowed
     # https://github.com/apache/infrastructure-asfyaml/blob/main/README.md#assigning-the-github-triage-role-to-external-collaborators
     - aritra24
-    - dirrao
     - omkar-foss
     - rawwar
     - nathadfield
     - sunank200
     - vatsrahul1001
     - cmarteepants
-    - gopidesupavan
     - bugraoz93
+    - briana-okyere
 
 notifications:
   jobs: [email protected]

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 
 # Kubernetes
 /airflow/kubernetes/ @dstandish @jedcunningham
-/airflow/providers/cncf/kubernetes/executors/ @dstandish @jedcunningham
+/providers/src/airflow/providers/cncf/kubernetes/executors/ @dstandish @jedcunningham
 
 # Helm Chart
 /chart/ @dstandish @jedcunningham @hussein-awala
@@ -61,27 +61,28 @@
 /airflow/secrets  @dstandish @potiuk @ashb
 
 # Providers
-/airflow/providers/amazon/ @eladkal @o-nikolas
-/airflow/providers/celery/ @hussein-awala
-/airflow/providers/cncf/kubernetes @jedcunningham @hussein-awala
-/airflow/providers/common/sql/ @eladkal
-/airflow/providers/dbt/cloud/ @josh-fell
-/airflow/providers/edge @jscheffl
-/airflow/providers/hashicorp/ @hussein-awala
-/airflow/providers/openlineage/ @mobuchowski
-/airflow/providers/slack/ @eladkal
-/airflow/providers/smtp/ @hussein-awala
-/airflow/providers/snowflake/ @potiuk @mik-laj
-/airflow/providers/tabular/ @Fokko
+/providers/src/airflow/providers/amazon/ @eladkal @o-nikolas
+/providers/src/airflow/providers/celery/ @hussein-awala
+/providers/src/airflow/providers/cncf/kubernetes @jedcunningham @hussein-awala
+/providers/src/airflow/providers/common/sql/ @eladkal
+/providers/src/airflow/providers/dbt/cloud/ @josh-fell
+/providers/src/airflow/providers/edge @jscheffl
+/providers/src/airflow/providers/hashicorp/ @hussein-awala
+/providers/src/airflow/providers/openlineage/ @mobuchowski
+/providers/src/airflow/providers/slack/ @eladkal
+/providers/src/airflow/providers/smtp/ @hussein-awala
+/providers/src/airflow/providers/snowflake/ @potiuk @mik-laj
+/providers/src/airflow/providers/apache/iceberg/ @Fokko
+/providers/tests/apache/iceberg/ @Fokko
 /docs/apache-airflow-providers-amazon/ @eladkal @o-nikolas
 /docs/apache-airflow-providers-cncf-kubernetes @jedcunningham
 /docs/apache-airflow-providers-common-sql/ @eladkal
 /docs/apache-airflow-providers-openlineage/ @mobuchowski
 /docs/apache-airflow-providers-slack/ @eladkal
-/tests/providers/amazon/ @eladkal @o-nikolas
-/tests/providers/common/sql/ @eladkal
-/tests/providers/openlineage/ @mobuchowski
-/tests/providers/slack/ @eladkal
+/providers/tests/amazon/ @eladkal @o-nikolas
+/providers/tests/common/sql/ @eladkal
+/providers/tests/openlineage/ @mobuchowski
+/providers/tests/slack/ @eladkal
 /tests/system/providers/amazon/ @eladkal @o-nikolas
 
 # Dev tools
@@ -108,6 +109,6 @@ ISSUE_TRIAGE_PROCESS.rst @eladkal
 
 # AIP-58 - Object Storage
 /airflow/io/ @bolkedebruin
-/airflow/providers/**/fs/ @bolkedebruin
-/airflow/providers/common/io/ @bolkedebruin
+/providers/src/airflow/providers/**/fs/ @bolkedebruin
+/providers/src/airflow/providers/common/io/ @bolkedebruin
 /docs/apache-airflow/core-concepts/objectstorage.rst @bolkedebruin

.github/ISSUE_TEMPLATE/airflow_bug_report.yml

@@ -25,7 +25,7 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.10.2"
+        - "2.10.3"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"
     validations:

.github/actions/install-pre-commit/action.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    default: 3.9
+  uv-version:
+    description: 'uv version to use'
+    default: 0.4.30
+  pre-commit-version:
+    description: 'pre-commit version to use'
+    default: 4.0.1
+  pre-commit-uv-version:
+    description: 'pre-commit-uv version to use'
+    default: 4.1.4
+runs:
+  using: "composite"
+  steps:
+    - name: Install pre-commit, uv, and pre-commit-uv
+      shell: bash
+      run: >
+        pip install
+        pre-commit==${{inputs.pre-commit-version}}
+        uv==${{inputs.uv-version}}
+        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+    - name: Cache pre-commit envs
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+        restore-keys: |
+          pre-commit-${{inputs.python-version}}-

.github/boring-cyborg.yml

@@ -626,7 +626,7 @@ labelPRBasedOnFilePath:
 
   area:Scheduler:
     - airflow/jobs/**/*
-    - airflow/task/task_runner/**/*
+    - airflow/task/standard_task_runner.py
     - airflow/dag_processing/**/*
     - docs/apache-airflow/administration-and-deployment/scheduler.rst
     - tests/jobs/**/*

.github/workflows/additional-ci-image-checks.yml

@@ -84,6 +84,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources (true/false)"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        required: true
+        type: string
 jobs:
   # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
   # tests to complete - it is run very early in the build process for "main" merges in order to refresh
@@ -113,7 +117,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ inputs.use-uv}}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -170,7 +174,7 @@ jobs:
 #      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: "true"
+#      use-uv: ${{ inputs.use-uv}}
 #      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
 #      docker-cache: ${{ inputs.docker-cache }}
 #      disable-airflow-repo-cache:  ${{ inputs.disable-airflow-repo-cache }}

.github/workflows/basic-tests.yml

@@ -189,7 +189,7 @@ jobs:
         run: |
           python -m pip install --upgrade pipx
           pipx ensurepath
-          pipx install hatch --force
+          pipx install hatch==1.9.7
       - name: Run tests
         run: hatch run run-coverage
         env:
@@ -201,11 +201,13 @@ jobs:
               --package-format wheel --skip-tag-check
       - name: "Install Airflow with fab for webserver tests"
         run: pip install . dist/apache_airflow_providers_fab-*.whl
-      - name: "Prepare Standard provider packages: wheel"
-        run: >
-          breeze release-management prepare-provider-packages standard --package-format wheel --skip-tag-check
       - name: "Install Airflow with standard provider for webserver tests"
         run: pip install . dist/apache_airflow_providers_standard-*.whl
+      - name: "Prepare Task SDK package: wheel"
+        run: >
+          breeze release-management prepare-task-sdk-package --package-format wheel
+      - name: "Install Task SDK package"
+        run: pip install ./dist/apache_airflow_task_sdk-*.whl
       - name: "Install Python client"
         run: pip install ./dist/apache_airflow_client-*.whl
       - name: "Initialize Airflow DB and start webserver"
@@ -283,16 +285,11 @@ jobs:
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: "\
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
-            ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

.github/workflows/build-images.yml

@@ -72,6 +72,7 @@ jobs:
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       default-branch: ${{ steps.selective-checks.outputs.default-branch }}
       disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
       runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
       runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
@@ -203,7 +204,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -248,7 +249,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: linux/amd64
       python-versions: ${{ needs.build-info.outputs.python-versions }}

.github/workflows/ci.yml

@@ -77,6 +77,7 @@ jobs:
       default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
       default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
@@ -205,7 +206,7 @@ jobs:
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -271,6 +272,7 @@ jobs:
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
 
 
   generate-constraints:
@@ -312,6 +314,7 @@ jobs:
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
     name: "Provider checks"
@@ -556,7 +559,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
@@ -639,6 +642,7 @@ jobs:
       kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }}
       kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||

.github/workflows/finalize-tests.yml

@@ -149,7 +149,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}

.github/workflows/integration-tests.yml

@@ -59,6 +59,7 @@ on:  # yamllint disable-line rule:truthy
 jobs:
   tests-integration:
     timeout-minutes: 130
+    if: inputs.testable-integrations != '[]'
     name: "Integration Tests: ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:

.github/workflows/k8s-tests.yml

@@ -44,6 +44,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to include success outputs"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv"
+        required: true
+        type: string
       debug-resources:
         description: "Whether to debug resources"
         required: true
@@ -96,6 +100,9 @@ jobs:
           key: "\
             k8s-env-${{ steps.breeze.outputs.host-python-version }}-\
             ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}"
+      - name: "Switch breeze to use uv"
+        run: breeze setup config --use-uv
+        if: inputs.use-uv == 'true'
       - name: Run complete K8S tests ${{ inputs.kubernetes-combos-list-as-string }}
         run: breeze k8s run-complete-tests --run-in-parallel --upgrade --no-copy-local-sources
         env:

.github/workflows/prod-image-build.yml

@@ -181,6 +181,11 @@ jobs:
         run: >
           breeze release-management prepare-airflow-package --package-format wheel
         if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
+      - name: "Prepare task-sdk package"
+        shell: bash
+        run: >
+          breeze release-management prepare-task-sdk-package --package-format wheel
+        if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
       - name: "Upload prepared packages as artifacts"
         uses: actions/upload-artifact@v4
         with:

.github/workflows/release_dockerhub_image.yml

@@ -85,6 +85,7 @@ jobs:
       "kaxil",
       "pierrejeambrun",
       "potiuk",
+      "utkarsharma2"
       ]'), github.event.sender.login)
     steps:
       - name: "Cleanup repo"