Move Edge to new provider structure (apache#45783)

.github/boring-cyborg.yml

@@ -225,9 +225,7 @@ labelPRBasedOnFilePath:
     - providers/tests/system/docker/**/*
 
   provider:edge:
-    - providers/src/airflow/providers/edge/**/*
-    - docs/apache-airflow-providers-edge/**/*
-    - providers/tests/edge/**/*
+    - providers/edge/**
 
   provider:elasticsearch:
     - providers/src/airflow/providers/elasticsearch/**/*

dev/breeze/src/airflow_breeze/commands/testing_commands.py

@@ -1217,8 +1217,16 @@ def _run_test_command(
     perform_environment_checks()
     if skip_providers:
         ignored_path_list = [
-            f"--ignore=providers/tests/{provider_id.replace('.','/')}"
-            for provider_id in skip_providers.split(" ")
+            # TODO(potiuk): remove the old ways once we migrate all providers to the new structure
+            *[
+                f"--ignore=providers/tests/{provider_id.replace('.','/')}"
+                for provider_id in skip_providers.split(" ")
+            ],
+            # New structure
+            *[
+                f"--ignore=providers/{provider_id.replace('.','/')}/tests"
+                for provider_id in skip_providers.split(" ")
+            ],
         ]
         extra_pytest_args = (*extra_pytest_args, *ignored_path_list)
     if run_in_parallel:

dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py

@@ -556,7 +556,11 @@ def _update_source_date_epoch_in_provider_yaml(
 
 def _verify_changelog_exists(package: str) -> Path:
     provider_details = get_provider_details(package)
-    changelog_path = Path(provider_details.root_provider_path) / "CHANGELOG.rst"
+    changelog_path = (
+        Path(provider_details.root_provider_path) / "docs" / "changelog.rst"
+        if provider_details.is_new_structure
+        else Path(provider_details.root_provider_path) / "CHANGELOG.rst"
+    )
     if not os.path.isfile(changelog_path):
         get_console().print(f"\n[error]ERROR: Missing {changelog_path}[/]\n")
         get_console().print("[info]Please add the file with initial content:")

dev/breeze/tests/test_pytest_args_for_test_types.py

@@ -66,7 +66,7 @@
         (
             GroupOfTests.PROVIDERS,
             "Providers",
-            ["providers/airbyte/tests", "providers/tests"],
+            ["providers/airbyte/tests", "providers/edge/tests", "providers/tests"],
         ),
         (
             GroupOfTests.PROVIDERS,
@@ -88,6 +88,7 @@
             "Providers[-amazon,google,microsoft.azure]",
             [
                 "providers/airbyte/tests",
+                "providers/edge/tests",
                 "providers/tests",
                 "--ignore=providers/tests/amazon",
                 "--ignore=providers/amazon",
@@ -105,7 +106,14 @@
         (
             GroupOfTests.PROVIDERS,
             "All-Quarantined",
-            ["providers/airbyte/tests", "providers/tests", "-m", "quarantined", "--include-quarantined"],
+            [
+                "providers/airbyte/tests",
+                "providers/edge/tests",
+                "providers/tests",
+                "-m",
+                "quarantined",
+                "--include-quarantined",
+            ],
         ),
         (
             GroupOfTests.CORE,
@@ -204,6 +212,7 @@ def test_pytest_args_for_missing_provider():
             "Providers",
             [
                 "providers/airbyte/tests",
+                "providers/edge/tests",
                 "providers/tests",
             ],
         ),
@@ -227,6 +236,7 @@ def test_pytest_args_for_missing_provider():
             "Providers[-amazon,google]",
             [
                 "providers/airbyte/tests",
+                "providers/edge/tests",
                 "providers/tests",
             ],
         ),
@@ -235,6 +245,7 @@ def test_pytest_args_for_missing_provider():
             "Providers[-amazon,google] Providers[amazon] Providers[google]",
             [
                 "providers/airbyte/tests",
+                "providers/edge/tests",
                 "providers/tests",
             ],
         ),

docs/.gitignore

@@ -1,3 +1,4 @@
 # TODO(potiuk): change it to apache-airflow-providers-* after all providers are migrated to the new structure
 # Eventually when we swtich to individually build docs for each provider, we should remove this altogether
 apache-airflow-providers-airbyte
+apache-airflow-providers-edge

providers/edge/README.rst

@@ -0,0 +1,63 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-edge``
+
+Release: ``0.10.1pre0``
+
+
+Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites
+
+
+Provider package
+----------------
+
+This is a provider package for ``edge`` provider. All classes for this provider package
+are in ``airflow.providers.edge`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-edge/0.10.1pre0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-edge``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+==================  ==================
+PIP package         Version required
+==================  ==================
+``apache-airflow``  ``>=2.10.0``
+``pydantic``        ``>=2.10.2``
+``retryhttp``       ``>=1.2.0``
+==================  ==================
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-edge/0.10.1pre0/changelog.html>`_.

providers/src/airflow/providers/edge/CHANGELOG.rst → providers/edge/docs/changelog.rst

@@ -146,7 +146,7 @@ Misc
 Misc
 ~~~~
 
-* ``Edge worker supports concurrency slots feature so that jobs which need more concurrency blocking other jobs beeing executed on the same worker in parallel.``
+* ``Edge worker supports concurrency slots feature so that jobs which need more concurrency blocking other jobs being executed on the same worker in parallel.``
 
 0.6.2pre0
 .........

providers/src/airflow/providers/edge/provider.yaml → providers/edge/provider.yaml

@@ -27,11 +27,6 @@ source-date-epoch: 1729683247
 versions:
   - 0.10.1pre0
 
-dependencies:
-  - apache-airflow>=2.10.0
-  - pydantic>=2.10.2
-  - retryhttp>=1.2.0
-
 plugins:
   - name: edge_executor
     plugin-class: airflow.providers.edge.plugins.edge_executor_plugin.EdgeExecutorPlugin

providers/edge/pyproject.toml

@@ -0,0 +1,76 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+[build-system]
+requires = ["flit_core==3.10.1"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-edge"
+version = "0.10.1pre0"
+description = "Provider package apache-airflow-providers-edge for Apache Airflow"
+readme = "README.rst"
+authors = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+maintainers = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+keywords = [ "airflow-provider", "edge", "airflow", "integration" ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Framework :: Apache Airflow",
+    "Framework :: Apache Airflow :: Provider",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.9"
+dependencies = [
+    "apache-airflow>=2.10.0",
+    "pydantic>=2.10.2",
+    "retryhttp>=1.2.0",
+]
+
+[project.urls]
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-edge/0.10.1pre0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-edge/0.10.1pre0/changelog.html"
+"Bug Tracker" = "https://github.com/apache/airflow/issues"
+"Source Code" = "https://github.com/apache/airflow"
+"Slack Chat" = "https://s.apache.org/airflow-slack"
+"Twitter" = "https://x.com/ApacheAirflow"
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = "airflow.providers.edge.get_provider_info:get_provider_info"
+[project.entry-points."airflow.plugins"]
+edge_executor = "airflow.providers.edge.plugins.edge_executor_plugin:EdgeExecutorPlugin"
+
+[tool.flit.module]
+name = "airflow.providers.edge"