Docs(python-cdk): Add automated PDoc docs generation CI job (#46977)

.github/workflows/python-cdk-pdoc-generate.yml

@@ -0,0 +1,37 @@
+name: "Python CDK: Generate Docs"
+
+on:
+  push:
+    branches:
+      - main
+  pull_request: {}
+
+jobs:
+  preview_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "1.7.1"
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: cd airbyte-cdk/python && poetry install --all-extras
+
+      - name: Generate documentation
+        run: |
+          cd airbyte-cdk/python && poetry run poe docs-generate
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: "docs-generated"
+          path: "airbyte-cdk/python/docs/generated"

airbyte-cdk/python/airbyte_cdk/__init__.py

@@ -1,7 +1,51 @@
-#
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
+"""
+# Welcome to the Airbyte Python CDK!
 
+The Airbyte Python CDK is a Python library that provides a set of tools to help you build
+connectors for the Airbyte platform.
+
+## Building Source Connectors
+
+To build a source connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.sources`
+- `airbyte_cdk.sources.concurrent_source`
+- `airbyte_cdk.sources.config`
+- `airbyte_cdk.sources.file_based`
+- `airbyte_cdk.sources.streams`
+
+## Building Destination Connectors
+
+To build a destination connector, you will want to refer to
+the following classes and modules:
+
+- `airbyte_cdk.destinations`
+- `airbyte_cdk.destinations.Destination`
+- `airbyte_cdk.destinations.vector_db_based`
+
+## Working with Airbyte Protocol Models
+
+The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
+
+- `airbyte_cdk.models.airbyte_protocol`
+- `airbyte_cdk.models.airbyte_protocol_serializers`
+
+---
+
+API Reference
+
+---
+
+"""
+
+# Warning: The below imports are not stable and will cause circular
+# dependencies if auto-sorted with isort. Please keep them in the same order.
+# TODO: Submodules should import from lower-level modules, rather than importing from here.
+# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
+# hints - again, to avoid circular dependencies.
+# Once those issues are resolved, the below can be sorted with isort.
 from importlib import metadata
 
 from .destinations import Destination
@@ -88,11 +132,9 @@
     # Availability strategy
     "AvailabilityStrategy",
     "HttpAvailabilityStrategy",
-
     # Checkpoint
     "LegacyCursor",
     "ResumableFullRefreshCursor",
-
     # Concurrent
     "ConcurrentCursor",
     "ConcurrentSource",
@@ -104,11 +146,9 @@
     "FinalStateCursor",
     "IsoMillisConcurrentStreamStateConverter",
     "StreamFacade",
-
     # Config observation
     "create_connector_config_control_message",
     "emit_configuration_as_airbyte_control_message",
-
     # Connector
     "AbstractSource",
     "BaseConfig",
@@ -117,7 +157,6 @@
     "Destination",
     "Source",
     "TState",
-
     # Declarative
     "AddFields",
     "AddedFieldDefinition",
@@ -167,11 +206,9 @@
     "StreamSlice",
     "SubstreamPartitionRouter",
     "YamlDeclarativeSource",
-
     # Entrypoint
     "launch",
     "AirbyteEntrypoint",
-
     # HTTP
     "AbstractAPIBudget",
     "AbstractHeaderAuthenticator",
@@ -192,11 +229,9 @@
     "SingleUseRefreshTokenOauth2Authenticator",
     "TokenAuthenticator",
     "UserDefinedBackoffException",
-
     # Logger
     "AirbyteLogFormatter",
     "init_logger",
-
     # Protocol classes
     "AirbyteStream",
     "AirbyteConnectionStatus",
@@ -215,20 +250,16 @@
     "ConnectorSpecification",
     "Level",
     "AirbyteRecordMessage",
-
     # Repository
     "InMemoryMessageRepository",
     "MessageRepository",
-
     # State management
     "ConnectorStateManager",
-
     # Stream
     "IncrementalMixin",
     "Stream",
     "StreamData",
     "package_name_from_class",
-
     # Utils
     "AirbyteTracedException",
     "is_cloud_environment",
@@ -244,7 +275,6 @@
     "OneOfOptionConfig",
     "resolve_refs",
     "as_airbyte_message",
-
     # Types
     "Config",
     "Record",

airbyte-cdk/python/airbyte_cdk/destinations/__init__.py

@@ -1,7 +1,8 @@
-#
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
+"""The destinations module provides classes for building destination connectors."""
 
 from .destination import Destination
 
-__all__ = ["Destination"]
+__all__ = [
+    "Destination",
+]

airbyte-cdk/python/airbyte_cdk/sources/__init__.py

@@ -19,4 +19,8 @@
 # this will not be thread-safe.
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 
-__all__ = ["AbstractSource", "BaseConfig", "Source"]
+__all__ = [
+    "AbstractSource",
+    "BaseConfig",
+    "Source",
+]

airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/__init__.py

@@ -1,3 +1,8 @@
-#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
+"""The concurrent source model replaces the legacy Source model.
+
+The concurrent source model is a new way to build sources in the Airbyte CDK. It is designed to
+be more ergonomic and performant than the legacy Source model.
+
+To implement a source using the concurrent source model, check out the submodules in this package.
+"""