Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: use poetry 2.0 #209

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/autofix-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/connector-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ jobs:
uses: actions/checkout@v4
with:
path: airbyte-python-cdk
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "poetry"
- name: Checkout Airbyte Monorepo
uses: actions/checkout@v4
if: steps.no_changes.outputs.status != 'cancelled'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pdoc_preview.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pdoc_publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/poetry-lock-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/pytest_fast.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"

- name: Check Poetry lock file is current
run: poetry check
Expand All @@ -44,7 +44,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pytest_matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
uses: Gr1N/setup-poetry@v9
if: steps.changes.outputs.src == 'true'
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
if: steps.changes.outputs.src == 'true'
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/python_lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand All @@ -40,7 +40,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand All @@ -63,7 +63,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ jobs:
- name: Set up Poetry
uses: Gr1N/setup-poetry@v9
with:
poetry-version: "1.8.4"
poetry-version: "2.0.0"
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
350 changes: 344 additions & 6 deletions poetry.lock

Large diffs are not rendered by default.

163 changes: 85 additions & 78 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
[build-system]
requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
requires = ["poetry-core>=2.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
build-backend = "poetry_dynamic_versioning.backend"


[tool.poetry]
[project]
name = "airbyte-cdk"
description = "A framework for writing Airbyte Connectors."
authors = ["Airbyte <[email protected]>"]
authors = [{name="Airbyte", email="[email protected]"}]
license = "MIT"
readme = "README.md"
homepage = "https://airbyte.com"
repository = "https://github.com/airbytehq/airbyte-python-cdk"
documentation = "https://docs.airbyte.io/"
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
Expand All @@ -24,93 +20,104 @@ keywords = ["airbyte", "connector-development-kit", "cdk"]

# Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
version = "0.0.0" # Version will be calculated dynamically.
requires-python = ">=3.10,<3.13"

dependencies = [
# https://peps.python.org/pep-0508/
# https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release

"airbyte-protocol-models-dataclasses >=0.14, <1.0",
"backoff",
"cachetools",
"dpath >=2.1.6, <3.0",
"dunamai >=1.22.0, <2.0",
"genson ==1.3.0",
"isodate >=0.6.1, < 0.7.0",
"Jinja2 >=3.1.2, <3.2.0",
"jsonref >=0.2, <0.3",
"jsonschema >=4.17.3, <4.18", # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
"pandas ==2.2.2",
"pendulum <3.0.0",
"psutil ==6.1.0",
"pydantic >=2.7, <3.0",
"pyrate-limiter >=3.1.0, <3.2",
"python-dateutil",
"python-ulid ~=3.0",
"PyYAML >=6.0.1, <7.0",
"rapidfuzz >=3.10.1, <4.0",
"requests",
"requests_cache",
"wcmatch ==10.0",
"numpy <=2",
"pyjwt >=2.8.0, <3.0",
"cryptography >=42.0.5, <44.0.0",
"pytz ==2024.2",
"orjson >=3.10.7, <4.0",
"serpyco-rs >=1.10.2, <2.0",
"xmltodict >=0.13.0, <1.0",
"Unidecode >=1.3, <2.0",
]

[project.optional-dependencies]
file-based = [
"avro >=1.11.2, <1.12",
"fastavro >=1.8.0, <1.9",
"pyarrow >=15.0.0, <16.0",
"unstructured[docx,pptx] == 0.10.27",
"unstructured.pytesseract >=0.3.12",
"pytesseract == 0.3.10",
"markdown",
"pdf2image == 1.16.3",
"pdfminer.six == 20221105",
"python-calamine == 0.2.3",
"python-snappy == 0.7.3",
"nltk == 3.9.1",
]

vector-db-based = [
"langchain == 0.1.16",
"langchain_core == 0.1.42",
"tiktoken == 0.8.0",
"openai[embeddings] == 0.27.9",
"cohere == 4.21",
]

sql = [
"sqlalchemy >=2.0, !=2.0.36, <3.0",
]
aaronsteers marked this conversation as resolved.
Show resolved Hide resolved

[project.urls]
Homepage="https://airbyte.com"
Repository="https://github.com/airbytehq/airbyte-python-cdk"
Documentation = "https://docs.airbyte.io/"

[tool.poetry-dynamic-versioning]
enable = true

[tool.poetry.dependencies]
python = "^3.10,<3.13"
airbyte-protocol-models-dataclasses = "^0.14"
backoff = "*"
cachetools = "*"
dpath = "^2.1.6"
dunamai = "^1.22.0"
genson = "1.3.0"
isodate = "~0.6.1"
Jinja2 = "~3.1.2"
jsonref = "~0.2"
jsonschema = "~4.17.3" # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
pandas = "2.2.2"
pendulum = "<3.0.0"
psutil = "6.1.0"
pydantic = "^2.7"
pyrate-limiter = "~3.1.0"
python-dateutil = "*"
python-ulid = "^3.0.0"
PyYAML = "^6.0.1"
rapidfuzz = "^3.10.1"
requests = "*"
requests_cache = "*"
wcmatch = "10.0"
# Extras depedencies
avro = { version = "~1.11.2", optional = true }
cohere = { version = "4.21", optional = true }
fastavro = { version = "~1.8.0", optional = true }
langchain = { version = "0.1.16", optional = true }
langchain_core = { version = "0.1.42", optional = true }
markdown = { version = "*", optional = true }
openai = { version = "0.27.9", extras = ["embeddings"], optional = true }
pdf2image = { version = "1.16.3", optional = true }
"pdfminer.six" = { version = "20221105", optional = true }
pyarrow = { version = "~15.0.0", optional = true }
pytesseract = { version = "0.3.10", optional = true }
python-calamine = { version = "0.2.3", optional = true }
python-snappy = { version = "0.7.3", optional = true }
tiktoken = { version = "0.8.0", optional = true }
nltk = { version = "3.9.1", optional = true }
# This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
# airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict.
numpy = "<2"
unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
"unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
pyjwt = "^2.8.0"
cryptography = ">=42.0.5,<44.0.0"
pytz = "2024.2"
orjson = "^3.10.7"
serpyco-rs = "^1.10.2"
sqlalchemy = {version = "^2.0,!=2.0.36", optional = true }
xmltodict = "^0.13.0"
Unidecode = "^1.3"
[tool.poetry]
requires-poetry = ">=2.0, <3.0"

[tool.poetry.group.dev.dependencies]
freezegun = "*"
mypy = "*"
asyncio = "3.4.3"
ruff = "^0.7.2"
pdoc = "^15.0.0"
poethepoet = "^0.24.2"
pyproject-flake8 = "^6.1.0"
pytest = "^7"
pytest-memray = "^1.6.0"
ruff = ">=0.7.2, <1.0"
pdoc = ">=15.0.0, <16.0"
poethepoet = ">=0.24.2, <1.0"
pyproject-flake8 = ">=6.1.0, <7.0"
pytest = ">=7, <8.0"
pytest-memray = ">=1.6.0, <2.0"
pympler = "*"
pytest-cov = "*"
pytest-httpserver = "*"
pytest-mock = "*"
requests-mock = "*"
# Stubs packages for mypy typing
types-requests = "^2.32.0.20241016"
types-python-dateutil = "^2.9.0.20241003"
types-pyyaml = "^6.0.12.20240917"
types-cachetools = "^5.5.0.20240820"

[tool.poetry.extras]
file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
sql = ["sqlalchemy"]

[tool.poetry.scripts]
types-requests = ">=2.32.0.20241016, <3.0"
types-python-dateutil = ">=2.9.0.20241003, <3.0"
types-pyyaml = ">=6.0.12.20240917, <7.0"
types-cachetools = ">=5.5.0.20240820, <6.0"

source-declarative-manifest = "airbyte_cdk.cli.source_declarative_manifest:run"

[tool.isort]
skip = ["__init__.py"] # TODO: Remove after this is fixed: https://github.com/airbytehq/airbyte-python-cdk/issues/12
Expand Down
Loading