From f4e8abedbe89fd4be40511125cc239511443bede Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 30 Jan 2024 18:06:33 +0100 Subject: [PATCH] Add Poetry for dependency and packaging management (#32) --- .gitignore | 3 ++ CONTRIBUTING.md | 37 +++++++++++++++++- Makefile | 24 ++++-------- pyproject.toml | 100 ++++++++++++++++++++++++------------------------ 4 files changed, 95 insertions(+), 69 deletions(-) diff --git a/.gitignore b/.gitignore index 64205a78a..c19ae5d49 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ __pycache__ .envrc .python-version +poetry.toml +poetry.lock + *.egg-info/ *.egg dist/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43262f8e1..ff4e1c267 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,41 @@ Docusaurus and published to GitHub pages. ## Release process -Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automatically through GitHub Actions. +Currently, releases to [PyPI](https://pypi.org/project/crawlee) are published manually. +To release a new version, follow these steps: + +At first, update the version number under `tool.poetry.version` in `pyproject.toml`. + +```toml +[tool.poetry] +name = "crawlee" +version = "x.z.y" +``` + +Generate the distribution archives for the package using Poetry. + +```shell +poetry build +``` + +Set up the PyPI API token for authentication. Replace `YOUR_API_TOKEN` with the actual PyPI token. + +```shell +poetry config pypi-token.pypi YOUR_API_TOKEN +``` + +Upload the package to the PyPI. + +```shell +poetry publish +``` + + ### Beta release checklist diff --git a/Makefile b/Makefile index 9484e8e40..b8846ec29 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean install-dev build publish twine-check lint unit-tests integration-tests type-check check-code format check-version-availability check-changelog-entry +.PHONY: clean install-dev lint type-check unit-tests unit-tests-cov integration-tests check-code format check-version-availability check-changelog-entry DIRS_WITH_CODE = src tests scripts @@ -6,25 +6,18 @@ DIRS_WITH_CODE = src tests scripts INTEGRATION_TESTS_CONCURRENCY = 1 clean: - rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__ + rm -rf .mypy_cache .pytest_cache .ruff_cache build dist htmlcov .coverage install-dev: - python3 -m pip install --upgrade pip - pip install --no-cache-dir -e ".[dev]" - pre-commit install - -build: - python3 -m build - -publish: - python3 -m twine upload dist/* - -twine-check: - python3 -m twine check dist/* + poetry install + source .venv/bin/activate && pre-commit install lint: python3 -m ruff check $(DIRS_WITH_CODE) +type-check: + python3 -m mypy $(DIRS_WITH_CODE) + unit-tests: python3 -m pytest -n auto -ra tests/unit --cov=src/crawlee @@ -34,9 +27,6 @@ unit-tests-cov: integration-tests: python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration -type-check: - python3 -m mypy $(DIRS_WITH_CODE) - check-code: lint type-check unit-tests format: diff --git a/pyproject.toml b/pyproject.toml index 29c770914..7496ef7d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,15 @@ -[project] +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] name = "crawlee" version = "0.0.1" description = "Crawlee for Python" +authors = ["Apify Technologies s.r.o. "] +license = "Apache-2.0" readme = "README.md" -license = { text = "Apache Software License" } -authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }] -keywords = ["apify", "automation", "chrome", "crawler", "headless", "scraper", "scraping"] - +packages = [{ include = "crawlee", from = "src" }] classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", @@ -18,56 +21,51 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries", ] - -requires-python = ">=3.9" - -# We use inclusive ordered comparison clause for non-Apify packages intentionally in order to enhance the Apify SDK's -# compatibility with a wide range of external packages. This decision was discussed in detail in the following PR: -# https://github.com/apify/apify-sdk-python/pull/154 -dependencies = [ - "colorama >= 0.4.6", - "more_itertools >= 10.2.0", - "pyee >= 11.1.0", - "typing-extensions >= 4.1.0", -] - -[project.optional-dependencies] -dev = [ - "build ~= 1.0.3", - "filelock ~= 3.13.1", - "mypy ~= 1.8.0", - "pre-commit ~= 3.4.0", - "pydoc-markdown ~= 4.8.2", - "pytest ~= 7.4.4", - "pytest-asyncio ~= 0.23.4", - "pytest-cov ~= 4.1.0", - "pytest-only ~= 2.0.0", - "pytest-timeout ~= 2.2.0", - "pytest-xdist ~= 3.5.0", - "respx ~= 0.20.1", - "ruff ~= 0.1.13", - "twine ~= 4.0.2", - "types-colorama ~= 0.4.15.20240106", +keywords = [ + "apify", + "automation", + "chrome", + "crawlee", + "crawler", + "headless", + "scraper", + "scraping", ] -[project.urls] +[tool.poetry.urls] "Homepage" = "https://todo.com/" -"Documentation" = "https://todo.com/" -"Source" = "https://github.com/apify/crawlee-py" -"Issue tracker" = "https://github.com/apify/crawlee-py/issues" -"Changelog" = "https://github.com/apify/crawlee-py/blob/master/CHANGELOG.md" "Apify Homepage" = "https://apify.com" - -[build-system] -requires = ["setuptools>=64.0.0", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.packages.find] -where = ["src"] -include = ["crawlee*"] - -[tool.setuptools.package-data] -crawlee = ["py.typed"] +"Changelog" = "https://github.com/apify/crawlee-py/blob/master/CHANGELOG.md" +"Documentation" = "https://todo.com/" +"Issue Tracker" = "https://github.com/apify/crawlee-py/issues" +"Repository" = "https://github.com/apify/crawlee-py" + +# We use inclusive ordered comparison clauses for external packages intentionally in order to enhance Crawlee's +# compatibility with external packages. This decision was discussed in detail in the following PR: +# https://github.com/apify/apify-sdk-python/pull/154. +[tool.poetry.dependencies] +python = "^3.9" +colorama = "^0.4.6" +more_itertools = "^10.2.0" +pyee = "^11.1.0" +typing-extensions = "^4.1.0" + +[tool.poetry.group.dev.dependencies] +build = "~1.0.3" +filelock = "~3.13.1" +mypy = "~1.8.0" +pre-commit = "~3.4.0" +pydoc-markdown = "~4.8.2" +pytest = "~7.4.4" +pytest-asyncio = "~0.23.4" +pytest-cov = "~4.1.0" +pytest-only = "~2.0.0" +pytest-timeout = "~2.2.0" +pytest-xdist = "~3.5.0" +respx = "~0.20.1" +ruff = "~0.1.13" +twine = "~4.0.2" +types-colorama = "~0.4.15.20240106" [tool.ruff] line-length = 120