diff --git a/CHANGELOG.md b/CHANGELOG.md index 23db0e1..ec3430e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/) +## [Unreleased] + +### Changed + +- integrate ruff (removing bandit, flake8, black and pylint) +- use plugin base 4.3.0 (cmem-cmempy >= 23.3) +- use poetry-dynamic-versioning option bump=true + - 0.0.1.devX.. instead of 0.0.0.postX + +### Fixed + +- race condition in deploy task (#19) +- missing check for poetry versioning plugin on build task + + ## [5.3.4] 2023-11-06 ### Fixed diff --git a/README.md b/README.md index 25afd66..e470a60 100644 --- a/README.md +++ b/README.md @@ -13,24 +13,33 @@ You can use it to bootstrap an [eccenca Corporate Memory](https://documentation. * [Features](#features) * [Usage](#usage) * [Project Initialization](#project-initialization) - * [Project Update](#project-update) + * [Template Updates](#template-updates) * [Other Tasks](#other-tasks) * [Setup](#setup) * [Local Requirements](#local-requirements) * [Integration Tests](#integration-tests) * [CI Build Plan](#ci-build-plan) + * [Editor / IDE Support](#editor--ide-support) + * [PyCharm](#pycharm) ## Features -- [Python / poetry](https://python-poetry.org/) project with [pylint](https://pylint.pycqa.org/), [pytest](https://www.pytest.org/), [flake8](https://flake8.pycqa.org/), [mypy](http://mypy-lang.org/), [bandit](https://bandit.readthedocs.io/), [memray](https://bloomberg.github.io/memray/) and [safety](https://pyup.io/safety/) integration -- Local build plan with [task](https://taskfile.dev/) (tested for Linux, MacOS and Windows/MinGW) -- [Github build plan](https://github.com/eccenca/cmem-plugin-template/tree/main/src/.github/workflows) -- [Gitlab build plan](https://github.com/eccenca/cmem-plugin-template/blob/main/src/.gitlab-ci.yml) -- Badges, junit XML files and coverage stat generation - +- [Python / poetry](https://python-poetry.org/) project with + - [pytest](https://www.pytest.org/) (incl. [memray](https://bloomberg.github.io/memray/) + [pytest-dotenv](https://github.com/quiqua/pytest-dotenv)) as testing framework, + - [ruff](https://docs.astral.sh/ruff/) as all-hands linter and formatter, + - [mypy](http://mypy-lang.org/) as type checker, and + - [safety](https://pyup.io/safety/) as dependency vulnerability scanner. +- Build plans for + - [gitlab](https://github.com/eccenca/cmem-plugin-template/blob/main/src/.gitlab-ci.yml), + - [github](https://github.com/eccenca/cmem-plugin-template/tree/main/src/.github/workflows), and + - locally with [task](https://taskfile.dev/) (tested for Linux, MacOS and Windows/MinGW). + - Including + - badge generation, + - JUnit XML file and + - coverage stat generation. ## Usage @@ -60,48 +69,52 @@ $ git commit -m "init" $ pre-commit install ``` -Then you can run the local test suite an build a first deployment artifact: +Then you can run the local test suite an build a first deployment artefact: ```shell-session $ task check build ``` -### Project Update +### Template Updates + +We [continuously update](https://github.com/eccenca/cmem-plugin-template/graphs/code-frequency) this repository. +This includes maintenance of dependencies, build plan updates and the adoption of new features from the [plugin base library](https://github.com/eccenca/cmem-plugin-base). -From time to time, this template will be upgraded, so you can update your repository as well: +In order to upgrade your plugin project to the latest template release, use the following command: ```shell-session $ copier update ``` +In order to prepare your plugin project for the upcoming next release, use this command: + ```shell-session $ copier update -r develop ``` -Please have a look at the [copier documentation](https://copier.readthedocs.io/en/stable/updating/). +Please also have a look at the [copier documentation](https://copier.readthedocs.io/en/stable/updating/). ### Other Tasks -Available tasks for your project can be listed like this: +All available tasks for your project can be listed like this: ```shell-session ∴ task task: Available tasks for this project: -* build: Build a tarball and a wheel package -* check: Run whole test suite incl. unit and integration tests -* clean: Removes dist, *.pyc and some caches -* deploy: Install plugin package in Corporate Memory -* check:bandit: Find common security issues -* check:flake8: Enforce standard source code style guide -* check:linters: Run all linter and static code analysis tests -* check:mypy: Find type errors -* check:pylint: Find code smells, errors and style issues -* check:pytest: Run unit and integration tests -* check:safety: Scan dependencies for vulnerabilities -* python:format: Format Python files +* build: Build a tarball and a wheel package +* check: Run whole test suite incl. unit and integration tests +* clean: Removes dist, *.pyc and some caches +* deploy: Install plugin package in Corporate Memory +* check:linters: Run all linter and static code analysis tests +* check:mypy: Complain about typing errors +* check:pytest: Run unit and integration tests +* check:ruff: Complain about everything else +* check:safety: Complain about vulnerabilities in dependencies +* format:fix: Format Python files and fix obvious issues +* format:fix-unsafe: Format Python files and fix 'unsafe' issues ``` -You can extend this task lisk by creating a file `TaskfileCustom.yaml` in your repository root: +You can extend this task list by creating a file `TaskfileCustom.yaml` in your repository root: ```shell-session $ cat TaskfileCustom.yaml @@ -124,8 +137,8 @@ tasks: The following tools are needed for local task execution: - Python 3.11 -- [copier](https://copier.readthedocs.io/) (>= v8.3) for project template rendering -- [poetry](https://python-poetry.org/) (>= v1.6) for packaging and dependency managing (+ [dynamic versioning plugin](https://github.com/mtkennerly/poetry-dynamic-versioning)) +- [copier](https://copier.readthedocs.io/) (>= v9) for project template rendering +- [poetry](https://python-poetry.org/) (>= v1.7) for packaging and dependency managing (+ [dynamic versioning plugin](https://github.com/mtkennerly/poetry-dynamic-versioning)) - [pre-commit](https://pre-commit.com/) (>= v2.20) - managing and maintaining pre-commit hooks - [task](https://taskfile.dev/) (>= v3.29) for build task running (make sure to follow the installation instructions to avoid confusion with taskwarrior) - [cmemc](https://eccenca.com/go/cmemc) (>= v23.1) for interacting with eccenca Corporate Memory @@ -145,8 +158,8 @@ $ poetry self add "poetry-dynamic-versioning[plugin]" ### Integration Tests -This template uses pytest for testing. -Testing your plugin is crucial and should be done locally and integrated with eccenca Corporate Memory. +This template uses the [pytest](https://pytest.org) testing framework. +Testing your plugin is crucial and should be done locally as well as integrated with eccenca Corporate Memory. In order to setup access to a Corporate Memory deployment, you need to provide correct environment variables. Without these variables, only standalone tests can be executed (see `1 skipped`): @@ -157,22 +170,18 @@ $ task check:pytest ... ===== 3 passed, 1 skipped in 0.09s ===== ``` -By giving the correct [cmemc](https://eccenca.com/go/cmemc) [environment variables](https://documentation.eccenca.com/22.1/automate/cmemc-command-line-interface/installation-and-configuration/file-based-configuration/#configuration-variables), your plugin can be tested in an integrated way: +By providing the correct [cmemc](https://eccenca.com/go/cmemc) [environment variables](https://documentation.eccenca.com/latest/automate/cmemc-command-line-interface/installation-and-configuration/file-based-configuration/#configuration-variables) in a `.env` file or directly in your environment, your plugin can be tested in an integrated way: -```shell-session +``` shell-session +# Environment as direct variables: $ export CMEM_BASE_URI="https://cmem.example.org" $ export OAUTH_CLIENT_ID="cmem-service-account" $ export OAUTH_CLIENT_SECRET="..." $ export OAUTH_GRANT_TYPE="client_credentials" -$ task check:pytest -... -... ===== 4 passed in 1.71s ===== - ``` -You can also add these variables to the `.env` file in your repository root (just make sure to never commit this file). - -```shell-session +``` shell-session +# Environment as .env files $ cat .env CMEM_BASE_URI="https://cmem.example.org" OAUTH_CLIENT_ID="cmem-service-account" @@ -182,15 +191,23 @@ OAUTH_GRANT_TYPE="client_credentials" ### CI Build Plan -The gitlab workflow / github action pipelines need the same environment variables as secrets: +The gitlab workflow as well as the github action pipelines need the same environment variables as secrets: - For github, go to Settings > Secret > Actions > [New Repository Secret](https://docs.github.com/en/actions/security-guides/encrypted-secrets) - For gitlab, go to Settings > CI/CD > Variables (Expand) > [Add Variable (protected, masked, all environments)](https://docs.gitlab.com/ee/ci/variables/) -Example github pipelines can be seen [here](https://github.com/eccenca/cmem-plugin-kafka/actions) and [here](https://github.com/eccenca/cmem-plugin-graphql/actions). +An example github pipeline can be seen [here](https://github.com/eccenca/cmem-plugin-kafka/actions). In addition to the eccenca Corporate Memory credential secrets, a `PYPI_TOKEN` secret can be set in order to use the `publish` task/workflow. +### Editor / IDE Support + +#### PyCharm + +In order to have the best PyCharm experience when starting a project with this template, we suggest the following PyCharm plugins: + +- [Ruff](https://plugins.jetbrains.com/plugin/20574-ruff) will provide the linting hints which will be raised by the pipeline anyway. +- [Taskfile](https://plugins.jetbrains.com/plugin/17058-taskfile) will allow for starting tasks. [version-shield]: https://img.shields.io/github/v/tag/eccenca/cmem-plugin-template?label=version&sort=semver [changelog]: https://github.com/eccenca/cmem-plugin-template/blob/main/CHANGELOG.md diff --git a/src/.flake8 b/src/.flake8 deleted file mode 100644 index ffd12bc..0000000 --- a/src/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8 -[flake8] -max-line-length = 88 -extend-ignore = E203 diff --git a/src/.github/workflows/check.yml b/src/.github/workflows/check.yml index 6e645ce..d5bd94d 100644 --- a/src/.github/workflows/check.yml +++ b/src/.github/workflows/check.yml @@ -42,21 +42,13 @@ jobs: run: | poetry self add "poetry-dynamic-versioning[plugin]" - - name: bandit - run: | - task check:bandit - - - name: flake8 - run: | - task check:flake8 - - name: mypy run: | task check:mypy - - name: pylint + - name: ruff run: | - task check:pylint + task check:ruff - name: pytest env: diff --git a/src/.gitlab-ci.yml b/src/.gitlab-ci.yml index 245f02b..13f67e9 100644 --- a/src/.gitlab-ci.yml +++ b/src/.gitlab-ci.yml @@ -19,25 +19,15 @@ stages: - build - publish -bandit: +ruff: stage: test script: - - task check:bandit + - task check:ruff artifacts: when: always reports: junit: - - dist/junit-bandit.xml - -flake8: - stage: test - script: - - task check:flake8 - artifacts: - when: always - reports: - junit: - - dist/junit-flake8.xml + - dist/junit-ruff.xml mypy: stage: test @@ -49,16 +39,6 @@ mypy: junit: - dist/junit-mypy.xml -pylint: - stage: test - script: - - task check:pylint - artifacts: - when: always - reports: - junit: - - dist/junit-pylint.xml - pytest: stage: test coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' @@ -86,7 +66,6 @@ safety: build: stage: build needs: - - bandit - mypy - pytest - safety @@ -102,8 +81,7 @@ pypi: # publishing only available on a tag stage: publish needs: - - flake8 - - pylint + - ruff - build allow_failure: true when: manual diff --git a/src/Taskfile.yaml b/src/Taskfile.yaml index 0b4527a..5578f6a 100644 --- a/src/Taskfile.yaml +++ b/src/Taskfile.yaml @@ -66,11 +66,19 @@ tasks: cmds: - poetry install - python:format: - desc: Format Python files + format:fix: + desc: Format Python files and fix obvious issues <<: *preparation cmds: - - poetry run black . + - poetry run ruff format tests {{.PACKAGE}} + - poetry run ruff check tests {{.PACKAGE}} --fix-only + + format:fix-unsafe: + desc: Format Python files and fix 'unsafe' issues + <<: *preparation + cmds: + - poetry run ruff format tests {{.PACKAGE}} + - poetry run ruff check tests {{.PACKAGE}} --fix-only --unsafe-fixes clean: desc: Removes dist, *.pyc and some caches @@ -83,18 +91,16 @@ tasks: check: desc: Run whole test suite incl. unit and integration tests - deps: - - check:linters - - check:pytest + cmds: + - task: check:linters + - task: check:pytest check:linters: desc: Run all linter and static code analysis tests - deps: - - check:bandit - - check:flake8 - - check:mypy - - check:pylint - - check:safety + cmds: + - task: check:ruff + - task: check:mypy + - task: check:safety check:pytest: desc: Run unit and integration tests @@ -125,19 +131,8 @@ tasks: BADGE_COVERAGE: ./{{.DIST_DIR}}/badge-coverage.svg BADGE_TESTS: ./{{.DIST_DIR}}/badge-tests.svg - check:pylint: - desc: Find code smells, errors and style issues - <<: *preparation - cmds: - - poetry run pylint --exit-zero {{.PACKAGE}} - - poetry run pylint {{.PACKAGE}} {{.XML_PARAMS}} - vars: - FORMAT: --output-format=pylint_junit.JUnitReporter - JUNIT_FILE: ./{{.DIST_DIR}}/junit-pylint.xml - XML_PARAMS: --output={{.JUNIT_FILE}} {{.FORMAT}} - check:mypy: - desc: Find type errors + desc: Complain about typing errors <<: *preparation cmds: - poetry run mypy -p tests -p {{.PACKAGE}} --junit-xml {{.JUNIT_FILE}} @@ -145,31 +140,23 @@ tasks: JUNIT_FILE: ./{{.DIST_DIR}}/junit-mypy.xml check:safety: - desc: Scan dependencies for vulnerabilities + desc: Complain about vulnerabilities in dependencies <<: *preparation cmds: # ignore 51358 safety - dev dependency only # ignore 61489 pillow - dev dependency only - poetry run safety check -i 51358 -i 61489 - check:bandit: - desc: Find common security issues - <<: *preparation - cmds: - - poetry run bandit --exit-zero -r {{.PACKAGE}} - - poetry run bandit --format xml -r {{.PACKAGE}} -o {{.JUNIT_FILE}} - vars: - JUNIT_FILE: ./{{.DIST_DIR}}/junit-bandit.xml - - check:flake8: - desc: Enforce standard source code style guide + check:ruff: + desc: Complain about everything else <<: *preparation cmds: - - poetry run flake8 --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}} - - poetry run flake8 --show-source tests {{.PACKAGE}} + - poetry run ruff check --show-source tests {{.PACKAGE}} + - poetry run ruff check --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}} + - poetry run ruff format --check tests {{.PACKAGE}} vars: - JUNIT_FILE: ./{{.DIST_DIR}}/junit-flake8.xml - XML_PARAMS: --format junit-xml --output-file {{.JUNIT_FILE}} + JUNIT_FILE: ./{{.DIST_DIR}}/junit-ruff.xml + XML_PARAMS: --output-format junit --output-file {{.JUNIT_FILE}} # }}} # {{{ build and deploy tasks @@ -177,7 +164,6 @@ tasks: deploy: desc: Install plugin package in Corporate Memory deps: - - clean - build cmds: - cmemc admin workspace python install dist/*.tar.gz @@ -186,6 +172,9 @@ tasks: build: desc: Build a tarball and a wheel package <<: *preparation + deps: + - clean + - poetry:check cmds: - poetry build diff --git a/src/cmem_plugin_{{ project_slug }}/__init__.py b/src/cmem_plugin_{{ project_slug }}/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cmem_plugin_{{ project_slug }}/__init__.py.jinja b/src/cmem_plugin_{{ project_slug }}/__init__.py.jinja new file mode 100644 index 0000000..1da29f9 --- /dev/null +++ b/src/cmem_plugin_{{ project_slug }}/__init__.py.jinja @@ -0,0 +1 @@ +"""{{ project_slug }} - main package""" diff --git a/src/cmem_plugin_{{ project_slug }}/example_icon.svg b/src/cmem_plugin_{{ project_slug }}/example_icon.svg new file mode 100644 index 0000000..4710e0b --- /dev/null +++ b/src/cmem_plugin_{{ project_slug }}/example_icon.svg @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/cmem_plugin_{{ project_slug }}/transform/__init__.py.jinja b/src/cmem_plugin_{{ project_slug }}/example_transform.py similarity index 89% rename from src/cmem_plugin_{{ project_slug }}/transform/__init__.py.jinja rename to src/cmem_plugin_{{ project_slug }}/example_transform.py index 4d57854..e561948 100644 --- a/src/cmem_plugin_{{ project_slug }}/transform/__init__.py.jinja +++ b/src/cmem_plugin_{{ project_slug }}/example_transform.py @@ -1,7 +1,7 @@ """lifetime(age) transform plugin module""" import datetime +from collections.abc import Sequence from datetime import date -from typing import Sequence from cmem_plugin_base.dataintegration.description import ( Plugin, @@ -11,8 +11,7 @@ @Plugin( - label="Lifetime (age - {{ project_slug }})", - plugin_id="Example-Lifetime-{{ project_slug }}", + label="Lifetime (age - example from template)", description="From the input date," "the value gets transformed into number of years (age)." " Supports only xsd:date(YYYY-MM-DD) format.", @@ -48,6 +47,7 @@ def __init__(self, start_date: str): self.start_date = start_date def transform(self, inputs: Sequence[Sequence[str]]) -> Sequence[str]: + """Do the actual transformation of values""" result = [] if len(inputs) != 0: for collection in inputs: @@ -57,9 +57,9 @@ def transform(self, inputs: Sequence[Sequence[str]]) -> Sequence[str]: return result def _calculate_age(self, value: str) -> int: - """calculate age in years""" - today = date.today() - born = datetime.datetime.strptime(value, self.DATE_FORMAT).date() + """Calculate age in years""" + today = date.today() # noqa: DTZ011 + born = datetime.datetime.strptime(value, self.DATE_FORMAT).date() # noqa: DTZ007 try: birthday = born.replace(year=today.year) diff --git a/src/cmem_plugin_{{ project_slug }}/workflow/__init__.py.jinja b/src/cmem_plugin_{{ project_slug }}/example_workflow.py similarity index 86% rename from src/cmem_plugin_{{ project_slug }}/workflow/__init__.py.jinja rename to src/cmem_plugin_{{ project_slug }}/example_workflow.py index b63f901..31d4058 100644 --- a/src/cmem_plugin_{{ project_slug }}/workflow/__init__.py.jinja +++ b/src/cmem_plugin_{{ project_slug }}/example_workflow.py @@ -1,21 +1,21 @@ """Random values workflow plugin module""" import uuid +from collections.abc import Sequence from secrets import token_urlsafe -from typing import Sequence from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport -from cmem_plugin_base.dataintegration.description import Plugin, PluginParameter +from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter from cmem_plugin_base.dataintegration.entity import ( Entities, Entity, - EntitySchema, EntityPath, + EntitySchema, ) from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin @Plugin( - label="Random Values ({{ project_slug }})", + label="Random Values (example from template)", description="Generates random values of X rows a Y values.", documentation=""" This example workflow operator generates random values. @@ -25,6 +25,7 @@ - 'number_of_entities': How many rows do you need. - 'number_of_values': How many values per row do you need. """, + icon=Icon(package=__package__, file_name="example_icon.svg"), parameters=[ PluginParameter( name="number_of_entities", @@ -54,14 +55,17 @@ def __init__(self, number_of_entities: int = 10, number_of_values: int = 5) -> N self.number_of_values = number_of_values def execute( - self, inputs: Sequence[Entities], context: ExecutionContext + self, + inputs: Sequence[Entities], # noqa: ARG002 + context: ExecutionContext, ) -> Entities: + """Run the workflow operator.""" self.log.info("Start creating random values.") self.log.info(f"Config length: {len(self.config.get())}") value_counter = 0 entities = [] for _ in range(self.number_of_entities): - entity_uri = f"urn:uuid:{str(uuid.uuid4())}" + entity_uri = f"urn:uuid:{uuid.uuid4()!s}" values = [] for _ in range(self.number_of_values): values.append([token_urlsafe(16)]) diff --git a/src/pyproject.toml.jinja b/src/pyproject.toml.jinja index baf1ba6..b20b60b 100644 --- a/src/pyproject.toml.jinja +++ b/src/pyproject.toml.jinja @@ -20,30 +20,19 @@ keywords = [ python = "^3.11" [tool.poetry.dependencies.cmem-plugin-base] -version = "^4.1.0" -# allow-prereleases = true +version = "^4.3.0" +allow-prereleases = false [tool.poetry.group.dev.dependencies] -bandit = "^1.7.5" -black = "^23.3.0" -coverage = "^7.2.3" -defusedxml = "^0.7.1" -flake8-formatter-junit-xml = "^0.0.6" -# https://github.com/smarie/python-genbadge/issues/31 -pillow = "^9.5.0" -genbadge = "^1.1.0" +genbadge = {extras = ["coverage"], version = "^1.1.1"} mypy = "^1.2.0" -# Avoid safety issue 62044 for pip less than 23.3 -pip = ">=23.3" -# https://github.com/rasjani/pylint-junit/issues/1 -pylint = "^2" -pylint-junit = "^0.3.2" +pip = ">=23.3" # Avoid safety issue 62044 for pip less than 23.3 pytest = "^7.3.1" -pytest-cov = "^4.0.0" -pytest-memray = { version = "^1.4.0", markers = "platform_system != 'Windows'" } +pytest-cov = "^4.1.0" +pytest-dotenv = "^0.5.2" +pytest-memray = { version = "^1.5.0", markers = "platform_system != 'Windows'" } +ruff = "^0.1.5" safety = "^1.10.3" -typed-ast = "^1.5.4" -wheel = "^0.38.4" [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] @@ -53,23 +42,7 @@ build-backend = "poetry_dynamic_versioning.backend" enable = true vcs = "git" dirty = true - -[tool.pylint.MASTER] -load-plugins="pylint_junit" - -[tool.pylint.General] -ignore = "version.py" - -[tool.pylint.'MESSAGES CONTROL'] -extension-pkg-whitelist = "pydantic" -disable = "fixme" - -# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#pylint -[tool.pylint.messages_control] -disable = "R0903" - -[tool.pylint.format] -max-line-length = "88" +bump = true [tool.mypy] warn_return_any = true @@ -78,3 +51,30 @@ ignore_missing_imports = true [tool.pytest.ini_options] addopts = "" +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.format] +line-ending = "lf" # Use `\n` line endings for all files + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "ANN101", # Missing type annotation for self in method + "ANN204", # Missing return type annotation for special method `__init__` + "COM812", # missing-trailing-comma + "D107", # Missing docstring in __init__ + "D203", # [*] 1 blank line required before class docstring + "D211", # No blank lines allowed before class docstring + "D213", # Multi-line docstring summary should start at the second line + "D400", # First line should end with a period + "D415", # First line should end with a period, question mark, or exclamation point + "EM", # Exception texts - https://docs.astral.sh/ruff/rules/#flake8-errmsg-em + "G004", # Logging statement uses f-string + "ISC001", # single-line-implicit-string-concatenation + "PD", # opinionated linting for pandas code + "S101", # use of assert detected + "TRY003", # Avoid specifying long messages outside the exception class +] + diff --git a/src/tests/__init__.py b/src/tests/__init__.py index e69de29..30ea453 100644 --- a/src/tests/__init__.py +++ b/src/tests/__init__.py @@ -0,0 +1 @@ +"""tests""" diff --git a/src/tests/test_{{ project_slug }}.py.jinja b/src/tests/test_example.py.jinja similarity index 67% rename from src/tests/test_{{ project_slug }}.py.jinja rename to src/tests/test_example.py.jinja index aa22a8e..008cfbf 100644 --- a/src/tests/test_{{ project_slug }}.py.jinja +++ b/src/tests/test_example.py.jinja @@ -9,8 +9,8 @@ from cmem.cmempy.workspace.projects.resources.resource import ( get_resource_response, ) -from cmem_plugin_{{ project_slug }}.transform import Lifetime -from cmem_plugin_{{ project_slug }}.workflow import DollyPlugin +from cmem_plugin_{{ project_slug }}.example_transform import Lifetime +from cmem_plugin_{{ project_slug }}.example_workflow import DollyPlugin from tests.utils import TestExecutionContext, needs_cmem PROJECT_NAME = "{{ project_slug }}_test_project" @@ -19,9 +19,9 @@ RESOURCE_NAME = "sample_dataset.txt" DATASET_TYPE = "text" -@pytest.fixture -def setup(request): - """Provides the DI build project incl. assets.""" +@pytest.fixture() +def di_environment() -> object: + """Provide the DI build project incl. assets.""" make_new_project(PROJECT_NAME) make_new_dataset( project_name=PROJECT_NAME, @@ -37,12 +37,15 @@ def setup(request): file_resource=response_file, replace=True, ) - - request.addfinalizer(lambda: delete_project(PROJECT_NAME)) + yield { + "project": PROJECT_NAME, + "dataset": RESOURCE_NAME, + } + delete_project(PROJECT_NAME) @needs_cmem -def test_workflow_execution(): +def test_workflow_execution() -> None: """Test plugin execution""" entities = 100 values = 10 @@ -53,23 +56,21 @@ def test_workflow_execution(): assert len(item.values) == len(result.schema.paths) -def test_transform_execution_with_optional_input(): +def test_transform_execution_with_optional_input() -> None: """Test Lifetime with optional input""" result = Lifetime(start_date="2000-05-22").transform(inputs=[]) for item in result: assert item == "23" -def test_transform_execution_with_inputs(): +def test_transform_execution_with_inputs() -> None: """Test Lifetime with sequence of inputs.""" - result = Lifetime(start_date="").transform( - inputs=[["2000-05-22", "2021-12-12", "1904-02-29"]] - ) - assert result >= ["22", "0", "118"] + result = Lifetime(start_date="").transform(inputs=[["2000-05-22", "2021-12-12", "1904-02-29"]]) + assert list(result) >= ["22", "0", "118"] @needs_cmem -def test_integration_placeholder(setup): +def test_integration_placeholder(di_environment: dict) -> None: """Placeholder to write integration testcase with cmem""" - with get_resource_response(PROJECT_NAME, RESOURCE_NAME) as response: + with get_resource_response(di_environment["project"], di_environment["dataset"]) as response: assert response.text != "" diff --git a/src/tests/utils.py b/src/tests/utils.py index 1e5fe43..438fc63 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -1,16 +1,18 @@ """Testing utilities.""" import os +from typing import ClassVar import pytest + # check for cmem environment and skip if not present from cmem.cmempy.api import get_token from cmem.cmempy.config import get_oauth_default_credentials from cmem_plugin_base.dataintegration.context import ( - PluginContext, - UserContext, - TaskContext, ExecutionContext, + PluginContext, ReportContext, + TaskContext, + UserContext, ) needs_cmem = pytest.mark.skipif( @@ -22,7 +24,7 @@ class TestUserContext(UserContext): """dummy user context that can be used in tests""" __test__ = False - default_credential: dict = {} + default_credential: ClassVar[dict] = {} def __init__(self): # get access token from default service account