diff --git a/.github/workflows/test-kedro-new.yml b/.github/workflows/test-kedro-new.yml index b2fff72..a072554 100644 --- a/.github/workflows/test-kedro-new.yml +++ b/.github/workflows/test-kedro-new.yml @@ -111,10 +111,12 @@ jobs: python -m pip install --upgrade pip python -m pip install --upgrade poetry poetry config virtualenvs.create false - poetry install + poetry install --with test - name: Run kedro working-directory: test-project + env: + COLUMNS: 200 run: kedro run --env=test - name: Run tests (pytest) diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/conf/base/catalog.yml b/mini-poetry/{{ cookiecutter.repo_name }}/conf/base/catalog.yml index b709f23..31d0681 100644 --- a/mini-poetry/{{ cookiecutter.repo_name }}/conf/base/catalog.yml +++ b/mini-poetry/{{ cookiecutter.repo_name }}/conf/base/catalog.yml @@ -41,3 +41,12 @@ # # This is a data set used by the "Hello World" example pipeline provided with the project # template. Please feel free to remove it once you remove the example pipeline. + +test_data: + type: pandas.CSVDataset + filepath: data/01_raw/test-data.csv + load_args: + sep: "," + index_col: 0 +# parse_dates: +# - date diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/conf/test/.gitkeep b/mini-poetry/{{ cookiecutter.repo_name }}/conf/test/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/data/01_raw/test-data.csv b/mini-poetry/{{ cookiecutter.repo_name }}/data/01_raw/test-data.csv new file mode 100644 index 0000000..9889a0f --- /dev/null +++ b/mini-poetry/{{ cookiecutter.repo_name }}/data/01_raw/test-data.csv @@ -0,0 +1,2 @@ +Column1,Column2 +1,2 diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/pyproject.toml b/mini-poetry/{{ cookiecutter.repo_name }}/pyproject.toml index 2f2faf1..10251ee 100644 --- a/mini-poetry/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/mini-poetry/{{ cookiecutter.repo_name }}/pyproject.toml @@ -19,7 +19,7 @@ readme = "README.md" [tool.poetry.dependencies] python = ">=3.8.1" kedro = "^{{ cookiecutter.kedro_version }}" -kedro-datasets = "^1.0.0" +kedro-datasets = {version="^1.0.0", extras=["pandas.CSVDataset"]} [tool.poetry.group.dev] optional = true diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/__init__.py b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/__init__.py new file mode 100644 index 0000000..b1e511a --- /dev/null +++ b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/__init__.py @@ -0,0 +1,5 @@ +from .pipeline import create_pipeline + +__all__ = ["create_pipeline"] + +__version__ = "0.1" diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/nodes.py b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/nodes.py new file mode 100644 index 0000000..d69b3ba --- /dev/null +++ b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/nodes.py @@ -0,0 +1,3 @@ +import logging + +LOGGER = logging.getLogger(__name__) diff --git a/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py new file mode 100644 index 0000000..2db15bc --- /dev/null +++ b/mini-poetry/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_processing/pipeline.py @@ -0,0 +1,18 @@ +from kedro.pipeline import Pipeline, pipeline, node + +def create_pipeline(**kwargs) -> Pipeline: + return pipeline( + [ + node( + func=lambda x: x, + inputs="test_data", + outputs="processed_data", + name="data_processing", + ) + ], + inputs=None, + outputs=None, + parameters=None, + namespace=None, + tags=None, + )