From 641350211f01cedd49ef5444252222cb69811c65 Mon Sep 17 00:00:00 2001 From: Dmitrii Deriabin <44967953+DmitriiDeriabinQB@users.noreply.github.com> Date: Thu, 25 Jun 2020 17:59:26 +0100 Subject: [PATCH] [KED-1813] Add the docs for `kedro pipeline package` (#681) --- docs/source/04_user_guide/06_pipelines.md | 9 +++++++++ docs/source/06_resources/03_commands_reference.md | 9 ++++++++- kedro/framework/cli/pipeline.py | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/source/04_user_guide/06_pipelines.md b/docs/source/04_user_guide/06_pipelines.md index 5498209e00..f469ae31b5 100644 --- a/docs/source/04_user_guide/06_pipelines.md +++ b/docs/source/04_user_guide/06_pipelines.md @@ -233,6 +233,15 @@ def create_pipelines(**kwargs) -> Dict[str, Pipeline]: > *Note:* To find out how you can run a pipeline by its name, please navigate to [this section](#running-a-pipeline-by-name). +### How do I package a modular pipeline? + +Since Kedro 0.16.2 you can package a modular pipeline by executing `kedro pipeline package ` command, which will generate a new [wheel file](https://pythonwheels.com/) for it. By default, the wheel file will be saved into `src/dist` directory inside your project, however this can be changed using `--destination` (`-d`) option. + +When packaging your modular pipeline, Kedro will also automatically include all configuration parameters from `conf//pipelines/` and pipeline tests from `tests/pipelines/`, where `` defaults to `base`. If you need to capture the parameters from a different config environment, run `kedro pipeline package --env `. + +> Note that Kedro _will not_ package the catalog config files even if those are present in `conf//pipelines/`. + +If you plan to publish your packaged modular pipeline to some Python package repository like [PyPI](https://pypi.org/), you need to make sure that your modular pipeline name doesn't clash with any of the existing packages in that repository. However, there is no need to rename any of your source files if that is the case. Simply alias your package with a new name by running `kedro pipeline package --alias `. ### A modular pipeline example template diff --git a/docs/source/06_resources/03_commands_reference.md b/docs/source/06_resources/03_commands_reference.md index e919ccc777..73ba96b8df 100644 --- a/docs/source/06_resources/03_commands_reference.md +++ b/docs/source/06_resources/03_commands_reference.md @@ -101,7 +101,14 @@ This command adds a `git hook` which clears all notebook output cells before com This command shows datasets per pipeline per type. The result includes datasets that are/aren't used by a specific pipeline. It also accept optional `--pipeline` argument that allows specifying pipeline name(s) (comma-separated value) for which the datasets should be shown, e.g. `kedro catalog list --pipeline "ds,de"`. ### `kedro pipeline list` -This command shows a list of nodes for every pipeline in your project. `kedro pipeline list ...` will show a list of nodes for each pipeline you specify. `kedro pipeline list --simple` will show a list of all pipelines in your project. +This command shows a list of all pipelines in your project. + +### `kedro pipeline create` + +This command creates a new modular pipeline in your project. More details in [this section](../04_user_guide/06_pipelines.md#how-do-i-create-modular-pipelines). + +### `kedro pipeline package ` +This command packages a modular pipeline into a [wheel file](https://pythonwheels.com/). More details in [this section](../04_user_guide/06_pipelines.md#how-do-i-package-a-modular-pipeline). ## Using Python You can also invoke the Kedro CLI as a Python module: diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py index 0b77c18686..5e78608def 100644 --- a/kedro/framework/cli/pipeline.py +++ b/kedro/framework/cli/pipeline.py @@ -284,13 +284,13 @@ def _install_files( type=str, default="", callback=_check_pipeline_name, - help="Alternative name to package under", + help="Alternative name to package under.", ) @click.option( "-d", "--destination", type=click.Path(resolve_path=True, file_okay=False), - help="Location where to create the wheel file. Defaults to `src/dist`", + help="Location where to create the wheel file. Defaults to `src/dist`.", ) @click.argument("name", nargs=1) def package_pipeline(name, env, alias, destination):