From dc4c3dd410bc72751791b33a9d90ba73d0e78293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Mart=C3=ADn=20Bl=C3=A1zquez?= Date: Wed, 17 Apr 2024 11:58:21 +0200 Subject: [PATCH] Fixes in docs `API` section (#546) * Remove redundant information about the CLI * Add missing `llm` key in example * Reference `Learn` sections in API docs --- docs/api/cli.md | 41 +------------------ docs/api/llms/index.md | 3 ++ docs/api/pipeline/pipeline.md | 2 + docs/api/steps/index.md | 5 +++ docs/api/steps/steps.md | 3 -- docs/api/steps/tasks/text_generation.md | 2 + docs/sections/learn/pipelines/index.md | 54 +++++++++++++++---------- mkdocs.yml | 3 +- 8 files changed, 48 insertions(+), 65 deletions(-) create mode 100644 docs/api/llms/index.md create mode 100644 docs/api/steps/index.md delete mode 100644 docs/api/steps/steps.md diff --git a/docs/api/cli.md b/docs/api/cli.md index 953fb876bb..3d0df6620d 100644 --- a/docs/api/cli.md +++ b/docs/api/cli.md @@ -1,45 +1,6 @@ # Command Line Interface -This section contains the API reference for the command line interface. - -## CLI commands - -This section shows the CLI commands: - -### distilabel pipeline run - -```bash -$ distilabel pipeline info --help - - Usage: distilabel pipeline info [OPTIONS] - - Get information about a Distilabel pipeline. - -╭─ Options ───────────────────────────────────────────────────────────────────────────╮ -│ * --config TEXT Path or URL to the Distilabel pipeline configuration file. │ -│ [default: None] │ -│ [required] │ -│ --help Show this message and exit. │ -╰─────────────────────────────────────────────────────────────────────────────────────╯ -``` - -### distilabel pipeline info - -```bash -$ distilabel pipeline --help - - Usage: distilabel pipeline [OPTIONS] COMMAND [ARGS]... - - Commands to run and inspect Distilabel pipelines. - -╭─ Options ───────────────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰─────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ──────────────────────────────────────────────────────────────────────────────╮ -│ info Get information about a Distilabel pipeline. │ -│ run Run a Distilabel pipeline. │ -╰─────────────────────────────────────────────────────────────────────────────────────────╯ -``` +This section contains the API reference for the command line interface. For more information on how to use the command line interface, see the [Tutorial - CLI](../sections/learn/cli.md). ## Utility functions for the pipeline commands diff --git a/docs/api/llms/index.md b/docs/api/llms/index.md new file mode 100644 index 0000000000..c38c1badd2 --- /dev/null +++ b/docs/api/llms/index.md @@ -0,0 +1,3 @@ +# LLMs + +This section contains the API reference for the distilabel LLMs. For an example on how to implement and use an LLM, see the [Tutorial - LLMs](../sections/learn/llms/index.md). diff --git a/docs/api/pipeline/pipeline.md b/docs/api/pipeline/pipeline.md index e982dba704..a31245273b 100644 --- a/docs/api/pipeline/pipeline.md +++ b/docs/api/pipeline/pipeline.md @@ -1,5 +1,7 @@ # Pipeline +This section contains the API reference for the distilabel pipelines. For an example on how to use the pipelines, see the [Tutorial - Pipelines](../../sections/learn/pipelines/index.md). + ## Base Pipeline ::: distilabel.pipeline.base diff --git a/docs/api/steps/index.md b/docs/api/steps/index.md new file mode 100644 index 0000000000..c530d1e96a --- /dev/null +++ b/docs/api/steps/index.md @@ -0,0 +1,5 @@ +# Steps + +This section contains the API reference for the distilabel steps. For an example on how to create and use a step, see the [Tutorial - Steps](../../sections/learn/steps/index.md). + +::: distilabel.steps.base diff --git a/docs/api/steps/steps.md b/docs/api/steps/steps.md deleted file mode 100644 index 744227bb5b..0000000000 --- a/docs/api/steps/steps.md +++ /dev/null @@ -1,3 +0,0 @@ -# Steps - -::: distilabel.steps.base diff --git a/docs/api/steps/tasks/text_generation.md b/docs/api/steps/tasks/text_generation.md index c89aa95e32..09089f6326 100644 --- a/docs/api/steps/tasks/text_generation.md +++ b/docs/api/steps/tasks/text_generation.md @@ -1,5 +1,7 @@ # Tasks +This section contains the API reference for the distilabel tasks. For an example on how to create and use a task, see the [Tutorial - Tasks](../../../sections/learn/tasks/index.md). + ::: distilabel.steps.tasks.base ## General Text Generation diff --git a/docs/sections/learn/pipelines/index.md b/docs/sections/learn/pipelines/index.md index c7cdfda043..57a6130e3a 100644 --- a/docs/sections/learn/pipelines/index.md +++ b/docs/sections/learn/pipelines/index.md @@ -111,25 +111,31 @@ if __name__ == "__main__": "repo_id": "distilabel-internal-testing/instruction-dataset-mini", "split": "test", }, - "text_generation_with_openai-gpt-4-0125-preview": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "text_generation_with_gpt-4-0125-preview": { + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, "text_generation_with_mistral-large-2402": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, - "text_generation_with_vertexai-gemini-1.5-pro": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "text_generation_with_gemini-1.0-pro": { + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, - } + }, ) ``` @@ -276,21 +282,27 @@ To sump up, here is the full code of the pipeline we have created in this sectio "split": "test", }, "text_generation_with_gpt-4-0125-preview": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, "text_generation_with_mistral-large-2402": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, "text_generation_with_gemini-1.0-pro": { - "generation_kwargs": { - "temperature": 0.7, - "max_new_tokens": 512, + "llm": { + "generation_kwargs": { + "temperature": 0.7, + "max_new_tokens": 512, + } } }, }, diff --git a/mkdocs.yml b/mkdocs.yml index bb3d5f9b2e..a9152d7281 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -136,7 +136,7 @@ nav: - Pipeline: - "api/pipeline/pipeline.md" - Steps: - - "api/steps/steps.md" + - "api/steps/index.md" - Generator Steps: "api/steps/generator_steps/generator_steps.md" - Global Steps: "api/steps/global_steps/global_steps.md" - Tasks: @@ -145,6 +145,7 @@ nav: - "api/steps/tasks/embeddings.md" - "api/steps/decorator.md" - LLMs: + - "api/llms/index.md" - "api/llms/anthropic.md" - "api/llms/anyscale.md" - "api/llms/huggingface.md"