diff --git a/docs/website/docs/reference/performance.md b/docs/website/docs/reference/performance.md index 2c6c819374..58951702a5 100644 --- a/docs/website/docs/reference/performance.md +++ b/docs/website/docs/reference/performance.md @@ -1,10 +1,10 @@ --- -title: Performance +title: Optimizing dlt description: Scale-up, parallelize and finetune dlt pipelines keywords: [scaling, parallelism, finetuning] --- -# Performance +# Optimizing dlt ## Yield pages instead of rows diff --git a/docs/website/docs/tutorial/load-data-from-an-api.md b/docs/website/docs/tutorial/load-data-from-an-api.md index 1718ef642b..1e40531691 100644 --- a/docs/website/docs/tutorial/load-data-from-an-api.md +++ b/docs/website/docs/tutorial/load-data-from-an-api.md @@ -574,9 +574,8 @@ Interested in learning more? Here are some suggestions: - [Run in production: inspecting, tracing, retry policies and cleaning up](../running-in-production/running). - [Run resources in parallel, optimize buffers and local storage](../reference/performance.md) - [Use REST API client helpers](../general-usage/http/rest-client.md) to simplify working with REST APIs. -3. Check out our [how-to guides](../walkthroughs) to get answers to some common questions. -4. Explore [destinations](../dlt-ecosystem/destinations/) and [sources](../dlt-ecosystem/verified-sources/) provided by us and community. -5. Explore the [Examples](../examples) section to see how dlt can be used in real-world scenarios +3. Explore [destinations](../dlt-ecosystem/destinations/) and [sources](../dlt-ecosystem/verified-sources/) provided by us and community. +4. Explore the [Examples](../examples) section to see how dlt can be used in real-world scenarios diff --git a/docs/website/docs/walkthroughs/share-a-dataset.md b/docs/website/docs/walkthroughs/share-a-dataset.md index 75ba856468..3e231b8274 100644 --- a/docs/website/docs/walkthroughs/share-a-dataset.md +++ b/docs/website/docs/walkthroughs/share-a-dataset.md @@ -1,10 +1,10 @@ --- -title: 'Share a dataset: DuckDB -> BigQuery' +title: 'Moving from local to production' description: Share a local dataset by moving it to BigQuery keywords: [how to, share a dataset] --- -# Share a dataset: DuckDB -> BigQuery +# Moving from local to production In previous how-to guides you used the local stack to create and run your pipeline. This saved you the headache of setting up cloud account, credentials and often also money. Our choice for local diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 299bb2a642..61e75f2ce5 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -43,6 +43,19 @@ const sidebars = { 'tutorial/load-data-from-an-api', ] }, + { + type: 'category', + label: 'Core concepts', + items: [ + 'reference/explainers/how-dlt-works', + 'general-usage/source', + 'general-usage/resource', + 'general-usage/pipeline', + 'general-usage/destination', + 'general-usage/state', + 'general-usage/glossary' + ] + }, { type: 'category', label: 'Sources', @@ -58,18 +71,18 @@ const sidebars = { 'dlt-ecosystem/verified-sources/chess', 'dlt-ecosystem/verified-sources/facebook_ads', { - type: 'category', - label: 'Filesystem & buckets', - description: 'AWS S3, GCP, Azure, local files', - link: { - type: 'doc', - id: 'dlt-ecosystem/verified-sources/filesystem/index', - }, - items: [ - 'dlt-ecosystem/verified-sources/filesystem/basic', - 'dlt-ecosystem/verified-sources/filesystem/advanced', - ] - }, + type: 'category', + label: 'Filesystem & cloud storage', + description: 'AWS S3, Google Cloud Storage, Azure Blob Storage, local file system', + link: { + type: 'doc', + id: 'dlt-ecosystem/verified-sources/filesystem/index', + }, + items: [ + 'dlt-ecosystem/verified-sources/filesystem/basic', + 'dlt-ecosystem/verified-sources/filesystem/advanced', + ] + }, 'dlt-ecosystem/verified-sources/freshdesk', 'dlt-ecosystem/verified-sources/github', 'dlt-ecosystem/verified-sources/google_ads', @@ -123,7 +136,20 @@ const sidebars = { 'dlt-ecosystem/verified-sources/strapi', 'dlt-ecosystem/verified-sources/stripe', 'dlt-ecosystem/verified-sources/workable', - 'dlt-ecosystem/verified-sources/zendesk' + 'dlt-ecosystem/verified-sources/zendesk', + { + type: 'category', + label: 'REST API helpers', + link: { + type: 'doc', + id: 'general-usage/http/overview', + }, + items: [ + 'general-usage/http/rest-client', + 'general-usage/http/requests', + ] + }, + 'walkthroughs/add-a-verified-source', ] }, { @@ -150,22 +176,49 @@ const sidebars = { 'dlt-ecosystem/destinations/qdrant', 'dlt-ecosystem/destinations/dremio', 'dlt-ecosystem/destinations/destination', - 'dlt-ecosystem/destinations/motherduck' + 'dlt-ecosystem/destinations/motherduck', + 'walkthroughs/create-new-destination' ] }, { type: 'category', - label: 'Core concepts', + label: 'Using dlt', link: { type: 'generated-index', - title: 'Core concepts', + title: 'Using dlt', slug: 'general-usage', keywords: ['concepts', 'usage'], }, items: [ - 'reference/explainers/how-dlt-works', - 'general-usage/resource', - 'general-usage/source', + 'walkthroughs/create-a-pipeline', + 'walkthroughs/run-a-pipeline', + 'dlt-ecosystem/visualizations/exploring-the-data', + { + type: 'category', + label: 'Transform the data', + link: { + type: 'generated-index', + title: 'Transform the data', + description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', + slug: 'dlt-ecosystem/transformations', + keywords: ['transformations'], + }, + items: [ + { + type: 'category', + label: 'Transforming data with dbt', + items: [ + 'dlt-ecosystem/transformations/dbt/dbt', + 'dlt-ecosystem/transformations/dbt/dbt_cloud', + ] + }, + 'dlt-ecosystem/transformations/sql', + 'dlt-ecosystem/transformations/pandas', + 'general-usage/customising-pipelines/renaming_columns', + 'general-usage/customising-pipelines/pseudonymizing_columns', + 'general-usage/customising-pipelines/removing_columns' + ] + }, { type: 'category', label: 'Configuration and secrets', @@ -177,94 +230,39 @@ const sidebars = { 'general-usage/credentials/setup', 'general-usage/credentials/advanced', 'general-usage/credentials/complex_types', + // Unsure item + 'walkthroughs/add_credentials' ] }, - 'general-usage/pipeline', - 'general-usage/destination', - 'general-usage/destination-tables', { type: 'category', - label: 'REST API helpers', + label: 'Schema', link: { type: 'doc', - id: 'general-usage/http/overview', + id: 'general-usage/schema', }, items: [ - 'general-usage/http/rest-client', - 'general-usage/http/requests', + 'general-usage/schema-contracts', + 'general-usage/schema-evolution', + 'walkthroughs/adjust-a-schema', ] }, - 'dlt-ecosystem/staging', - 'general-usage/state', - 'general-usage/incremental-loading', - 'general-usage/full-loading', - 'general-usage/schema', - 'general-usage/naming-convention', - 'general-usage/schema-contracts', - 'general-usage/schema-evolution', - 'build-a-pipeline-tutorial', - 'reference/performance', { type: 'category', - label: 'File formats', - link: { - type: 'generated-index', - title: 'File formats', - description: 'Overview of our loader file formats', - slug: 'dlt-ecosystem/file-formats', - keywords: ['destination'], - }, + label: 'Loading Behavior', items: [ - 'dlt-ecosystem/file-formats/jsonl', - 'dlt-ecosystem/file-formats/parquet', - 'dlt-ecosystem/file-formats/csv', - 'dlt-ecosystem/file-formats/insert-format', + 'general-usage/incremental-loading', + 'walkthroughs/add-incremental-configuration', + 'general-usage/full-loading', ] - }, - ], + } + ] }, { type: 'category', - label: 'How-to guides', - link: { - type: 'generated-index', - title: 'How-to guides', - description: 'In this section you will find step-by-step instructions for the common tasks.', - slug: 'walkthroughs', - keywords: ['how-to'], - }, + label: 'Deploying dlt', items: [ - 'walkthroughs/create-a-pipeline', - 'walkthroughs/add-a-verified-source', - 'walkthroughs/add-incremental-configuration', - 'walkthroughs/add_credentials', - 'walkthroughs/run-a-pipeline', - 'walkthroughs/adjust-a-schema', 'walkthroughs/share-a-dataset', - 'dlt-ecosystem/visualizations/exploring-the-data', - { - type: 'category', - label: 'Transform the data', - link: { - type: 'generated-index', - title: 'Transform the data', - description: 'If you want to transform the data after loading, you can use one of the following methods: dbt, SQL, Pandas.', - slug: 'dlt-ecosystem/transformations', - keywords: ['transformations'], - }, - items: [ - { - type: 'category', - label: 'Transforming data with dbt', - items: [ - 'dlt-ecosystem/transformations/dbt/dbt', - 'dlt-ecosystem/transformations/dbt/dbt_cloud', - ] - }, - 'dlt-ecosystem/transformations/sql', - 'dlt-ecosystem/transformations/pandas', - ] - }, { type: 'category', label: 'Deploy a pipeline', @@ -285,24 +283,6 @@ const sidebars = { 'walkthroughs/deploy-a-pipeline/deploy-with-prefect', ] }, - { - type: 'category', - label: 'Customise pipelines', - items: [ - 'general-usage/customising-pipelines/renaming_columns', - 'general-usage/customising-pipelines/pseudonymizing_columns', - 'general-usage/customising-pipelines/removing_columns', - ] - }, - { - type: 'category', - label: 'Data enrichments', - items: [ - 'general-usage/data-enrichments/user_agent_device_data_enrichment', - 'general-usage/data-enrichments/currency_conversion_data_enrichment', - 'general-usage/data-enrichments/url-parser-data-enrichment' - ] - }, { type: 'category', label: 'Run in production', @@ -320,11 +300,9 @@ const sidebars = { 'running-in-production/tracing', ], }, - 'walkthroughs/dispatch-to-multiple-tables', - 'walkthroughs/create-new-destination', - 'walkthroughs/zendesk-weaviate', - ], + ] }, + 'reference/performance', { type: 'category', label: 'Code examples', @@ -336,6 +314,7 @@ const sidebars = { keywords: ['examples'], }, items: [ + 'walkthroughs/dispatch-to-multiple-tables', ], }, { @@ -344,26 +323,64 @@ const sidebars = { link: { type: 'generated-index', title: 'Reference', - description: 'The dlthub reference. Learn more about the dlt, CLI, and the telemetry.', + description: 'Learn more about the dlt, CLI, and the telemetry.', slug: 'reference', keywords: ['reference'], }, items: [ 'reference/command-line-interface', 'reference/telemetry', + // Unsure item + 'general-usage/destination-tables', + 'general-usage/naming-convention', + 'dlt-ecosystem/staging', + { + type: 'category', + label: 'File formats', + link: { + type: 'generated-index', + title: 'File formats', + description: 'Overview of our loader file formats', + slug: 'dlt-ecosystem/file-formats', + keywords: ['destination'], + }, + items: [ + 'dlt-ecosystem/file-formats/jsonl', + 'dlt-ecosystem/file-formats/parquet', + 'dlt-ecosystem/file-formats/csv', + 'dlt-ecosystem/file-formats/insert-format', + ] + }, 'reference/frequently-asked-questions', - 'general-usage/glossary', ], }, - // { - // "API Documentation": [ - // require("./docs/api_reference/sidebar.json") - // ], - // } + /* + { + type: 'category', + label: 'How-to guides', + link: { + type: 'generated-index', + title: 'How-to guides', + description: 'In this section you will find step-by-step instructions for the common tasks.', + slug: 'walkthroughs', + keywords: ['how-to'], + }, + items: [ + { + type: 'category', + label: 'Data enrichments', + items: [ + 'general-usage/data-enrichments/user_agent_device_data_enrichment', + 'general-usage/data-enrichments/currency_conversion_data_enrichment', + 'general-usage/data-enrichments/url-parser-data-enrichment' + ] + } + ] + } + */ ] }; - // insert examples for (const item of sidebars.tutorialSidebar) { if (item.label === 'Code examples') { @@ -386,3 +403,19 @@ if (fs.existsSync('./docs_processed/api_reference/sidebar.json')) { } module.exports = sidebars; + + +/* +blog: +'build-a-pipeline-tutorial', +'walkthroughs/zendesk-weaviate', +{ + type: 'category', + label: 'Data enrichments', + items: [ + 'general-usage/data-enrichments/user_agent_device_data_enrichment', + 'general-usage/data-enrichments/currency_conversion_data_enrichment', + 'general-usage/data-enrichments/url-parser-data-enrichment' + ] +}, +*/ \ No newline at end of file diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index 4d016a9a7f..baff5ba024 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -707,123 +707,163 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(1)>div>[ background-image: url(../../static/img/GettingStarted-Active-1.svg); } -/* Sources */ +/* Core concepts */ .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>a::before { - background-image: url(../../static/img/Sources-Inactive.svg); + background-image: url(../../static/img/UserGuide-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(2)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Sources-Active.svg); + background-image: url(../../static/img/UserGuide-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>a::before { - background-image: url(../../static/img/Sources-Inactive-1.svg); + background-image: url(../../static/img/UserGuide-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(2)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(2)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Sources-Active-1.svg); + background-image: url(../../static/img/UserGuide-Active-1.svg); } -/* Destinations */ +/* Sources */ .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>a::before { - background-image: url(../../static/img/Destinations-Inactive.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(3)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Destinations-Active.svg); + background-image: url(../../static/img/Sources-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>a::before { - background-image: url(../../static/img/Destinations-Inactive-1.svg); + background-image: url(../../static/img/Sources-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(3)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Destinations-Active-1.svg); + background-image: url(../../static/img/Sources-Active-1.svg); } -/* Using dlt */ +/* Destinations */ .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive.svg); + background-image: url(../../static/img/Destinations-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active.svg); + background-image: url(../../static/img/Destinations-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); + background-image: url(../../static/img/Destinations-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active-1.svg); + background-image: url(../../static/img/Destinations-Active-1.svg); } -/* How-to Guides */ +/* Using dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive.svg); + background-image: url(../../static/img/GeneralUsage-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active.svg); + background-image: url(../../static/img/GeneralUsage-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive-1.svg); + background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active-1.svg); + background-image: url(../../static/img/GeneralUsage-Active-1.svg); } -/* Code Examples */ +/* Deploying dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive.svg); + background-image: url(../../static/img/UsingLoadedData-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active.svg); + background-image: url(../../static/img/UsingLoadedData-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive-1.svg); + background-image: url(../../static/img/UsingLoadedData-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/UsingLoadedData-Active-1.svg); +} + +/* Optimizing dlt */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a::before { + background-image: url(../../static/img/Installation-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a.menu__link--active::before { + background-image: url(../../static/img/Installation-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a::before { + background-image: url(../../static/img/Installation-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>a.menu__link--active::before { + background-image: url(../../static/img/Installation-Active-1.svg); +} + +/* Code Examples */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Howdltworks-Active-1.svg); } /* Reference */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active-1.svg); }