From 8624c9a0056957ec1ec8ea242c0a0c9220c24627 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 27 Aug 2024 18:42:12 +0200 Subject: [PATCH 1/8] Move sources and destinations to the top level --- .../verified-sources/filesystem.md | 6 +- .../dlt-ecosystem/verified-sources/index.md | 46 ++++-- .../verified-sources/rest_api.md | 2 +- .../verified-sources/sql_database.md | 2 +- docs/website/sidebars.js | 138 ++++++++---------- 5 files changed, 99 insertions(+), 95 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md index 7552a0acb2..e6d19a4214 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem.md @@ -1,6 +1,6 @@ --- title: Filesystem -description: dlt verified source for Readers Source and Filesystem +description: AWS S3, Google Cloud Storage, Azure Blob Storage, local files keywords: [readers source and filesystem, filesystem, readers source] --- import Header from './_source-info-header.md'; @@ -123,7 +123,7 @@ For more information, read the 3. You can pass the bucket URL and glob pattern or use `config.toml`. For local filesystems, use `file://` as follows: - + ```toml [sources.filesystem] # use [sources.readers.credentials] for the "readers" source bucket_url='file://Users/admin/Documents/csv_files' @@ -136,7 +136,7 @@ For more information, read the bucket_url='~\Documents\csv_files\' file_glob="*" ``` - + In the example above we use Windows path to current user's Documents folder. Mind that literal toml string (single quotes) was used to conveniently use the backslashes without need to escape. diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index d105dccb9c..4a58c36cc2 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -1,31 +1,45 @@ --- -title: Verified sources -description: List of verified sources -keywords: ['verified source'] +title: Sources +description: Available sources +keywords: ['source'] --- -import DocCardList from '@theme/DocCardList'; import Link from '../../_book-onboarding-call.md'; +import DocCardList from '@theme/DocCardList'; +import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. - -Planning to use dlt in production and need a source that isn't listed? We're happy to help you build it: . +Planning to use dlt in production and need a source that isn't listed? We're happy to help you build it: . -### Popular sources +### Core sources -- [SQL databases](sql_database). Supports PostgreSQL, MySQL, MS SQL Server, BigQuery, Redshift, and more. -- [REST API generic source](rest_api). Loads data from REST APIs using declarative configuration. -- [OpenAPI source generator](openapi-generator). Generates a source from an OpenAPI 3.x spec using the REST API source. -- [Cloud and local storage](filesystem). Retrieves data from AWS S3, Google Cloud Storage, Azure Blob Storage, local files, and more. + item.label === '30+ SQL Databases' || item.label === 'REST API generic source' || item.label === 'Filesystem' +)} /> -### Full list of verified sources +### Verified sources - +Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip -If you're looking for a source that isn't listed and it provides a REST API, be sure to check out our [REST API generic source](rest_api) - source. +If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial) to learn how! ::: + item.label !== '30+ SQL Databases' && item.label !== 'REST API generic source'&& item.label !== 'Filesystem' +)} /> + +### What's the difference? + +The main difference between the [core sources](#core-sources) and [verified sources](#verified-sources) lies in their structure. +Core sources are generic collections, meaning they can connect to a variety of systems. For example, [SQL Database source](sql_database) can connect to any +database which supports SQLAlchemy. + +According to our telemetry, core sources are the most widely used among our users! + +It's also important to note that core sources are integrated into the `dlt` core library, +whereas verified sources are maintained in a separate [repository](https://github.com/dlt-hub/verified-sources). +To use a verified source, you need to run the dlt init command, which will download the verified source code to +your working directory. + ### Get help diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md index 7eea6d9aff..59f287a9a3 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md @@ -1,6 +1,6 @@ --- title: REST API generic source -description: dlt verified source for REST APIs +description: Loads data from REST APIs using declarative configuration keywords: [rest api, restful api] --- import Header from './_source-info-header.md'; diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md index 08fa8d2247..61a7f2e9e6 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database.md @@ -1,6 +1,6 @@ --- title: 30+ SQL Databases -description: dlt pipeline for SQL Database +description: PostgreSQL, MySQL, MS SQL Server, BigQuery, Redshift, and more keywords: [sql connector, sql database pipeline, sql database] --- import Header from './_source-info-header.md'; diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 56c2eb165c..76bb003e68 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -44,84 +44,74 @@ const sidebars = { }, { type: 'category', - label: 'Integrations', + label: 'Sources', link: { type: 'doc', - id: 'dlt-ecosystem/index', + id: 'dlt-ecosystem/verified-sources/index', }, items: [ - { - type: 'category', - label: 'Sources', - link: { - type: 'doc', - id: 'dlt-ecosystem/verified-sources/index', - }, - items: [ - 'dlt-ecosystem/verified-sources/airtable', - 'dlt-ecosystem/verified-sources/amazon_kinesis', - 'dlt-ecosystem/verified-sources/arrow-pandas', - 'dlt-ecosystem/verified-sources/asana', - 'dlt-ecosystem/verified-sources/chess', - 'dlt-ecosystem/verified-sources/facebook_ads', - 'dlt-ecosystem/verified-sources/filesystem', - 'dlt-ecosystem/verified-sources/freshdesk', - 'dlt-ecosystem/verified-sources/github', - 'dlt-ecosystem/verified-sources/google_ads', - 'dlt-ecosystem/verified-sources/google_analytics', - 'dlt-ecosystem/verified-sources/google_sheets', - 'dlt-ecosystem/verified-sources/hubspot', - 'dlt-ecosystem/verified-sources/inbox', - 'dlt-ecosystem/verified-sources/jira', - 'dlt-ecosystem/verified-sources/kafka', - 'dlt-ecosystem/verified-sources/matomo', - 'dlt-ecosystem/verified-sources/mongodb', - 'dlt-ecosystem/verified-sources/mux', - 'dlt-ecosystem/verified-sources/notion', - 'dlt-ecosystem/verified-sources/personio', - 'dlt-ecosystem/verified-sources/pg_replication', - 'dlt-ecosystem/verified-sources/pipedrive', - 'dlt-ecosystem/verified-sources/rest_api', - 'dlt-ecosystem/verified-sources/openapi-generator', - 'dlt-ecosystem/verified-sources/salesforce', - 'dlt-ecosystem/verified-sources/scrapy', - 'dlt-ecosystem/verified-sources/shopify', - 'dlt-ecosystem/verified-sources/sql_database', - 'dlt-ecosystem/verified-sources/slack', - 'dlt-ecosystem/verified-sources/strapi', - 'dlt-ecosystem/verified-sources/stripe', - 'dlt-ecosystem/verified-sources/workable', - 'dlt-ecosystem/verified-sources/zendesk' - ] - }, - { - type: 'category', - label: 'Destinations', - link: { - type: 'doc', - id: 'dlt-ecosystem/destinations/index', - }, - items: [ - 'dlt-ecosystem/destinations/bigquery', - 'dlt-ecosystem/destinations/databricks', - 'dlt-ecosystem/destinations/duckdb', - 'dlt-ecosystem/destinations/mssql', - 'dlt-ecosystem/destinations/synapse', - 'dlt-ecosystem/destinations/clickhouse', - 'dlt-ecosystem/destinations/filesystem', - 'dlt-ecosystem/destinations/postgres', - 'dlt-ecosystem/destinations/redshift', - 'dlt-ecosystem/destinations/snowflake', - 'dlt-ecosystem/destinations/athena', - 'dlt-ecosystem/destinations/weaviate', - 'dlt-ecosystem/destinations/lancedb', - 'dlt-ecosystem/destinations/qdrant', - 'dlt-ecosystem/destinations/dremio', - 'dlt-ecosystem/destinations/destination', - 'dlt-ecosystem/destinations/motherduck' - ] - }, - ], + 'dlt-ecosystem/verified-sources/airtable', + 'dlt-ecosystem/verified-sources/amazon_kinesis', + 'dlt-ecosystem/verified-sources/arrow-pandas', + 'dlt-ecosystem/verified-sources/asana', + 'dlt-ecosystem/verified-sources/chess', + 'dlt-ecosystem/verified-sources/facebook_ads', + 'dlt-ecosystem/verified-sources/filesystem', + 'dlt-ecosystem/verified-sources/freshdesk', + 'dlt-ecosystem/verified-sources/github', + 'dlt-ecosystem/verified-sources/google_ads', + 'dlt-ecosystem/verified-sources/google_analytics', + 'dlt-ecosystem/verified-sources/google_sheets', + 'dlt-ecosystem/verified-sources/hubspot', + 'dlt-ecosystem/verified-sources/inbox', + 'dlt-ecosystem/verified-sources/jira', + 'dlt-ecosystem/verified-sources/kafka', + 'dlt-ecosystem/verified-sources/matomo', + 'dlt-ecosystem/verified-sources/mongodb', + 'dlt-ecosystem/verified-sources/mux', + 'dlt-ecosystem/verified-sources/notion', + 'dlt-ecosystem/verified-sources/personio', + 'dlt-ecosystem/verified-sources/pg_replication', + 'dlt-ecosystem/verified-sources/pipedrive', + 'dlt-ecosystem/verified-sources/rest_api', + 'dlt-ecosystem/verified-sources/openapi-generator', + 'dlt-ecosystem/verified-sources/salesforce', + 'dlt-ecosystem/verified-sources/scrapy', + 'dlt-ecosystem/verified-sources/shopify', + 'dlt-ecosystem/verified-sources/sql_database', + 'dlt-ecosystem/verified-sources/slack', + 'dlt-ecosystem/verified-sources/strapi', + 'dlt-ecosystem/verified-sources/stripe', + 'dlt-ecosystem/verified-sources/workable', + 'dlt-ecosystem/verified-sources/zendesk' + ] + }, + { + type: 'category', + label: 'Destinations', + link: { + type: 'doc', + id: 'dlt-ecosystem/destinations/index', + }, + items: [ + 'dlt-ecosystem/destinations/bigquery', + 'dlt-ecosystem/destinations/databricks', + 'dlt-ecosystem/destinations/duckdb', + 'dlt-ecosystem/destinations/mssql', + 'dlt-ecosystem/destinations/synapse', + 'dlt-ecosystem/destinations/clickhouse', + 'dlt-ecosystem/destinations/filesystem', + 'dlt-ecosystem/destinations/postgres', + 'dlt-ecosystem/destinations/redshift', + 'dlt-ecosystem/destinations/snowflake', + 'dlt-ecosystem/destinations/athena', + 'dlt-ecosystem/destinations/weaviate', + 'dlt-ecosystem/destinations/lancedb', + 'dlt-ecosystem/destinations/qdrant', + 'dlt-ecosystem/destinations/dremio', + 'dlt-ecosystem/destinations/destination', + 'dlt-ecosystem/destinations/motherduck' + ] }, { type: 'category', From 49582386782d46740968cf0f8e3a28dc4fc2e874 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Wed, 28 Aug 2024 15:42:51 +0200 Subject: [PATCH 2/8] Update the css --- docs/website/docs/dlt-ecosystem/index.md | 14 +-- .../dlt-ecosystem/verified-sources/index.md | 4 +- docs/website/sidebars.js | 9 ++ docs/website/src/css/custom.css | 86 ++++++++++++++----- docs/website/static/img/Sources-Active-1.svg | 1 + docs/website/static/img/Sources-Active.svg | 1 + .../website/static/img/Sources-Inactive-1.svg | 1 + docs/website/static/img/Sources-Inactive.svg | 1 + 8 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 docs/website/static/img/Sources-Active-1.svg create mode 100644 docs/website/static/img/Sources-Active.svg create mode 100644 docs/website/static/img/Sources-Inactive-1.svg create mode 100644 docs/website/static/img/Sources-Inactive.svg diff --git a/docs/website/docs/dlt-ecosystem/index.md b/docs/website/docs/dlt-ecosystem/index.md index 740a3a3a39..9fc3fb4ed9 100644 --- a/docs/website/docs/dlt-ecosystem/index.md +++ b/docs/website/docs/dlt-ecosystem/index.md @@ -3,16 +3,4 @@ title: Integrations description: List of integrations keywords: ['integrations, sources, destinations'] --- -import DocCardList from '@theme/DocCardList'; -import Link from '../_book-onboarding-call.md'; - -Speed up the process of creating data pipelines by using dlt's multiple pre-built sources and destinations: - -- Each [dlt verified source](verified-sources) allows you to create [pipelines](../general-usage/pipeline) that extract data from a particular source: a database, a cloud service, or an API. -- [Destinations](destinations) are where you want to load your data. dlt supports a variety of destinations, including databases, data warehouses, and data lakes. - - - -:::tip -Most source-destination pairs work seamlessly together. If the merge [write disposition](../general-usage/incremental-loading#choosing-a-write-disposition) is not supported by a destination (for example, [file sytem destination](destinations/filesystem)), dlt will automatically fall back to the [append](../general-usage/incremental-loading#append) write disposition. -::: \ No newline at end of file +Could not remove this page, please ignore it for now \ No newline at end of file diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index 4a58c36cc2..d2bce768b2 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -20,7 +20,7 @@ item => item.label === '30+ SQL Databases' || item.label === 'REST API generic s Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip -If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial) to learn how! +If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial/grouping-resources) to learn how! ::: item.label !== '30+ SQL Databases' && item.label !== 'REST API generic s ### What's the difference? The main difference between the [core sources](#core-sources) and [verified sources](#verified-sources) lies in their structure. -Core sources are generic collections, meaning they can connect to a variety of systems. For example, [SQL Database source](sql_database) can connect to any +Core sources are generic collections, meaning they can connect to a variety of systems. For example, the [SQL Database source](sql_database) can connect to any database which supports SQLAlchemy. According to our telemetry, core sources are the most widely used among our users! diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 76bb003e68..201fb8e1da 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -42,6 +42,15 @@ const sidebars = { 'tutorial/grouping-resources', ] }, + { + type: 'category', + label: 'Integration', + link: { + type: 'doc', + id: 'dlt-ecosystem/index', + }, + items: ['dlt-ecosystem/index'], + }, { type: 'category', label: 'Sources', diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index fca5b933ba..e443a6cda2 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -601,97 +601,137 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[ background-image: url(../../static/img/UsingLoadedData-Active-1.svg); } -/* Using dlt */ +/* Sources */ .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active.svg); +.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Sources-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); + background-image: url(../../static/img/Sources-Inactive.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/GeneralUsage-Active-1.svg); + background-image: url(../../static/img/Sources-Active-1.svg); } -/* How-to Guides */ +/* Destinations */ .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive.svg); + background-image: url(../../static/img/Destinations-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active.svg); + background-image: url(../../static/img/Destinations-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { - background-image: url(../../static/img/Walkthrough-Inactive-1.svg); + background-image: url(../../static/img/Destinations-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Walkthrough-Active-1.svg); + background-image: url(../../static/img/Destinations-Active-1.svg); } -/* Code Examples */ +/* Using dlt */ .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive.svg); + background-image: url(../../static/img/GeneralUsage-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active.svg); + background-image: url(../../static/img/GeneralUsage-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { - background-image: url(../../static/img/Howdltworks-Inactive-1.svg); + background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Howdltworks-Active-1.svg); + background-image: url(../../static/img/GeneralUsage-Active-1.svg); } -/* Reference */ +/* How-to Guides */ .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { - background-image: url(../../static/img/Reference-Inactive.svg); + background-image: url(../../static/img/Walkthrough-Inactive.svg); } .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/Reference-Active.svg); + background-image: url(../../static/img/Walkthrough-Active.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { - background-image: url(../../static/img/Reference-Inactive-1.svg); + background-image: url(../../static/img/Walkthrough-Inactive-1.svg); } html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Walkthrough-Active-1.svg); +} + +/* Code Examples */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { + background-image: url(../../static/img/Howdltworks-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Howdltworks-Active-1.svg); +} + +/* Reference */ + +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { + background-image: url(../../static/img/Reference-Inactive.svg); +} + +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { + background-image: url(../../static/img/Reference-Active.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { + background-image: url(../../static/img/Reference-Inactive-1.svg); +} + +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active-1.svg); } /* Devel / Stable switch */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>a svg { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a svg { display: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>a { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a { flex: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(9) { +.theme-doc-sidebar-menu.menu__list>li:nth-child(11) { margin-top:30px; padding-top: 10px; justify-content: center; diff --git a/docs/website/static/img/Sources-Active-1.svg b/docs/website/static/img/Sources-Active-1.svg new file mode 100644 index 0000000000..c409ba1c0f --- /dev/null +++ b/docs/website/static/img/Sources-Active-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Active.svg b/docs/website/static/img/Sources-Active.svg new file mode 100644 index 0000000000..5e13f5d797 --- /dev/null +++ b/docs/website/static/img/Sources-Active.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Inactive-1.svg b/docs/website/static/img/Sources-Inactive-1.svg new file mode 100644 index 0000000000..dff6d8f514 --- /dev/null +++ b/docs/website/static/img/Sources-Inactive-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/website/static/img/Sources-Inactive.svg b/docs/website/static/img/Sources-Inactive.svg new file mode 100644 index 0000000000..5506061f25 --- /dev/null +++ b/docs/website/static/img/Sources-Inactive.svg @@ -0,0 +1 @@ + \ No newline at end of file From c87502207071cd7444e2f35d6b1030f8e07d8f25 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 10 Sep 2024 16:38:50 +0200 Subject: [PATCH 3/8] Remove the integrations page --- docs/website/docs/dlt-ecosystem/index.md | 6 ------ docs/website/sidebars.js | 9 --------- 2 files changed, 15 deletions(-) delete mode 100644 docs/website/docs/dlt-ecosystem/index.md diff --git a/docs/website/docs/dlt-ecosystem/index.md b/docs/website/docs/dlt-ecosystem/index.md deleted file mode 100644 index 9fc3fb4ed9..0000000000 --- a/docs/website/docs/dlt-ecosystem/index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Integrations -description: List of integrations -keywords: ['integrations, sources, destinations'] ---- -Could not remove this page, please ignore it for now \ No newline at end of file diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 201fb8e1da..76bb003e68 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -42,15 +42,6 @@ const sidebars = { 'tutorial/grouping-resources', ] }, - { - type: 'category', - label: 'Integration', - link: { - type: 'doc', - id: 'dlt-ecosystem/index', - }, - items: ['dlt-ecosystem/index'], - }, { type: 'category', label: 'Sources', From 25e03d11f617c3e0a39bf634465610c7b81cf8c8 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 10 Sep 2024 17:18:36 +0200 Subject: [PATCH 4/8] Update the icons --- docs/website/src/css/custom.css | 20 ------------------- docs/website/static/img/Sources-Active-1.svg | 14 ++++++++++++- docs/website/static/img/Sources-Active.svg | 14 ++++++++++++- .../website/static/img/Sources-Inactive-1.svg | 6 +++++- docs/website/static/img/Sources-Inactive.svg | 6 +++++- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index e443a6cda2..a3e9b9e0dd 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -581,26 +581,6 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[ background-image: url(../../static/img/Pipelines-Active-1.svg); } -/* Integrations */ - -.theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/UsingLoadedData-Inactive.svg); -} - -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/UsingLoadedData-Active.svg); -} - -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { - background-image: url(../../static/img/UsingLoadedData-Inactive-1.svg); -} - -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { - background-image: url(../../static/img/UsingLoadedData-Active-1.svg); -} - /* Sources */ .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { diff --git a/docs/website/static/img/Sources-Active-1.svg b/docs/website/static/img/Sources-Active-1.svg index c409ba1c0f..d72f580a10 100644 --- a/docs/website/static/img/Sources-Active-1.svg +++ b/docs/website/static/img/Sources-Active-1.svg @@ -1 +1,13 @@ - \ No newline at end of file + + + + + + + + + + + + + diff --git a/docs/website/static/img/Sources-Active.svg b/docs/website/static/img/Sources-Active.svg index 5e13f5d797..6d19bdaab3 100644 --- a/docs/website/static/img/Sources-Active.svg +++ b/docs/website/static/img/Sources-Active.svg @@ -1 +1,13 @@ - \ No newline at end of file + + + + + + + + + + + + + diff --git a/docs/website/static/img/Sources-Inactive-1.svg b/docs/website/static/img/Sources-Inactive-1.svg index dff6d8f514..6974888d61 100644 --- a/docs/website/static/img/Sources-Inactive-1.svg +++ b/docs/website/static/img/Sources-Inactive-1.svg @@ -1 +1,5 @@ - \ No newline at end of file + + + + + diff --git a/docs/website/static/img/Sources-Inactive.svg b/docs/website/static/img/Sources-Inactive.svg index 5506061f25..29751e59ed 100644 --- a/docs/website/static/img/Sources-Inactive.svg +++ b/docs/website/static/img/Sources-Inactive.svg @@ -1 +1,5 @@ - \ No newline at end of file + + + + + From 0c5df29989c18c75e9f30afce664f283cc2c203f Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 10 Sep 2024 17:29:02 +0200 Subject: [PATCH 5/8] Minor --- .../docs/dlt-ecosystem/verified-sources/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index d2bce768b2..b0f824746f 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -7,7 +7,7 @@ import Link from '../../_book-onboarding-call.md'; import DocCardList from '@theme/DocCardList'; import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -Planning to use dlt in production and need a source that isn't listed? We're happy to help you build it: . +Planning to use `dlt` in production and need a source that isn't listed? We're happy to help you build it: . ### Core sources @@ -17,7 +17,7 @@ item => item.label === '30+ SQL Databases' || item.label === 'REST API generic s ### Verified sources -Choose from our collection of verified sources, developed and maintained by the dlt team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. +Choose from our collection of verified sources, developed and maintained by the `dlt` team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial/grouping-resources) to learn how! @@ -27,7 +27,7 @@ If you couldn't find a source implementation, you can easily create your own, ch item => item.label !== '30+ SQL Databases' && item.label !== 'REST API generic source'&& item.label !== 'Filesystem' )} /> -### What's the difference? +### What's the difference between core and verified sources? The main difference between the [core sources](#core-sources) and [verified sources](#verified-sources) lies in their structure. Core sources are generic collections, meaning they can connect to a variety of systems. For example, the [SQL Database source](sql_database) can connect to any @@ -37,7 +37,7 @@ According to our telemetry, core sources are the most widely used among our user It's also important to note that core sources are integrated into the `dlt` core library, whereas verified sources are maintained in a separate [repository](https://github.com/dlt-hub/verified-sources). -To use a verified source, you need to run the dlt init command, which will download the verified source code to +To use a verified source, you need to run the `dlt` init command, which will download the verified source code to your working directory. @@ -45,4 +45,4 @@ your working directory. * Source missing? [Request a new verified source.](https://github.com/dlt-hub/verified-sources/issues/new?template=source-request.md) * Missing endpoint or a feature? [Request or contribute](https://github.com/dlt-hub/verified-sources/issues/new?template=extend-a-source.md) -* [Join our Slack community](https://dlthub.com/community) and ask in the technical-help channel. +* [Join our Slack community](https://dlthub.com/community) and ask in the technical-help channel. \ No newline at end of file From 2a636a1aa39bfd50d591c51ba57d6b9b8ae49b63 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 10 Sep 2024 18:13:42 +0200 Subject: [PATCH 6/8] Fix icons --- docs/website/src/css/custom.css | 80 ++++++++++++++++----------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index a3e9b9e0dd..e4d7793372 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -583,135 +583,135 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(3)>div>[ /* Sources */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { background-image: url(../../static/img/Sources-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Sources-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { - background-image: url(../../static/img/Sources-Inactive.svg); +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>a::before { + background-image: url(../../static/img/Sources-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(4)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(4)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Sources-Active-1.svg); } /* Destinations */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { background-image: url(../../static/img/Destinations-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Destinations-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>a::before { background-image: url(../../static/img/Destinations-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(5)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(5)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Destinations-Active-1.svg); } /* Using dlt */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { background-image: url(../../static/img/GeneralUsage-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/GeneralUsage-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>a::before { background-image: url(../../static/img/GeneralUsage-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(6)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(6)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/GeneralUsage-Active-1.svg); } /* How-to Guides */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { background-image: url(../../static/img/Walkthrough-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Walkthrough-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>a::before { background-image: url(../../static/img/Walkthrough-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(7)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(7)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Walkthrough-Active-1.svg); } /* Code Examples */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { background-image: url(../../static/img/Howdltworks-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Howdltworks-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>a::before { background-image: url(../../static/img/Howdltworks-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(8)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(8)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Howdltworks-Active-1.svg); } /* Reference */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive.svg); } -.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, -.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { +.theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +.theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>a::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>a::before { background-image: url(../../static/img/Reference-Inactive-1.svg); } -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(10)>div>a::before, -html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>div>[aria-expanded="true"]::before { +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:hover:nth-child(9)>div>a::before, +html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[aria-expanded="true"]::before { background-image: url(../../static/img/Reference-Active-1.svg); } /* Devel / Stable switch */ -.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a svg { +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>a svg { display: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(11)>a { +.theme-doc-sidebar-menu.menu__list>li:nth-child(10)>a { flex: none; } -.theme-doc-sidebar-menu.menu__list>li:nth-child(11) { +.theme-doc-sidebar-menu.menu__list>li:nth-child(10) { margin-top:30px; padding-top: 10px; justify-content: center; From e3a950731f05d09657a8c49eb074c4624b7057c0 Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Tue, 10 Sep 2024 18:35:31 +0200 Subject: [PATCH 7/8] Fix icons --- docs/website/static/img/Sources-Active-1.svg | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/website/static/img/Sources-Active-1.svg b/docs/website/static/img/Sources-Active-1.svg index d72f580a10..fa65eda6f2 100644 --- a/docs/website/static/img/Sources-Active-1.svg +++ b/docs/website/static/img/Sources-Active-1.svg @@ -1,9 +1,6 @@ - - - - + From b5faf1c5354a6ec624e44e0aa9f75bfa1858e8ce Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Wed, 11 Sep 2024 10:27:23 +0200 Subject: [PATCH 8/8] Change link to the custom source --- docs/website/docs/dlt-ecosystem/verified-sources/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/index.md b/docs/website/docs/dlt-ecosystem/verified-sources/index.md index b0f824746f..cc54c8dc67 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/index.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/index.md @@ -20,7 +20,7 @@ item => item.label === '30+ SQL Databases' || item.label === 'REST API generic s Choose from our collection of verified sources, developed and maintained by the `dlt` team and community. Each source is rigorously tested against a real API and provided as Python code for easy customization. :::tip -If you couldn't find a source implementation, you can easily create your own, check our [tutorial](../../tutorial/grouping-resources) to learn how! +If you couldn't find a source implementation, you can easily create your own, check out the [resource page](../../general-usage/resource) to learn how! :::