diff --git a/clickhouseapi.js b/clickhouseapi.js index e1b1039ac7b..3e02e564568 100644 --- a/clickhouseapi.js +++ b/clickhouseapi.js @@ -52,7 +52,7 @@ function generateDocusaurusMarkdown(spec, groupedEndpoints, prefix) { markdownContent += `| Method | Path |\n` markdownContent += `| :----- | :--- |\n` - markdownContent += `| ${method.toUpperCase()} | ${path} |\n\n` + markdownContent += `| ${method.toUpperCase()} | \`${path}\` |\n\n` markdownContent += `### Request\n\n`; diff --git a/copyClickhouseRepoDocs.sh b/copyClickhouseRepoDocs.sh index 643bd7a4942..a931105e227 100755 --- a/copyClickhouseRepoDocs.sh +++ b/copyClickhouseRepoDocs.sh @@ -1,7 +1,8 @@ -#! ./bin/bash +#! /bin/bash SCRIPT_NAME=$(basename "$0") +rm -rf ClickHouse echo "[$SCRIPT_NAME] Start tasks for copying docs from ClickHouse repo" # Clone ClickHouse repo diff --git a/docs/en/cloud/reference/changelog.md b/docs/en/cloud/reference/changelog.md index 5e124eeb285..95f174b7eb6 100644 --- a/docs/en/cloud/reference/changelog.md +++ b/docs/en/cloud/reference/changelog.md @@ -957,7 +957,7 @@ Adds support for a subset of features in ClickHouse 23.1, for example: - New functions, including `age()`, `quantileInterpolatedWeighted()`, `quantilesInterpolatedWeighted()` - Ability to use structure from insertion table in `generateRandom` without arguments - Improved database creation and rename logic that allows the reuse of previous names -- See the 23.1 release [webinar slides](https://presentations.clickhouse.com/release_23.1/#cover) and [23.1 release changelog](/docs/en/whats-new/changelog/index.md/#clickhouse-release-231) for more details +- See the 23.1 release [webinar slides](https://presentations.clickhouse.com/release_23.1/#cover) and [23.1 release changelog](/docs/en/whats-new/changelog/index.md#clickhouse-release-231) for more details ### Integrations changes - [Kafka-Connect](/docs/en/integrations/data-ingestion/kafka/index.md): Added support for Amazon MSK diff --git a/docs/en/data-compression/compression-modes.md b/docs/en/data-compression/compression-modes.md index 4c67654f7be..db4c21d94ad 100644 --- a/docs/en/data-compression/compression-modes.md +++ b/docs/en/data-compression/compression-modes.md @@ -41,7 +41,7 @@ From [facebook benchmarks](https://facebook.github.io/zstd/#benchmarks): | mode | byte | Compression mode | | compressed_data | binary | Block of compressed data | -![compression block diagram](../native-protocol/images/ch_compression_block.drawio.svg) +![compression block diagram](./images/ch_compression_block.png) Header is (raw_size + data_size + mode), raw size consists of len(header + compressed_data). diff --git a/docs/en/data-compression/images/ch_compression_block.png b/docs/en/data-compression/images/ch_compression_block.png new file mode 100644 index 00000000000..af57c4b00aa Binary files /dev/null and b/docs/en/data-compression/images/ch_compression_block.png differ diff --git a/docs/en/data-modeling/backfilling.md b/docs/en/data-modeling/backfilling.md index 1f99136ce35..2663923ba07 100644 --- a/docs/en/data-modeling/backfilling.md +++ b/docs/en/data-modeling/backfilling.md @@ -448,7 +448,7 @@ GROUP BY Here, we create a Null table, `pypi_v2,` to receive the rows that will be used to build our materialized view. Note how we limit the schema to only the columns we need. Our materialized view performs an aggregation over rows inserted into this table (one block at a time), sending the results to our target table, `pypi_downloads_per_day`. -::note +:::note We have used `pypi_downloads_per_day` as our target table here. For additional resiliency, users could create a duplicate table, `pypi_downloads_per_day_v2`, and use this as the target table of the view, as shown in previous examples. On completion of the insert, partitions in `pypi_downloads_per_day_v2` could, in turn, be moved to `pypi_downloads_per_day.` This would allow recovery in the case our insert fails due to memory issues or server interruptions i.e. we just truncate `pypi_downloads_per_day_v2`, tune settings, and retry. ::: diff --git a/docs/en/guides/developer/understanding-query-execution-with-the-analyzer.md b/docs/en/guides/developer/understanding-query-execution-with-the-analyzer.md index b644d0249f8..f7015a06590 100644 --- a/docs/en/guides/developer/understanding-query-execution-with-the-analyzer.md +++ b/docs/en/guides/developer/understanding-query-execution-with-the-analyzer.md @@ -63,12 +63,10 @@ Each node has corresponding children and the overall tree represents the overall ## Analyzer - - -ClickHouse currently has two architectures for the Analyzer. You can use the old architecture by setting: `allow_experimental_analyzer=0`. If you want to use the new architecture, you should set `allow_experimental_analyzer=1`. We are going to describe only the new architecture here, given the old one is going to be deprecated once the new analyzer is generally available. +ClickHouse currently has two architectures for the Analyzer. You can use the old architecture by setting: `enable_analyzer=0`. The new architecture is enabled by default. We are going to describe only the new architecture here, given the old one is going to be deprecated once the new analyzer is generally available. :::note -The new analyzer is in Beta. The new architecture should provide us with a better framework to improve ClickHouse's performance. However, given it is a fundamental component of the query processing steps, it also might have a negative impact on some queries. After moving to the new analyzer, you may see performance degradation, queries failing, or queries giving you an unexpected result. You can revert back to the old analyzer by changing the `allow_experimental_analyzer` setting at the query or user level. Please report any issues in GitHub. +The new architecture should provide us with a better framework to improve ClickHouse's performance. However, given it is a fundamental component of the query processing steps, it also might have a negative impact on some queries and there are [known incompatibilities](/docs/en/operations/analyzer#known-incompatibilities). You can revert back to the old analyzer by changing the `enable_analyzer` setting at the query or user level. ::: The analyzer is an important step of the query execution. It takes an AST and transforms it into a query tree. The main benefit of a query tree over an AST is that a lot of the components will be resolved, like the storage for instance. We also know from which table to read, aliases are also resolved, and the tree knows the different data types used. With all these benefits, the analyzer can apply optimizations. The way these optimizations work is via “passes”. Every pass is going to look for different optimizations. You can see all the passes [here](https://github.com/ClickHouse/ClickHouse/blob/76578ebf92af3be917cd2e0e17fea2965716d958/src/Analyzer/QueryTreePassManager.cpp#L249), let’s see it in practice with our previous query: diff --git a/docs/en/guides/inserting-data.md b/docs/en/guides/inserting-data.md index 757d0c9ba1b..1c5ae43c41b 100644 --- a/docs/en/guides/inserting-data.md +++ b/docs/en/guides/inserting-data.md @@ -89,7 +89,7 @@ With asynchronous inserts, data is inserted into a buffer first and then written
NEEDS ALT diff --git a/docs/en/guides/sre/user-management/index.md b/docs/en/guides/sre/user-management/index.md index ced25ccba24..800bf72f650 100644 --- a/docs/en/guides/sre/user-management/index.md +++ b/docs/en/guides/sre/user-management/index.md @@ -200,10 +200,6 @@ This article shows the basics of defining SQL users and roles and applying those GRANT ALL ON *.* TO clickhouse_admin WITH GRANT OPTION; ``` - - - - ## ALTER permissions This article is intended to provide you with a better understanding of how to define permissions, and how permissions work when using `ALTER` statements for privileged users. diff --git a/docs/en/integrations/data-ingestion/clickpipes/index.md b/docs/en/integrations/data-ingestion/clickpipes/index.md index d275ed0d339..5fe84619fa7 100644 --- a/docs/en/integrations/data-ingestion/clickpipes/index.md +++ b/docs/en/integrations/data-ingestion/clickpipes/index.md @@ -18,7 +18,7 @@ import PostgresSVG from "../../images/logos/postgresql.svg"; ## Introduction -[ClickPipes](https://clickhouse.com/cloud/clickpipes) is a managed integration platform that makes ingesting data from a diverse set of sources as simple as clicking a few buttons. Designed for the most demanding workloads, ClickPipes's robust and scalable architecture ensures consistent performance and reliability. ClickPipes can be used for long-term streaming needs or one-time data loading job. +[ClickPipes](/docs/en/integrations/clickpipes) is a managed integration platform that makes ingesting data from a diverse set of sources as simple as clicking a few buttons. Designed for the most demanding workloads, ClickPipes's robust and scalable architecture ensures consistent performance and reliability. ClickPipes can be used for long-term streaming needs or one-time data loading job. ![ClickPipes stack](./images/clickpipes_stack.png) @@ -64,7 +64,7 @@ Steps: ![Assign a custom role](./images/cp_custom_role.png) ## Error reporting -ClickPipes will create a table next to your destination table with the postfix `_clickpipes_error`. This table will contain any errors from the operations of your ClickPipe (network, connectivity, etc.) and also any data that don't conform to the schema. The error table has a [TTL](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl) of 7 days. +ClickPipes will create a table next to your destination table with the postfix `_clickpipes_error`. This table will contain any errors from the operations of your ClickPipe (network, connectivity, etc.) and also any data that don't conform to the schema. The error table has a [TTL](/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl) of 7 days. If ClickPipes cannot connect to a data source or destination after 15min., ClickPipes instance stops and stores an appropriate message in the error table (providing the ClickHouse instance is available). ## F.A.Q @@ -74,7 +74,7 @@ If ClickPipes cannot connect to a data source or destination after 15min., Click - **Does ClickPipes support data transformation?** - Yes, ClickPipes supports basic data transformation by exposing the DDL creation. You can then apply more advanced transformations to the data as it is loaded into its destination table in a ClickHouse Cloud service leveraging ClickHouse's [materialized views feature](https://clickhouse.com/docs/en/guides/developer/cascading-materialized-views). + Yes, ClickPipes supports basic data transformation by exposing the DDL creation. You can then apply more advanced transformations to the data as it is loaded into its destination table in a ClickHouse Cloud service leveraging ClickHouse's [materialized views feature](/docs/en/guides/developer/cascading-materialized-views). - **Does using ClickPipes incur an additional cost?** diff --git a/docs/en/integrations/data-ingestion/clickpipes/postgres/index.md b/docs/en/integrations/data-ingestion/clickpipes/postgres/index.md index c24702676c0..7ed0ca66a10 100644 --- a/docs/en/integrations/data-ingestion/clickpipes/postgres/index.md +++ b/docs/en/integrations/data-ingestion/clickpipes/postgres/index.md @@ -4,12 +4,14 @@ description: Seamlessly connect your Postgres to ClickHouse Cloud. slug: /en/integrations/clickpipes/postgres --- +import PrivatePreviewBadge from '@theme/badges/PrivatePreviewBadge'; + # Ingesting Data from Postgres to ClickHouse (using CDC) -:::info + +:::info Currently, ingesting data from Postgres to ClickHouse Cloud via ClickPipes is in Private Preview. If you are interested in trying it out, please sign up [here](https://clickpipes.peerdb.io/). - ::: diff --git a/docs/en/integrations/data-ingestion/data-formats/csv-tsv.md b/docs/en/integrations/data-ingestion/data-formats/csv-tsv.md index dcc58faa43e..56cdf7ad690 100644 --- a/docs/en/integrations/data-ingestion/data-formats/csv-tsv.md +++ b/docs/en/integrations/data-ingestion/data-formats/csv-tsv.md @@ -40,7 +40,7 @@ FROM INFILE 'data_small.csv' FORMAT CSV ``` -Here, we use the `FORMAT CSV` clause so ClickHouse understands the file format. We can also load data directly from URLs using [url()](/docs/en/sql-reference/table-functions/url.md/) function or from S3 files using [s3()](/docs/en/sql-reference/table-functions/s3.md/) function. +Here, we use the `FORMAT CSV` clause so ClickHouse understands the file format. We can also load data directly from URLs using [url()](/docs/en/sql-reference/table-functions/url.md) function or from S3 files using [s3()](/docs/en/sql-reference/table-functions/s3.md) function. :::tip We can skip explicit format setting for `file()` and `INFILE`/`OUTFILE`. diff --git a/docs/en/integrations/data-ingestion/data-formats/json/formats.md b/docs/en/integrations/data-ingestion/data-formats/json/formats.md index e4d53dd8767..9561fae982d 100644 --- a/docs/en/integrations/data-ingestion/data-formats/json/formats.md +++ b/docs/en/integrations/data-ingestion/data-formats/json/formats.md @@ -273,7 +273,7 @@ Note that `JSONAsString` works perfectly fine in cases we have JSON object-per-l ## Schema for nested objects -In cases when we're dealing with [nested JSON objects](../assets/list-nested.json), we can additionally define schema and use complex types ([`Array`](/docs/en/sql-reference/data-types/array.md/), [`Object Data Type`](/en/sql-reference/data-types/object-data-type) or [`Tuple`](/docs/en/sql-reference/data-types/tuple.md/)) to load data: +In cases when we're dealing with [nested JSON objects](../assets/list-nested.json), we can additionally define schema and use complex types ([`Array`](/docs/en/sql-reference/data-types/array.md), [`Object Data Type`](/en/sql-reference/data-types/object-data-type) or [`Tuple`](/docs/en/sql-reference/data-types/tuple.md)) to load data: ```sql SELECT * diff --git a/docs/en/integrations/data-ingestion/data-formats/json/schema.md b/docs/en/integrations/data-ingestion/data-formats/json/schema.md index 7196e7a023c..a7f26ee47f8 100644 --- a/docs/en/integrations/data-ingestion/data-formats/json/schema.md +++ b/docs/en/integrations/data-ingestion/data-formats/json/schema.md @@ -7,7 +7,7 @@ keywords: [json, clickhouse, inserting, loading, formats, schema] # Designing your schema -While [schema inference](/docs/en/integrations/data-formats/JSON/inference) can be used to establish an initial schema for JSON data and query JSON data files in place, e.g., in S3, users should aim to establish an optimized versioned schema for their data. We discuss the options for modeling JSON structures below. +While [schema inference](/docs/en/integrations/data-formats/json/inference) can be used to establish an initial schema for JSON data and query JSON data files in place, e.g., in S3, users should aim to establish an optimized versioned schema for their data. We discuss the options for modeling JSON structures below. ## Extract where possible diff --git a/docs/en/integrations/data-ingestion/data-formats/templates-regex.md b/docs/en/integrations/data-ingestion/data-formats/templates-regex.md index 615495fd27c..d3077a3484a 100644 --- a/docs/en/integrations/data-ingestion/data-formats/templates-regex.md +++ b/docs/en/integrations/data-ingestion/data-formats/templates-regex.md @@ -126,7 +126,7 @@ FORMAT Template SETTINGS format_template_resultset = 'output.results', ``` ### Exporting to HTML files -Template-based results can also be exported to files using an [`INTO OUTFILE`](/docs/en/sql-reference/statements/select/into-outfile.md/) clause. Let's generate HTML files based on given [resultset](assets/html.results) and [row](assets/html.row) formats: +Template-based results can also be exported to files using an [`INTO OUTFILE`](/docs/en/sql-reference/statements/select/into-outfile.md) clause. Let's generate HTML files based on given [resultset](assets/html.results) and [row](assets/html.row) formats: ```sql SELECT diff --git a/docs/en/integrations/data-ingestion/dbms/postgresql/data-type-mappings.md b/docs/en/integrations/data-ingestion/dbms/postgresql/data-type-mappings.md index 18fde8a5df2..3ef932d7270 100644 --- a/docs/en/integrations/data-ingestion/dbms/postgresql/data-type-mappings.md +++ b/docs/en/integrations/data-ingestion/dbms/postgresql/data-type-mappings.md @@ -39,4 +39,4 @@ The following table shows the equivalent ClickHouse data types for Postgres. | JSON* | [String](/en/sql-reference/data-types/string), [Variant](/en/sql-reference/data-types/variant), [Nested](/en/sql-reference/data-types/nested-data-structures/nested#nestedname1-type1-name2-type2-), [Tuple](/en/sql-reference/data-types/tuple) | | JSONB | [String](/en/sql-reference/data-types/string) | -*\* Production support for JSON in ClickHouse is under development. Currently users can either map JSON as String, and use [JSON functions](/en/sql-reference/functions/json-functions), or map the JSON directly to [Tuples](/en/sql-reference/data-types/tuple) and [Nested](/en/sql-reference/data-types/nested-data-structures/nested) if the structure is predictable. Read more about JSON [here](/en/integrations/data-formats/json#handle-as-structured-data).* +*\* Production support for JSON in ClickHouse is under development. Currently users can either map JSON as String, and use [JSON functions](/en/sql-reference/functions/json-functions), or map the JSON directly to [Tuples](/en/sql-reference/data-types/tuple) and [Nested](/en/sql-reference/data-types/nested-data-structures/nested) if the structure is predictable. Read more about JSON [here](/en/integrations/data-formats/json/overview).* diff --git a/docs/en/integrations/images/logos/datalens.svg b/docs/en/integrations/images/logos/datalens.svg new file mode 100644 index 00000000000..c605085c37f --- /dev/null +++ b/docs/en/integrations/images/logos/datalens.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docs/en/integrations/index.mdx b/docs/en/integrations/index.mdx index ffcfc9059f8..7c27707d46e 100644 --- a/docs/en/integrations/index.mdx +++ b/docs/en/integrations/index.mdx @@ -75,7 +75,7 @@ import SodaPNG from "./images/logos/soda_logo.png"; import SqliteSVG from "./images/logos/sqlite.svg"; import SupersetSVG from "./images/logos/superset.svg"; import TablumSVG from "./images/logos/tablum.svg"; -import Teleport from "./images/logos/teleport.png"; +import Teleport from "./images/logos/teleport_logo.png"; import TricksterCacheSVG from "./images/logos/trickster-logo.svg"; import UpstashSVG from "./images/logos/upstash.svg"; import YepcodeSVG from "./images/logos/yepcode.svg"; @@ -97,55 +97,55 @@ We are actively compiling this list of ClickHouse integrations below, so it's no Core integrations -
+
|Name|Logo|Category|Description|Resources| |------|----|----------------|------------------|-------------| |Amazon Kinesis||Data ingestion|Integration with Amazon Kinesis.|[Documentation](/docs/en/integrations/clickpipes/kinesis/)| |Amazon MSK||Data ingestion|Integration with Amazon Managed Streaming for Apache Kafka (MSK).|[Documentation](/docs/en/integrations/kafka/cloud/amazon-msk/)| |Amazon S3||Data ingestion|Import from, export to, and transform S3 data in flight with ClickHouse built-in S3 functions.|[Documentation](/docs/en/integrations/data-ingestion/s3/index.md)| -|Amazon Glue|Amazon Glue logo|Data ingestion|Query ClickHouse over JDBC|[Documentation](/docs/en/integrations/glue)| +|Amazon Glue|Amazon Glue logo|Data ingestion|Query ClickHouse over JDBC|[Documentation](/docs/en/integrations/glue)| |Apache Spark||Data ingestion|Spark ClickHouse Connector is a high performance connector built on top of Spark DataSource V2.|[GitHub](https://github.com/housepower/spark-clickhouse-connector),
[Documentation](/docs/en/integrations/data-ingestion/apache-spark/index.md)| |Azure Event Hubs||Data ingestion|A data streaming platform that supports Apache Kafka's native protocol|[Website](https://azure.microsoft.com/en-gb/products/event-hubs)| -|C++|Cpp logo|Language client|C++ client for ClickHouse|[GitHub](https://github.com/ClickHouse/clickhouse-cpp)| -|Cassandra|Cassandra logo|Data ingestion|Allows ClickHouse to use [Cassandra](https://cassandra.apache.org/) as a dictionary source.|[Documentation](/docs/en/sql-reference/dictionaries/index.md#cassandra)| +|C++|Cpp logo|Language client|C++ client for ClickHouse|[GitHub](https://github.com/ClickHouse/clickhouse-cpp)| +|Cassandra|Cassandra logo|Data ingestion|Allows ClickHouse to use [Cassandra](https://cassandra.apache.org/) as a dictionary source.|[Documentation](/docs/en/sql-reference/dictionaries/index.md#cassandra)| |CHDB||AI/ML|An embedded OLAP SQL Engine|[GitHub](https://github.com/chdb-io/chdb#/),
[Documentation](https://doc.chdb.io/)| |ClickHouse Client||SQL client|ClickHouse Client is the native command-line client for ClickHouse.|[Documentation](/docs/en/interfaces/cli.md)| |Confluent||Data ingestion|Integration with Apache Kafka on Confluent platform.|[Documentation](/docs/en/integrations/kafka/cloud/confluent/custom-connector)| |dbt||Data integration|Use dbt (data build tool) to transform data in ClickHouse by simply writing select statements. dbt puts the T in ELT.|[Documentation](/docs/en/integrations/data-ingestion/etl-tools/dbt/index.md)| -|DeltaLake|Delta Lake logo|Data ingestion| provides a read-only integration with existing [Delta Lake](https://github.com/delta-io/delta) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/deltalake)| +|DeltaLake|Delta Lake logo|Data ingestion| provides a read-only integration with existing [Delta Lake](https://github.com/delta-io/delta) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/deltalake)| |EmbeddedRocksDB||Data integration|Allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).|[Documentation](/docs/en/engines/table-engines/integrations/embedded-rocksdb)| |Fivetran||Data ingestion|[ClickHouse Cloud](https://clickhouse.com/cloud) destination for the [Fivetran data movement platform](https://www.fivetran.com/).|[Documentation](/docs/en/integrations/data-ingestion/etl-tools/fivetran/index.md)| |Google Cloud Storage||Data ingestion|Import from, export to, and transform GCS data in flight with ClickHouse built-in `S3` functions.|[Documentation](/docs/en/integrations/data-ingestion/s3/index.md)| |Golang||Language client|The Go client uses the native interface for a performant, low-overhead means of connecting to ClickHouse.|[Documentation](/docs/en/integrations/language-clients/go/index.md)| |HDFS||Data ingestion|Provides integration with the [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html) via ClickHouse.|[Documentation](/docs/en/engines/table-engines/integrations/hdfs)| |Hive||Data ingestionn|The Hive engine allows you to perform `SELECT` quries on HDFS Hive table.|[Documentation](/docs/en/engines/table-engines/integrations/hive)| -|Hudi|Apache Hudi logo|Data ingestion| provides a read-only integration with existing Apache [Hudi](https://hudi.apache.org/) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/hudi)| -|Iceberg|Apache Iceberg logo|Data ingestion|Provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/iceberg)| +|Hudi|Apache Hudi logo|Data ingestion| provides a read-only integration with existing Apache [Hudi](https://hudi.apache.org/) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/hudi)| +|Iceberg|Apache Iceberg logo|Data ingestion|Provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3.|[Documentation](/docs/en/engines/table-engines/integrations/iceberg)| |Java, JDBC||Language client|The Java client and JDBC driver.|[Documentation](/docs/en/integrations/language-clients/java/index.md)| |Kafka||Data ingestion|Integration with Apache Kafka, the open-source distributed event streaming platform.|[Documentation](/docs/en/integrations/kafka)| |Looker Studio||Data visualization|Looker Studio is a free tool that turns your data into informative, easy to read, easy to share, and fully customizable dashboards and reports.|[Documentation](/docs/en/integrations/lookerstudio)| |Looker||Data visualization|Looker is an enterprise platform for BI, data applications, and embedded analytics that helps you explore and share insights in real time.|[Documentation](/docs/en/integrations/looker)| -|Metabase|Metabase logo|Data visualization|Metabase is an easy-to-use, open source UI tool for asking questions about your data.|[Documentation](/docs/en/integrations/metabase)| -|MinIO|Metabase logo|Data ingestion|MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with the Amazon S3 cloud storage service|[Documentation](/docs/en/integrations/minio)| +|Metabase|Metabase logo|Data visualization|Metabase is an easy-to-use, open source UI tool for asking questions about your data.|[Documentation](/docs/en/integrations/metabase)| +|MinIO|Metabase logo|Data ingestion|MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with the Amazon S3 cloud storage service|[Documentation](/docs/en/integrations/minio)| |MongoDB||Data ingestion|MongoDB engine is read-only table engine which allows to read data (`SELECT` queries) from remote MongoDB collection.|[Documentation](/docs/en/engines/table-engines/integrations/mongodb)| |MySQL||Data ingestion|The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.|[Documentation](/docs/en/engines/table-engines/integrations/mysql)| |NATS||Data ingestion|Allows integrating ClickHouse with [NATS](https://nats.io/).|[Documentation](/docs/en/engines/table-engines/integrations/nats)| |Node.JS||Language client|The official JS client for connecting to ClickHouse.|[Documentation](/docs/en/integrations/language-clients/js.md)| -|ODBC|ODBC logo|Data integration|Allows ClickHouse to connect to external databases via [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity) table engine.|[Documentation](/docs/en/engines/table-engines/integrations/odbc)| -|OpenTelemetry|Otel logo|Data ingestion|Exporter that supports sending logs, metrics, trace OpenTelemetry data to ClickHouse|[GitHub](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter)| +|ODBC|ODBC logo|Data integration|Allows ClickHouse to connect to external databases via [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity) table engine.|[Documentation](/docs/en/engines/table-engines/integrations/odbc)| +|OpenTelemetry|Otel logo|Data ingestion|Exporter that supports sending logs, metrics, trace OpenTelemetry data to ClickHouse|[GitHub](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter)| |PostgreSQL||Data ingestion|Snapshot and real-time CDC data replication from PostgreSQL databases to ClickHouse Cloud.|[Documentation](/docs/en/integrations/postgresql)| -|PowerBI|PowerBI logo|Data visualization|Microsoft Power BI is an interactive data visualization software product developed by Microsoft with a primary focus on business intelligence.|[Documentation](/docs/en/integrations/powerbi) +|PowerBI|PowerBI logo|Data visualization|Microsoft Power BI is an interactive data visualization software product developed by Microsoft with a primary focus on business intelligence.|[Documentation](/docs/en/integrations/powerbi) |Python||Language client|A suite of Python packages for connecting Python to ClickHouse.|[Documentation](/docs/en/integrations/language-clients/python/index.md)| |QuickSight||Data visualization|Amazon QuickSight powers data-driven organizations with unified business intelligence (BI) at hyperscale.|[Documentation](/docs/en/integrations/quicksight)| |RabbitMQ||Data ingestion|Allows ClickHouse to connect [RabbitMQ](https://www.rabbitmq.com/).|[Documentation](/docs/en/engines/table-engines/integrations/rabbitmq)| |Redis||Data ingestion|Allows ClickHouse to use [Redis](https://redis.io/) as a dictionary source.|[Documentation](/docs/en/sql-reference/dictionaries/index.md#redis)| -|Redpanda|Redpanda logo|Data ingestion|Redpanda is the streaming data platform for developers. It’s API-compatible with Apache Kafka, but 10x faster, much easier to use, and more cost effective|[Blog](https://redpanda.com/blog/real-time-olap-database-clickhouse-redpanda)| -|Rust|Rust logo|Language client|A typed client for ClickHouse|[Documentation](/docs/en/integrations/language-clients/rust.md)| +|Redpanda|Redpanda logo|Data ingestion|Redpanda is the streaming data platform for developers. It’s API-compatible with Apache Kafka, but 10x faster, much easier to use, and more cost effective|[Blog](https://redpanda.com/blog/real-time-olap-database-clickhouse-redpanda)| +|Rust|Rust logo|Language client|A typed client for ClickHouse|[Documentation](/docs/en/integrations/language-clients/rust.md)| |SQLite||Data ingestion|Allows to import and export data to SQLite and supports queries to SQLite tables directly from ClickHouse.|[Documentation](/docs/en/engines/table-engines/integrations/sqlite)| |Superset||Data visualization|Explore and visualize your ClickHouse data with Apache Superset.|[Documentation](/docs/en/integrations/data-visualization/superset-and-clickhouse.md)| -|Tableau|Tableau logo|Data visualization|Interactive data visualization software focused on business intelligence|[Documentation](/docs/en/integrations/tableau)| -|Tableau Online|Tableau Online logo|Data visualization|Tableau Online streamlines the power of data to make people faster and more confident decision makers from anywhere|[Documentation](/docs/en/integrations/tableau-online)| +|Tableau|Tableau logo|Data visualization|Interactive data visualization software focused on business intelligence|[Documentation](/docs/en/integrations/tableau)| +|Tableau Online|Tableau Online logo|Data visualization|Tableau Online streamlines the power of data to make people faster and more confident decision makers from anywhere|[Documentation](/docs/en/integrations/tableau-online)| @@ -154,138 +154,138 @@ We are actively compiling this list of ClickHouse integrations below, so it's no Partner integrations -
+
|Name|Logo|Category|Description|Resources| |------|----|----------------|------------------|-------------| -|Airbyte|Airbyte logo|Data ingestion|Use Airbyte, to create ELT data pipelines with more than 140 connectors to load and sync your data into ClickHouse.|[Documentation](/docs/en/integrations/data-ingestion/etl-tools/airbyte-and-clickhouse.md)| -|AccelData|AccelData Logo|Data management|ADOC allows users to monitor and ensure the dependability and integrity of their visualized data, facilitating rea-time data processing and analytics.|[Documentation](https://docs.acceldata.io/documentation/clickhouse) | -|Atlas|Atlas logo|Schema management|Manage your ClickHouse schema as code. |[Documentation](https://atlasgo.io/guides/clickhouse?utm_source=clickhouse&utm_term=docs)| +|Airbyte|Airbyte logo|Data ingestion|Use Airbyte, to create ELT data pipelines with more than 140 connectors to load and sync your data into ClickHouse.|[Documentation](/docs/en/integrations/data-ingestion/etl-tools/airbyte-and-clickhouse.md)| +|AccelData|AccelData Logo|Data management|ADOC allows users to monitor and ensure the dependability and integrity of their visualized data, facilitating rea-time data processing and analytics.|[Documentation](https://docs.acceldata.io/documentation/clickhouse) | +|Atlas|Atlas logo|Schema management|Manage your ClickHouse schema as code. |[Documentation](https://atlasgo.io/guides/clickhouse?utm_source=clickhouse&utm_term=docs)| |Astrato||Data visualization|Astrato brings true Self-Service BI to Enterprises & Data Businesses by putting analytics in the hands of every user, enabling them to build dashboards, reports & data apps without IT.|[Documentation](/docs/en/integrations/astrato)| -|AutoMQ|AutoMQ logo|Data ingestion|Cloud-native Kafka and RocketMQ alternatives that decouples durability to S3 and EBS|[Website](https://www.automq.com/)| -|BlinkOps|BlinkOps Logo|Security automation|Create automations to manage data and user permissions.|[Documentation](https://docs.blinkops.com/docs/integrations/clickhouse)| +|AutoMQ|AutoMQ logo|Data ingestion|Cloud-native Kafka and RocketMQ alternatives that decouples durability to S3 and EBS|[Website](https://www.automq.com/)| +|BlinkOps|BlinkOps Logo|Security automation|Create automations to manage data and user permissions.|[Documentation](https://docs.blinkops.com/docs/integrations/clickhouse)| |Bytewax||Data ingestion|Open source Python stream processor for transforming and ingesting data to ClickHouse|[Documentation](https://bytewax.io/blog/building-a-click-house-sink-for-bytewax)| -|Calyptia (Fluent Bit)|Calyptia logo|Data ingestion|CNCF graduated open-source project for the collection, processing, and delivery of logs, metrics, and traces|[Blog](https://clickhouse.com/blog/kubernetes-logs-to-clickhouse-fluent-bit)| +|Calyptia (Fluent Bit)|Calyptia logo|Data ingestion|CNCF graduated open-source project for the collection, processing, and delivery of logs, metrics, and traces|[Blog](https://clickhouse.com/blog/kubernetes-logs-to-clickhouse-fluent-bit)| |CloudCanal||Data integration|A data synchronization and migration tool.|[Website](https://www.cloudcanalx.com/us/)| |CloudQuery||Data ingestion|Open source high-performance ELT framework.|[Documentation](https://www.cloudquery.io/docs/plugins/destinations/clickhouse/overview)| |Cube.js||Data visualization|Cube is the Semantic Layer for building data apps.|[Website](https://cube.dev/for/clickhouse-dashboard)| |DBeaver||SQL client|Free multi-platform database administration tool. Connects to Clickhouse through JDBC driver.|[Documentation](/docs/en/integrations/sql-clients/dbeaver.md)| |DataGrip||SQL client|DataGrip is a powerful database IDE with dedicated support for ClickHouse.|[Documentation](/docs/en/integrations/sql-clients/datagrip.md)| -|Dataddo|Dataddo logo|Data integration|Data integration platform|[Website](https://www.dataddo.com/storage/clickhouse)| -|DbVisualizer|DbVisualizer logo|SQL client|DbVisualizer is a database tool with extended support for ClickHouse.|[Documentation](/docs/en/integrations/sql-clients/dbvisualizer.md)| -|Decodable|Decodable logo|Data ingestion|Powerful Stream Processing Built On Apache Flink|[Website](https://www.decodable.co/connectors/clickhouse)| +|Dataddo|Dataddo logo|Data integration|Data integration platform|[Website](https://www.dataddo.com/storage/clickhouse)| +|DbVisualizer|DbVisualizer logo|SQL client|DbVisualizer is a database tool with extended support for ClickHouse.|[Documentation](/docs/en/integrations/sql-clients/dbvisualizer.md)| +|Decodable|Decodable logo|Data ingestion|Powerful Stream Processing Built On Apache Flink|[Website](https://www.decodable.co/connectors/clickhouse)| |Deepnote||Data visualization|Deepnote is a collaborative Jupyter-compatible data notebook built for teams to discover and share insights.|[Documentation](/docs/en/integrations/data-visualization/deepnote.md)| |DLT||Data integration|An open source Python library that makes data loading easy.|[Documentation](/docs/en/integrations/dlt)| |Draxlr||Data visualization|Draxlr is a Business intelligence tool with data visualization and analytics.|[Documentation](/docs/en/integrations/data-visualization/draxlr-and-clickhouse.md)| |EMQX||Data ingestion|EMQX is an open source MQTT broker with a high-performance real-time message processing engine, powering event streaming for IoT devices at massive scale.|[Documentation](/docs/en/integrations/emqx)| -|Explo|Explo logo|Data visualization|Explo is a customer-facing analytics tool for any platform|[Documentation](/docs/en/integrations/explo)| -|Gigasheet|Gigasheet logo|Data visualization|A cloud big data analytics spreadsheet that enables business users to instantly analyze and explore ClickHouse data.|[Website](https://gigasheet.com/enterprise)| -|GlassFlow|GlassFlow Logo|Data ingestion|Real-time event streaming and data transformation in Python for ClickHouse |[Documentation](https://docs.glassflow.dev/integrations/managed-connectors/sinks/clickhouse)| -|Goldsky||Data Integration|High-performance Web3 data indexing via Subgraphs and realtime data replication pipelines.|[Documentation](https://docs.goldsky.com/introduction)| +|Explo|Explo logo|Data visualization|Explo is a customer-facing analytics tool for any platform|[Documentation](/docs/en/integrations/explo)| +|Gigasheet|Gigasheet logo|Data visualization|A cloud big data analytics spreadsheet that enables business users to instantly analyze and explore ClickHouse data.|[Website](https://gigasheet.com/enterprise)| +|GlassFlow|GlassFlow Logo|Data ingestion|Real-time event streaming and data transformation in Python for ClickHouse |[Documentation](https://docs.glassflow.dev/integrations/managed-connectors/sinks/clickhouse)| +|Goldsky||Data Integration|High-performance Web3 data indexing via Subgraphs and realtime data replication pipelines.|[Documentation](https://docs.goldsky.com/introduction)| |Grafana||Data visualization|With Grafana you can create, explore and share all of your data through dashboards.|[Documentation](/docs/en/integrations/data-visualization/grafana/index.md)| -|Great Expectations|Great Expectations logo|Data management|An open-source data management tool, with a paid cloud offering.|[Website](https://greatexpectations.io/)| -|GrowthBook|GrowthBook logo|Data visualization|Warehouse native experimentation platform (feature flagging and A/B testing).|[Documentation](https://docs.growthbook.io/warehouses/clickhouse)| -|HEX|HEX logo|Data visualization|Hex is a modern, collaborative platform with notebooks, data apps, SQL, Python, no-code, R, and so much more.|[Documentation](https://learn.hex.tech/docs/connect-to-data/data-connections/overview)| +|Great Expectations|Great Expectations logo|Data management|An open-source data management tool, with a paid cloud offering.|[Website](https://greatexpectations.io/)| +|GrowthBook|GrowthBook logo|Data visualization|Warehouse native experimentation platform (feature flagging and A/B testing).|[Documentation](https://docs.growthbook.io/warehouses/clickhouse)| +|HEX|HEX logo|Data visualization|Hex is a modern, collaborative platform with notebooks, data apps, SQL, Python, no-code, R, and so much more.|[Documentation](https://learn.hex.tech/docs/connect-to-data/data-connections/overview)| |Hashboard||Data visualization|[Hashboard](https://hashboard.com) is a business intelligence platform that enables self-service data exploration and metric tracking.|[Documentation](https://docs.hashboard.com/docs/database-connections/clickhouse)| -|HighTouch|HighTouch logo|Data integration|Sync your data directly from your warehouse to 140+ destinations|[Website](https://hightouch.com/docs/sources/clickhouse)| -|Holistics|Holistics logo|Data visualization|Business Intelligence for ClickHouse database|[Website](https://www.holistics.io/integrations/clickhouse/)| -|HouseWatch|HouseWatch logo|Data management| Open source tool for monitoring and managing ClickHouse clusters. |[GitHub](https://github.com/PostHog/HouseWatch)| +|HighTouch|HighTouch logo|Data integration|Sync your data directly from your warehouse to 140+ destinations|[Website](https://hightouch.com/docs/sources/clickhouse)| +|Holistics|Holistics logo|Data visualization|Business Intelligence for ClickHouse database|[Website](https://www.holistics.io/integrations/clickhouse/)| +|HouseWatch|HouseWatch logo|Data management| Open source tool for monitoring and managing ClickHouse clusters. |[GitHub](https://github.com/PostHog/HouseWatch)| |IBM Instana| |Data management|Instana can auto-discover and monitor ClickHouse server processes|[Documentation](https://www.ibm.com/docs/en/instana-observability/current?topic=technologies-monitoring-clickhouse)| |Jitsu||Data analytics|An open-source event collection platform.|[Documentation](https://classic.jitsu.com/docs/destinations-configuration/clickhouse-destination)| |LangChain|🦜️🔗|SDK|LangChain is a framework for developing applications powered by language models|[Documentation](https://python.langchain.com/v0.2/docs/integrations/vectorstores/clickhouse/)| -|Luzmo|Luzmo logo|Data visualization|Luzmo is an embedded analytics platform with a native ClickHouse integration, purpose-built for Software and SaaS applications.|[Documentation](/docs/en/integrations/data-visualization/luzmo-and-clickhouse.md)| -|Mage|Metaplane logo|Data Ingestion|Open-source data pipeline tool for transforming and integrating data|[Documentation](https://docs.mage.ai/integrations/databases/ClickHouse) -|Metaplane|Metaplane logo|Data management|Data observability for every data team|[Website](https://www.metaplane.dev/integrations)| -|MindsDB|MindsDB logo|AI/ML| The platform for customizing AI from enterprise data |[Website](https://mindsdb.com/clickhouse-machine-learning )| -|Mitzu|Mitzu logo|Data visualization|Mitzu is a no-code warehouse-native product analytics application. Find funnel, retention, user segmentation insights without copying your data.|[Documentation](/docs/en/integrations/mitzu)| -|Mode Analytics|Mode logo|Data visualization|Business Intelligence built around data teams|[Website](https://mode.com/)| -|Omni||Data visualization|Business intelligence that speaks your language. Explore, visualize, and model data your way with Omni. From spreadsheets to SQL—in a single platform.| [Website](https://omni.co/)| -|Openblocks|Openblocks logo|SQL client|Openblocks is a low code platform for building UIs|[Documentation](https://blog.openblocks.dev/blog/openblocks-x-clickhouse)| -|OpsRamp (HP)| OpsRamp Logo|Data management| Provides observability metrics for ClickHouse|[Documentation](https://docs.opsramp.com/integrations/database-no-sql/automonitor-clickhouse-monitoring/)| +|Luzmo|Luzmo logo|Data visualization|Luzmo is an embedded analytics platform with a native ClickHouse integration, purpose-built for Software and SaaS applications.|[Documentation](/docs/en/integrations/data-visualization/luzmo-and-clickhouse.md)| +|Mage|Metaplane logo|Data Ingestion|Open-source data pipeline tool for transforming and integrating data|[Documentation](https://docs.mage.ai/integrations/databases/ClickHouse) +|Metaplane|Metaplane logo|Data management|Data observability for every data team|[Website](https://www.metaplane.dev/integrations)| +|MindsDB|MindsDB logo|AI/ML| The platform for customizing AI from enterprise data |[Website](https://mindsdb.com/clickhouse-machine-learning )| +|Mitzu|Mitzu logo|Data visualization|Mitzu is a no-code warehouse-native product analytics application. Find funnel, retention, user segmentation insights without copying your data.|[Documentation](/docs/en/integrations/mitzu)| +|Mode Analytics|Mode logo|Data visualization|Business Intelligence built around data teams|[Website](https://mode.com/)| +|Omni||Data visualization|Business intelligence that speaks your language. Explore, visualize, and model data your way with Omni. From spreadsheets to SQL—in a single platform.| [Website](https://omni.co/)| +|Openblocks|Openblocks logo|SQL client|Openblocks is a low code platform for building UIs|[Documentation](https://blog.openblocks.dev/blog/openblocks-x-clickhouse)| +|OpsRamp (HP)| OpsRamp Logo|Data management| Provides observability metrics for ClickHouse|[Documentation](https://docs.opsramp.com/integrations/database-no-sql/automonitor-clickhouse-monitoring/)| |Popsink||Data integration|Build real-time Change Data Capture (CDC) pipelines to ClickHouse.|[Documentation](https://docs.popsink.com/connectors/target/clickhouse/)| |Prequel||Data sharing|Connect your ClickHouse instance to Prequel to share data to or sync data from your users and partners.|[Documentation](https://docs.prequel.co/docs/sources-clickhouse-generic)| |Quesma||Data integration|Use Kibana and OpenSearch Dashboards with data in ClickHouse.|[Website](https://quesma.com/quesma-for-elk)| -|Redash|Redash logo|Data visualization|Connect and query your data sources, build dashboards to visualize data and share|[Website](https://redash.io/help/data-sources/querying/supported-data-sources)| -|Restack Data Hub|Restack logo|Data governance|Users can achieve more comprehensive data governance and observability framework with Restack Data Hub.|[Documentation](https://www.restack.io/docs/datahub-knowledge-datahub-clickhouse-integration)| -|Restack OpenMetadata|Restack logo|Data quality|Restack OpenMetadata supports metadata extraction, query usage tracking, data profiling, and data quality checks.|[Documentation](https://www.restack.io/docs/openmetadata-knowledge-openmetadata-clickhouse-integration)| +|Redash|Redash logo|Data visualization|Connect and query your data sources, build dashboards to visualize data and share|[Website](https://redash.io/help/data-sources/querying/supported-data-sources)| +|Restack Data Hub|Restack logo|Data governance|Users can achieve more comprehensive data governance and observability framework with Restack Data Hub.|[Documentation](https://www.restack.io/docs/datahub-knowledge-datahub-clickhouse-integration)| +|Restack OpenMetadata|Restack logo|Data quality|Restack OpenMetadata supports metadata extraction, query usage tracking, data profiling, and data quality checks.|[Documentation](https://www.restack.io/docs/openmetadata-knowledge-openmetadata-clickhouse-integration)| |Retool||No code|Create your application with drag-and-drop interface. |[Documentation](/docs/en/integrations/retool)| |Rill||Data visualization|Rill is an Operational BI tool purpose-built for slicing & dicing data with OLAP engines.|[Documentation](https://docs.rilldata.com/reference/olap-engines/clickhouse)| |RisingWave||Data ingestion| SQL stream processing with a Postgres-like experience. 10x faster and more cost-efficient than Apache Flink. |[Documentation](https://docs.risingwave.com/docs/current/sink-to-clickhouse/)| |RudderStack||Data ingestion|RudderStack makes it easy to collect and send customer data to the tools and teams that need it|[Documentation](https://www.rudderstack.com/docs/destinations/warehouse-destinations/clickhouse/)| -|RunReveal|RunReveal logo|Data ingestion|Ingest and normalize audit logs from any SaaS application into ClickHouse. Create alerts and detections from scheduled queries.|[Website](https://runreveal.com)| -|Sematext|Sematext logo|Data management|Observability monitoring for ClickHouse databases.|[Documentation](https://sematext.com/docs/integration/clickhouse/)| -|SiSense|SiSense logo|Data visualization|Embed analytics into any application or workflow|[Website](https://www.sisense.com/data-connectors/)| -|SigNoz|SigNoz logo|Data visualization|Open Source Observability Platform|[Documentation](https://www.signoz.io/docs/architecture/)| -|Snappy Flow|Snappy Flow logo|Data management|Collects ClickHouse database metrics via plugin.|[Documentation](https://docs.snappyflow.io/docs/Integrations/clickhouse/instance)| -|Soda|Soda logo|Data quality|Soda integration makes it easy for organziations to detect, resolve, and prevent data quality issues by running data quality checks on data before it is loaded into the database.|[Website](https://www.soda.io/integrations/clickhouse)| -|StreamingFast|StreamingFast logo|Data ingestion| Blockchain-agnostic, parallelized and streaming-first data engine. |[Website](https://www.streamingfast.io/)| -|Streamkap|Streamkap logo|Data ingestion|Setup real-time CDC (Change Data Capture) streaming to ClickHouse with high throughput in minutes.|[Documentation](https://docs.streamkap.com/docs/clickhouse)| -|Supabase|Supabase logo|Data ingestion|Open source Firebase alternative|[GitHub](https://github.com/supabase/wrappers/tree/main/wrappers/src/fdw/clickhouse_fdw),[Blog](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres)| -|Teleport|Teleport logo|Secure connection|Teleport Database Service authenticates to ClickHouse using x509 certificates, which are available for the ClickHouse HTTP and Native (TCP) interfaces.|[Documentation](https://goteleport.com/docs/database-access/enroll-self-hosted-databases/clickhouse-self-hosted/)| +|RunReveal|RunReveal logo|Data ingestion|Ingest and normalize audit logs from any SaaS application into ClickHouse. Create alerts and detections from scheduled queries.|[Website](https://runreveal.com)| +|Sematext|Sematext logo|Data management|Observability monitoring for ClickHouse databases.|[Documentation](https://sematext.com/docs/integration/clickhouse/)| +|SiSense|SiSense logo|Data visualization|Embed analytics into any application or workflow|[Website](https://www.sisense.com/data-connectors/)| +|SigNoz|SigNoz logo|Data visualization|Open Source Observability Platform|[Documentation](https://www.signoz.io/docs/architecture/)| +|Snappy Flow|Snappy Flow logo|Data management|Collects ClickHouse database metrics via plugin.|[Documentation](https://docs.snappyflow.io/docs/Integrations/clickhouse/instance)| +|Soda|Soda logo|Data quality|Soda integration makes it easy for organziations to detect, resolve, and prevent data quality issues by running data quality checks on data before it is loaded into the database.|[Website](https://www.soda.io/integrations/clickhouse)| +|StreamingFast|StreamingFast logo|Data ingestion| Blockchain-agnostic, parallelized and streaming-first data engine. |[Website](https://www.streamingfast.io/)| +|Streamkap|Streamkap logo|Data ingestion|Setup real-time CDC (Change Data Capture) streaming to ClickHouse with high throughput in minutes.|[Documentation](https://docs.streamkap.com/docs/clickhouse)| +|Supabase|Supabase logo|Data ingestion|Open source Firebase alternative|[GitHub](https://github.com/supabase/wrappers/tree/main/wrappers/src/fdw/clickhouse_fdw),[Blog](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres)| +|Teleport|Teleport logo|Secure connection|Teleport Database Service authenticates to ClickHouse using x509 certificates, which are available for the ClickHouse HTTP and Native (TCP) interfaces.|[Documentation](https://goteleport.com/docs/database-access/enroll-self-hosted-databases/clickhouse-self-hosted/)| |TABLUM.IO||SQL client|TABLUM.IO ingests data from a variety of sources, normalizes and cleans inconsistencies, and gives you access to it via SQL.|[Documentation](/docs/en/integrations/sql-clients/tablum.md)| -|Tooljet|Tooljet logo|Data Visualization|ToolJet is an open-source low-code framework to build and deploy custom internal tools.|[Documentation](https://docs.tooljet.com/docs/data-sources/clickhouse/)| +|Tooljet|Tooljet logo|Data Visualization|ToolJet is an open-source low-code framework to build and deploy custom internal tools.|[Documentation](https://docs.tooljet.com/docs/data-sources/clickhouse/)| |Upstash||Data Ingestion|A data platform offering serverless Kafka and other solutions|[Website](https://upstash.com/)| -|Vector|Vector logo|Data ingestion|A lightweight, ultra-fast tool for building observability pipelines with built-in compatibility with ClickHouse.|[Documentation](/docs/en/integrations/vector/)| +|Vector|Vector logo|Data ingestion|A lightweight, ultra-fast tool for building observability pipelines with built-in compatibility with ClickHouse.|[Documentation](/docs/en/integrations/vector/)| |WarpStream||Data Ingestion|A Kafka compatible data streaming platform built directly on top of object storage|[Website](https://www.warpstream.com/)| |YepCode||Data integration|YepCode is the integration & automation tool that loves source code.|[Documentation](https://yepcode.io/docs/integrations/clickhouse/)| -|Zing Data|Zing logo|Data visualization|Simple social business intelligence for ClickHouse, made for iOS, Android and the web.|[Documentation](https://docs.getzingdata.com/docs/)| +|Zing Data|Zing logo|Data visualization|Simple social business intelligence for ClickHouse, made for iOS, Android and the web.|[Documentation](https://docs.getzingdata.com/docs/)|

Community integrations -
+
|Name|Logo|Category|Description|Resources| |------|----|----------------|------------------|-------------| -|Apache Airflow|Airflow logo|Data ingestion|Open-source workflow management platform for data engineering pipelines|[Github](https://github.com/bryzgaloff/airflow-clickhouse-plugin)| -|Apache Beam|Beam logo|Data ingestion|Open source, unified model and set of language-specific SDKs for defining and executing data processing workflows. Compatible with Google Dataflow.|[Documentation](https://clickhouse.com/docs/en/integrations/apache-beam),
[Examples](https://github.com/ClickHouse/clickhouse-beam-connector/)| -|Apache InLong|InLong logo|Data ingestion|One-stop integration framework for massive data|[Documentation](https://inlong.apache.org/docs/data_node/load_node/clickhouse)| -|Apache Nifi|Nifi logo|Data ingestion|Automates the flow of data between software systems|[Documentation](/docs/en/integrations/nifi)| -|Apache SeaTunnel|SeaTunnel logo|Data ingestion|SeaTunnel is a very easy-to-use ultra-high-performance distributed data integration platform|[Website](https://seatunnel.apache.org/docs/2.3.0/connector-v2/sink/Clickhouse)| -|Apache SkyWalking|SkyWalking logo|Data management|Open-source APM system that provides monitoring, tracing and diagnosing capabilities for distributed systems in Cloud Native architectures.|[Blog](https://skywalking.apache.org/blog/2024-03-12-monitoring-clickhouse-through-skywalking/)| -|Apache StreamPark|SeaTunnel logo|Data ingestion|A stream processing application development framework and stream processing operation platform. |[Website](https://streampark.apache.org/docs/intro)| -|Bytebase|ByteBase logo|Data management|Open-source database DevOps tool, it's the GitLab for managing databases throughout the application development lifecycle|[Documentation](https://www.bytebase.com/docs/introduction/supported-databases)| +|Apache Airflow|Airflow logo|Data ingestion|Open-source workflow management platform for data engineering pipelines|[Github](https://github.com/bryzgaloff/airflow-clickhouse-plugin)| +|Apache Beam|Beam logo|Data ingestion|Open source, unified model and set of language-specific SDKs for defining and executing data processing workflows. Compatible with Google Dataflow.|[Documentation](https://clickhouse.com/docs/en/integrations/apache-beam),
[Examples](https://github.com/ClickHouse/clickhouse-beam-connector/)| +|Apache InLong|InLong logo|Data ingestion|One-stop integration framework for massive data|[Documentation](https://inlong.apache.org/docs/data_node/load_node/clickhouse)| +|Apache Nifi|Nifi logo|Data ingestion|Automates the flow of data between software systems|[Documentation](/docs/en/integrations/nifi)| +|Apache SeaTunnel|SeaTunnel logo|Data ingestion|SeaTunnel is a very easy-to-use ultra-high-performance distributed data integration platform|[Website](https://seatunnel.apache.org/docs/2.3.0/connector-v2/sink/Clickhouse)| +|Apache SkyWalking|SkyWalking logo|Data management|Open-source APM system that provides monitoring, tracing and diagnosing capabilities for distributed systems in Cloud Native architectures.|[Blog](https://skywalking.apache.org/blog/2024-03-12-monitoring-clickhouse-through-skywalking/)| +|Apache StreamPark|SeaTunnel logo|Data ingestion|A stream processing application development framework and stream processing operation platform. |[Website](https://streampark.apache.org/docs/intro)| +|Bytebase|ByteBase logo|Data management|Open-source database DevOps tool, it's the GitLab for managing databases throughout the application development lifecycle|[Documentation](https://www.bytebase.com/docs/introduction/supported-databases)| |C#||Language client|ClickHouse.Client is a feature-rich ADO.NET client implementation for ClickHouse|[Documentation](https://github.com/DarkWanderer/ClickHouse.Client/wiki/Quick-start)| |CHProxy| |Data management|Chproxy is an HTTP proxy and load balancer for the ClickHouse database|[GitHub](https://github.com/ContentSquare/chproxy)| |Chat-DBT| |AI Integration|Create ClickHouse queries using Chat GPT.|[GitHub](https://github.com/plmercereau/chat-dbt)| |ClickHouse Monitoring Dashboard||Dashboard|A simple monitoring dashboard for ClickHouse|[Github](https://github.com/duyet/clickhouse-monitoring)| -|Common Lisp|clickhouse-cl Logo|Language client|Common Lisp ClickHouse Client Library|[GitHub](https://github.com/juliojimenez/clickhouse-cl)| -|DBNet|Airflow logo|Software IDE|Web-based SQL IDE using Go as a back-end, and the browser as the front-end.|[Github](https://github.com/dbnet-io/dbnet)| -|DataLens|Datalens logo|Data visualization|An open-source data analytics and visualization tool.|[Website](https://datalens.tech/),
[Documentation](https://datalens.tech/docs/en/)| -|Dataease|Dataease logo|Data visualization|Open source data visualization analysis tool to help users analyze data and gain insight into business trends.|[Website](https://dataease.io/)| -|Datahub|Datahub logo|Data management|Open Source Data Catalog that enables data discovery, data observability and federated governance|[Documentation](https://datahubproject.io/docs/generated/ingestion/sources/clickhouse/)| +|Common Lisp|clickhouse-cl Logo|Language client|Common Lisp ClickHouse Client Library|[GitHub](https://github.com/juliojimenez/clickhouse-cl)| +|DBNet|Airflow logo|Software IDE|Web-based SQL IDE using Go as a back-end, and the browser as the front-end.|[Github](https://github.com/dbnet-io/dbnet)| +|DataLens|Datalens logo|Data visualization|An open-source data analytics and visualization tool.|[Website](https://datalens.tech/),
[Documentation](https://datalens.tech/docs/en/)| +|Dataease|Dataease logo|Data visualization|Open source data visualization analysis tool to help users analyze data and gain insight into business trends.|[Website](https://dataease.io/)| +|Datahub|Datahub logo|Data management|Open Source Data Catalog that enables data discovery, data observability and federated governance|[Documentation](https://datahubproject.io/docs/generated/ingestion/sources/clickhouse/)| |Dbmate| |Data management|Database migration tool that will keep your database schema in sync across multiple developers and servers|[GitHub](https://github.com/amacneil/dbmate#clickhouse)| -|DeepFlow|Deepflow logo|Data ingestion|Application Observability using eBPF|[Website](https://deepflow.io)| -|Easypanel|Easypanel logo|Deployment method|It's a modern server control panel. You can use it to deploy ClickHouse on your own server.|[Website](https://easypanel.io),
[Documentation](docs/en/integrations/deployment/easypanel/index.md)| +|DeepFlow|Deepflow logo|Data ingestion|Application Observability using eBPF|[Website](https://deepflow.io)| +|Easypanel|Easypanel logo|Deployment method|It's a modern server control panel. You can use it to deploy ClickHouse on your own server.|[Website](https://easypanel.io),
[Documentation](docs/en/integrations/deployment/easypanel/index.md)| |Explo||Data visualization|Explo helps companies build real-time analytics dashboard by providing flexible components.|[Website](https://www.explo.co/integrations/clickhouse)| -|Flink|Flink logo|Data ingestion|Flink sink for ClickHouse database, powered by Async Http Client|[GitHub](https://github.com/itinycheng/flink-connector-clickhouse)| -|Goose|Goose logo|Data migration|A database migration tool that supports SQL migrations and Go functions.|[GitHub](https://github.com/pressly/goose),
[Documentation](https://pressly.github.io/goose/)| -|Ibis|Ibis logo|Language client|The flexibility of Python analytics with the scale and performance of modern SQL|[Website](https://ibis-project.org/backends/ClickHouse/)| -|Jaeger|Jaeger logo|Data ingestion|Jaeger gRPC storage plugin implementation for storing traces in ClickHouse|[GitHub](https://github.com/jaegertracing/jaeger-clickhouse)| -|JupySQL|JupySQL logo|SQL client|The native SQL client for Jupyter notebooks.|[Documentation](/docs/en/integrations/jupysql)| +|Flink|Flink logo|Data ingestion|Flink sink for ClickHouse database, powered by Async Http Client|[GitHub](https://github.com/itinycheng/flink-connector-clickhouse)| +|Goose|Goose logo|Data migration|A database migration tool that supports SQL migrations and Go functions.|[GitHub](https://github.com/pressly/goose),
[Documentation](https://pressly.github.io/goose/)| +|Ibis|Ibis logo|Language client|The flexibility of Python analytics with the scale and performance of modern SQL|[Website](https://ibis-project.org/backends/ClickHouse/)| +|Jaeger|Jaeger logo|Data ingestion|Jaeger gRPC storage plugin implementation for storing traces in ClickHouse|[GitHub](https://github.com/jaegertracing/jaeger-clickhouse)| +|JupySQL|JupySQL logo|SQL client|The native SQL client for Jupyter notebooks.|[Documentation](/docs/en/integrations/jupysql)| |Kestra||Data orchestration|Open source data orchestration and scheduling platform|[Website](https://kestra.io/plugins/plugin-jdbc-clickhouse/)| -|Logchain|Adaptive Logchain logo|Security|Data security and privileged access management|[Website](https://github.com/adaptive-scale/logchain)| +|Logchain|Adaptive Logchain logo|Security|Data security and privileged access management|[Website](https://github.com/adaptive-scale/logchain)| |Meltano||Data ingestion|Meltano is an open-source, full-stack data integration platform|[Documentation](https://hub.meltano.com/extractors/tap-clickhouse) -|Mprove|mprove logo|Data visualization|Self-service Business Intelligence with Version Control|[Website](https://mprove.io/)| +|Mprove|mprove logo|Data visualization|Self-service Business Intelligence with Version Control|[Website](https://mprove.io/)| |Netobserv||Data management|An OpenShift and Kubernetes operator for network observability.|[Blog](https://cloud.redhat.com/blog/deploying-network-observability-without-loki-an-example-with-clickhouse) |Observable||Data visualization|Observable is a platform where you can collaboratively explore, analyze, visualize, and communicate with data on the web.|[Website](https://observablehq.com/@stas-sl/clickhouse-playground)| -|PHP|PHP logo|Language client|This extension provides the ClickHouse integration for the Yii framework 2.0|[GitHub](https://github.com/smi2/phpClickHouse)| +|PHP|PHP logo|Language client|This extension provides the ClickHouse integration for the Yii framework 2.0|[GitHub](https://github.com/smi2/phpClickHouse)| |Pgwarehouse||Data ingestion|Simple tool to quickly replicate Postgres tables into ClickHouse|[GitHub](https://github.com/scottpersinger/pgwarehouse)| -|Pinax|Pinax logo|Blockchain analytics|Indexing, analytics, and search tools for blockchains.|[Blog](https://blog.pinax.network/substreams/simplify-real-time-blockchain-analytics-with-clickhouse/)| -|Pulse|PulseUI logo|Data management|A developer platform for internal data UIs.|[Website](https://www.timestored.com/pulse/)| -|QStudio|QStudio logo|GUI|A simple to use GUI for interacting with ClickHouse databases.|[Website](https://www.timestored.com/qstudio/database/clickhouse)| -|QStudio|qStudio logo|SQL client|qStudio is a free SQL GUI for running SQL scripts and charting results|[Documentation](https://www.timestored.com/qstudio/database/clickhouse)| -|Qryn|qryn logo|Data Ingestion, Management, Visualization | qryn is a polyglot observability stack built on top of ClickHouse, transparently compatible with Loki, Prometheus, Tempo, Opentelemetry and many other formats and standard APIs without requiring custom clients, code or plugins|[Documentation](https://qryn.dev), [Github](https://github.com/metrico), [Website](https://qryn.cloud)| -|RSyslog|RSyslog logo|Data Ingestion|This module provides native support for logging to ClickHouse.|[Documentation](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)| +|Pinax|Pinax logo|Blockchain analytics|Indexing, analytics, and search tools for blockchains.|[Blog](https://blog.pinax.network/substreams/simplify-real-time-blockchain-analytics-with-clickhouse/)| +|Pulse|PulseUI logo|Data management|A developer platform for internal data UIs.|[Website](https://www.timestored.com/pulse/)| +|QStudio|QStudio logo|GUI|A simple to use GUI for interacting with ClickHouse databases.|[Website](https://www.timestored.com/qstudio/database/clickhouse)| +|QStudio|qStudio logo|SQL client|qStudio is a free SQL GUI for running SQL scripts and charting results|[Documentation](https://www.timestored.com/qstudio/database/clickhouse)| +|Qryn|qryn logo|Data Ingestion, Management, Visualization | qryn is a polyglot observability stack built on top of ClickHouse, transparently compatible with Loki, Prometheus, Tempo, Opentelemetry and many other formats and standard APIs without requiring custom clients, code or plugins|[Documentation](https://qryn.dev), [Github](https://github.com/metrico), [Website](https://qryn.cloud)| +|RSyslog|RSyslog logo|Data Ingestion|This module provides native support for logging to ClickHouse.|[Documentation](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html)| |Rocket.BI||Data visualization|RocketBI is a self-service business intelligence platform that helps you quickly analyze data, build drag-n-drop visualizations and collaborate with colleagues right on your web browser.|[GitHub](https://github.com/datainsider-co/rocket-bi),
[Documentation](/docs/en/integrations/data-visualization/rocketbi-and-clickhouse.md)| -|Ruby|Ruby logo|Language client|A modern Ruby database driver for ClickHouse|[GitHub](https://github.com/shlima/click_house)| -|R|R logo|Language client|R package is a DBI interface for the ClickHouse database|[GitHub](https://github.com/IMSMWU/RClickHouse)| +|Ruby|Ruby logo|Language client|A modern Ruby database driver for ClickHouse|[GitHub](https://github.com/shlima/click_house)| +|R|R logo|Language client|R package is a DBI interface for the ClickHouse database|[GitHub](https://github.com/IMSMWU/RClickHouse)| |SQLPad||SQL client|SQLPad is a web app for writing and running SQL queries and visualizing the results|[Documentation](https://getsqlpad.com/en/connections/#clickhouse)| -|Scala|Scala logo|Language client|ClickHouse Scala Client that uses Akka Http|[GitHub](https://github.com/crobox/clickhouse-scala-client)| -|SchemaSpy|SchemaSpy logo|Data visualization|SchemaSpy supports ClickHouse schema visualuzation|[GitHub](https://github.com/schemaspy/schemaspy)| +|Scala|Scala logo|Language client|ClickHouse Scala Client that uses Akka Http|[GitHub](https://github.com/crobox/clickhouse-scala-client)| +|SchemaSpy|SchemaSpy logo|Data visualization|SchemaSpy supports ClickHouse schema visualuzation|[GitHub](https://github.com/schemaspy/schemaspy)| |TricksterCache||Data visualization| Open Source HTTP Reverse Proxy Cache and Time Series Dashboard Accelerator |[Website](https://trickstercache.org/)| -|Visual Studio Client|VS logo|Language client|Visual studio lightweight client|[Marketplace](https://marketplace.visualstudio.com/items?itemName=fanruten.clickhouse-light)| -|VulcanSQL|VulcanSQL logo|Data API Framework|It's a Data API Framework for data applications that helps data folks create and share data APIs faster. It turns your SQL templates into data APIs. No backend skills required.|[Website](https://vulcansql.com/),
[Documentation](https://vulcansql.com/docs/connect/clickhouse)| +|Visual Studio Client|VS logo|Language client|Visual studio lightweight client|[Marketplace](https://marketplace.visualstudio.com/items?itemName=fanruten.clickhouse-light)| +|VulcanSQL|VulcanSQL logo|Data API Framework|It's a Data API Framework for data applications that helps data folks create and share data APIs faster. It turns your SQL templates into data APIs. No backend skills required.|[Website](https://vulcansql.com/),
[Documentation](https://vulcansql.com/docs/connect/clickhouse)|
diff --git a/docs/en/integrations/language-clients/java/client-v1.md b/docs/en/integrations/language-clients/java/client-v1.md index 7884b2b3ff8..4f0d262c6e7 100644 --- a/docs/en/integrations/language-clients/java/client-v1.md +++ b/docs/en/integrations/language-clients/java/client-v1.md @@ -214,7 +214,7 @@ ClickHouseClient client = ClickHouseClient.builder() .build(); ``` -See the [compression documentation](/en/native-protocol/compression) to learn more about different compression options. +See the [compression documentation](/docs/en/data-compression/compression-modes) to learn more about different compression options. ### Multiple queries diff --git a/docs/en/integrations/language-clients/js.md b/docs/en/integrations/language-clients/js.md index d4b26ba4fd5..bb264b963b1 100644 --- a/docs/en/integrations/language-clients/js.md +++ b/docs/en/integrations/language-clients/js.md @@ -147,17 +147,16 @@ It is possible to configure most of the client instance parameters with a URL. T | `session_id` | an arbitrary string. | | `request_timeout` | non-negative number. | | `max_open_connections` | non-negative number, greater than zero. | -| `compression_request` | boolean. See below [^1] | +| `compression_request` | boolean. See below (1) | | `compression_response` | boolean. | | `log_level` | allowed values: `OFF`, `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`. | | `keep_alive_enabled` | boolean. | -| `clickhouse_setting_*` or `ch_*` | see below [^2] | -| `http_header_*` | see below [^3] | +| `clickhouse_setting_*` or `ch_*` | see below (2) | +| `http_header_*` | see below (3) | | (Node.js only) `keep_alive_idle_socket_ttl` | non-negative number. | -[^1] For booleans, valid values will be `true`/`1` and `false`/`0`. - -[^2] Any parameter prefixed with `clickhouse_setting_` or `ch_` will have this prefix removed and the rest added to client's `clickhouse_settings`. For example, `?ch_async_insert=1&ch_wait_for_async_insert=1` will be the same as: +- (1) For booleans, valid values will be `true`/`1` and `false`/`0`. +- (2) Any parameter prefixed with `clickhouse_setting_` or `ch_` will have this prefix removed and the rest added to client's `clickhouse_settings`. For example, `?ch_async_insert=1&ch_wait_for_async_insert=1` will be the same as: ```ts createClient({ @@ -170,7 +169,7 @@ createClient({ Note: boolean values for `clickhouse_settings` should be passed as `1`/`0` in the URL. -[^3] Similar to [^2], but for `http_header` configuration. For example, `?http_header_x-clickhouse-auth=foobar` will be an equivalent of: +- (3) Similar to (2), but for `http_header` configuration. For example, `?http_header_x-clickhouse-auth=foobar` will be an equivalent of: ```ts createClient({ diff --git a/docs/en/integrations/migration/rockset.md b/docs/en/integrations/migration/rockset.md index 21a26624a5a..40181503ab6 100644 --- a/docs/en/integrations/migration/rockset.md +++ b/docs/en/integrations/migration/rockset.md @@ -47,7 +47,7 @@ Rockset and ClickHouse both support loading data from a variety of sources. In Rockset, you create a data source and then create a _collection_ based on that data source. There are fully managed integrations for event streaming platforms, OLTP databases, and cloud bucket storage. -In ClickHouse Cloud, the equivalent of fully managed integrations is [ClickPipes](/en/integrations/ClickPipes). +In ClickHouse Cloud, the equivalent of fully managed integrations is [ClickPipes](/docs/en/integrations/clickpipes). ClickPipes supports continuously loading data from event streaming platforms and cloud bucket storage. ClickPipes loads data into _tables_. @@ -102,7 +102,7 @@ There are multiple ways to work with JSON in ClickHouse: * JSON extract at query time * JSON extract at insert time -To understand the best approach for your user case, see [our JSON documentation](/docs/en/integrations/data-formats/json). +To understand the best approach for your user case, see [our JSON documentation](/docs/en/integrations/data-formats/json/overview). In addition, ClickHouse will soon have [a Semistructured column data type](https://github.com/ClickHouse/ClickHouse/issues/54864). This new type should give users the flexibility Rockset's JSON type offers. diff --git a/docs/en/quick-start.mdx b/docs/en/quick-start.mdx index ef876155aec..f9157474c9d 100644 --- a/docs/en/quick-start.mdx +++ b/docs/en/quick-start.mdx @@ -28,6 +28,9 @@ then downloads an appropriate ClickHouse binary: curl https://clickhouse.com/ | sh ``` + :::note + For Mac users: If you are getting errors that the developer of the binary cannot be verified, please see [here](https://clickhouse.com/docs/knowledgebase/fix-developer-verification-error-in-macos). + ::: ## 2. Start the server diff --git a/docusaurus.config.js b/docusaurus.config.js index dcdd09df1da..6274a7b6130 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -1,7 +1,8 @@ -const darkTheme = require('prism-react-renderer/themes/vsDark') -const path = require("path") -const math = require('remark-math'); -const katex = require('rehype-katex'); +import { themes } from 'prism-react-renderer'; +import math from 'remark-math'; +import katex from 'rehype-katex'; +import chHeader from './plugins/header.js'; +import fixLinks from './src/hooks/fixLinks.js'; // Helper function to skip over index.md files. function skipIndex(items) { @@ -39,9 +40,10 @@ const config = { // url: process.env.VERCEL_URL ? `https://${process.env.VERCEL_URL}` : 'https://bookish-disco-5997zvo.pages.github.io', baseUrl: '/docs/', baseUrlIssueBanner: true, - onBrokenLinks: 'throw', + onBrokenLinks: 'warn', onBrokenMarkdownLinks: 'throw', onDuplicateRoutes: 'throw', + // onBrokenAnchors: 'throw', favicon: 'img/docs_favicon.ico', organizationName: 'ClickHouse', trailingSlash: false, @@ -109,6 +111,7 @@ const config = { sidebarCollapsed: true, routeBasePath: '/', remarkPlugins: [math], + beforeDefaultRemarkPlugins: [fixLinks], rehypePlugins: [katex], }, blog: { @@ -198,7 +201,7 @@ const config = { { type: 'dropdown', label: 'Product', - hoverable: true, + hoverable: 'true', className: 'ch-menu', position: 'left', items: [ @@ -215,7 +218,7 @@ const config = { { type: 'dropdown', - hoverable: true, + hoverable: 'true', label: 'Resources', className: 'ch-menu', position: 'left', @@ -279,8 +282,8 @@ const config = { copyright: `© 2016–${new Date().getFullYear()} ClickHouse, Inc.`, }, prism: { - theme: darkTheme, - darkTheme: darkTheme, + theme: themes.darkTheme, + darkTheme: themes.darkTheme, additionalLanguages: ['java', 'cpp', 'rust'], magicComments: [ // Remember to extend the default highlight class name as well! @@ -309,7 +312,6 @@ const config = { plugins: [ 'docusaurus-plugin-sass', - 'remark-docusaurus-tabs', function (context, options) { return { name: 'docusaurus-plugin', @@ -2390,7 +2392,7 @@ const config = { ], }, ], - path.resolve(__dirname, 'plugins', 'header') + chHeader ], customFields: { galaxyApiEndpoint: process.env.NEXT_PUBLIC_GALAXY_API_ENDPOINT || 'http://localhost:3000', @@ -2548,7 +2550,7 @@ const config = { }, { type: 'dropdown', - hoverable: false, + hoverable: 'false', html: '\n' + '\n' + diff --git a/knowledgebase/Insert_select_settings_tuning.md b/knowledgebase/Insert_select_settings_tuning.md index 248caa52252..b77036d89a1 100644 --- a/knowledgebase/Insert_select_settings_tuning.md +++ b/knowledgebase/Insert_select_settings_tuning.md @@ -1,4 +1,6 @@ --- +title: TOO MANY PARTS error during an INSERT...SELECT +description: "Resolve the TOO_MANY_PARTS error in ClickHouse during an `INSERT...SELECT` by tuning expert-level settings for larger blocks and increasing partition thresholds." date: 2023-07-21 --- @@ -10,6 +12,8 @@ When executing a `INSERT...SELECT` statement, I am getting too many parts (TOO_M How can I solve this? + + ## Answer Below are some of the settings to tune to avoid this error, this is expert level tuning of ClickHouse and these values should be set only after understanding the specifications of the ClickHouse cloud service or on-prem cluster where these will be used, so do not take these values as "one size fits all". diff --git a/knowledgebase/ODBC-authentication-failed-error-using-PowerBI-CH-connector.md b/knowledgebase/ODBC-authentication-failed-error-using-PowerBI-CH-connector.md index 54f09640e4e..1849a3e2880 100644 --- a/knowledgebase/ODBC-authentication-failed-error-using-PowerBI-CH-connector.md +++ b/knowledgebase/ODBC-authentication-failed-error-using-PowerBI-CH-connector.md @@ -8,7 +8,12 @@ date: 2024-07-10 ### Question -When trying to connect from PowerBI to ClickHouse using the connector, you receive the following authentication error: +When trying to connect from PowerBI to ClickHouse using the connector, you receive a authentication error. + + + +This error usually looks like the following: + ``` We encountered an error while trying to connect. Details: "ODBC: ERROR [HY000] HTTP status code: 403 diff --git a/knowledgebase/about-quotas-and-query-complexity.md b/knowledgebase/about-quotas-and-query-complexity.md index cc6ed4ec4e5..e97ac4055aa 100644 --- a/knowledgebase/about-quotas-and-query-complexity.md +++ b/knowledgebase/about-quotas-and-query-complexity.md @@ -7,6 +7,8 @@ title: About Quotas and Query complexity [Quotas](https://clickhouse.com/docs/en/operations/quotas) and [query complexity](https://clickhouse.com/docs/en/operations/settings/query-complexity) are powerful ways to limit and restrict what users can do in ClickHouse. + + Quotas do apply restrictions within the context of a time interval, while query complexity applies regardless of time intervals. This KB article shows examples on how to apply these two different approaches. diff --git a/knowledgebase/add-column.md b/knowledgebase/add-column.md index 79a3033893e..a287d666cb8 100644 --- a/knowledgebase/add-column.md +++ b/knowledgebase/add-column.md @@ -5,6 +5,9 @@ date: 2024-12-18 --- In this guide, we'll learn how to add a column to an existing table. + + + We'll be using clickhouse-local: ```bash diff --git a/knowledgebase/alter-user-settings-exception.md b/knowledgebase/alter-user-settings-exception.md index 2a3423fae64..7cac5709923 100644 --- a/knowledgebase/alter-user-settings-exception.md +++ b/knowledgebase/alter-user-settings-exception.md @@ -1,10 +1,16 @@ --- +title: Alter User Settings Exception +description: Handing the an exception thrown when altering user settings date: 2023-08-26 --- # DB::Exception: Cannot update user `default` in users.xml because this storage is readonly. (ACCESS_STORAGE_READONLY) -When you try to alter a user's settings, you may encounter the above exception. Here are a few options to troubleshoot this error: +When you try to alter a user's settings, you may encounter the above exception. + + + +Here are a few options to troubleshoot this error: ## Edit users.xml directly diff --git a/knowledgebase/are_materialized_views_inserted_asynchronously.md b/knowledgebase/are_materialized_views_inserted_asynchronously.md index c393e08ae91..c4ed4cff316 100644 --- a/knowledgebase/are_materialized_views_inserted_asynchronously.md +++ b/knowledgebase/are_materialized_views_inserted_asynchronously.md @@ -1,4 +1,6 @@ --- +title: Are Materialized Views inserted synchronously? +description: Insert behavior of materialized views date: 2023-03-01 --- @@ -6,6 +8,8 @@ date: 2023-03-01 **Question:** When a source table has new rows inserted into it, those new rows are also sent to all of the materialized views of that source table. Are inserts into Materialized Views performed synchronously, meaning that once the insert is acknowledged successfully from the server to the client, it means that all Materialized Views have been fully updated and available for queries? + + **Answer:** 1. When an `INSERT` succeeds, the data is inserted both to the table and all materialized views. diff --git a/knowledgebase/async_vs_optimize_read_in_order.md b/knowledgebase/async_vs_optimize_read_in_order.md index 62d7b156b7f..5159c365c72 100644 --- a/knowledgebase/async_vs_optimize_read_in_order.md +++ b/knowledgebase/async_vs_optimize_read_in_order.md @@ -1,10 +1,15 @@ --- +title: Synchronous data reading description: "The new setting allow_asynchronous_read_from_io_pool_for_merge_tree allows the number of reading threads (streams) to be higher than the number of threads in the rest of the query execution pipeline." date: 2023-03-01 --- # Synchronous data reading +The new setting allow_asynchronous_read_from_io_pool_for_merge_tree allows the number of reading threads (streams) to be higher than the number of threads in the rest of the query execution pipeline. + + + Normally the [max_threads](https://clickhouse.com/docs/en/operations/settings/settings/#settings-max_threads) setting [controls](https://clickhouse.com/company/events/query-performance-introspection) the number of parallel reading threads and parallel query processing threads: ![Untitled scene](https://user-images.githubusercontent.com/97666923/212138072-5410b684-d00d-4218-93c5-6f49523928a5.png) diff --git a/knowledgebase/aws-privatelink-setup-for-clickpipes.md b/knowledgebase/aws-privatelink-setup-for-clickpipes.md index 587d0b5ab88..5a9d0cbe035 100644 --- a/knowledgebase/aws-privatelink-setup-for-clickpipes.md +++ b/knowledgebase/aws-privatelink-setup-for-clickpipes.md @@ -1,9 +1,15 @@ --- -Date: 2024-11-27 +title: AWS PrivateLink setup to expose private RDS for ClickPipes +description: Setup steps to expose a private RDS via AWS PrivateLink to ClickPipes. +date: 2024-11-27 --- # AWS PrivateLink setup to expose private RDS for ClickPipes +Setup steps to expose a private RDS via AWS PrivateLink to ClickPipes. + + + ## Requirements The VPC must be located in one of our ClickPipes regions: us-east-1, us-east-2 or eu-central-1. diff --git a/knowledgebase/backing_up_a_specific_partition.md b/knowledgebase/backing_up_a_specific_partition.md index bcc87d6dc7d..ed86c242937 100644 --- a/knowledgebase/backing_up_a_specific_partition.md +++ b/knowledgebase/backing_up_a_specific_partition.md @@ -1,5 +1,6 @@ --- title: Backing up a specific partition +description: How can I backup a specific partition in ClickHouse? date: 2024-02-14 --- @@ -7,6 +8,8 @@ date: 2024-02-14 How can I backup a specific partition in ClickHouse? + + ## Answer See the below example, this uses the S3(Minio) disk [configuration](https://github.com/ClickHouse/examples/blob/main/docker-compose-recipes/recipes/ch-and-minio-S3/README.md) listed in our [docker compose examples](https://github.com/ClickHouse/examples/blob/main/docker-compose-recipes/README.md) page. diff --git a/knowledgebase/billing.md b/knowledgebase/billing.md index 77d7256a489..4cbd59083b6 100644 --- a/knowledgebase/billing.md +++ b/knowledgebase/billing.md @@ -6,4 +6,8 @@ date: 2022-12-06 import Content from '@site/docs/en/cloud/manage/billing.md'; +Overview of billing information. + + + diff --git a/knowledgebase/calculate-pi-using-sql.md b/knowledgebase/calculate-pi-using-sql.md index fac7b9ae2d5..621633964a6 100644 --- a/knowledgebase/calculate-pi-using-sql.md +++ b/knowledgebase/calculate-pi-using-sql.md @@ -1,4 +1,6 @@ --- +title: Let's calculate pi using SQL +description: It's Pi Day! Let's calculate pi using ClickHouse SQL date: 2023-03-14 --- @@ -6,6 +8,9 @@ date: 2023-03-14 Happy Pi Day! We thought it would be fun to calculate pi using SQL queries in ClickHouse. Here is what we came up with so far... + + + 1. This one uses the ClickHouse `numbers_mt` table function to return 1B rows and only takes 40ms to compute the calculation: ```sql diff --git a/knowledgebase/calculate_ratio_of_zero_sparse_serialization.md b/knowledgebase/calculate_ratio_of_zero_sparse_serialization.md index 255a03597b0..de99328679e 100644 --- a/knowledgebase/calculate_ratio_of_zero_sparse_serialization.md +++ b/knowledgebase/calculate_ratio_of_zero_sparse_serialization.md @@ -1,4 +1,6 @@ --- +title: How to calculate the ratio of empty/zero values in every column in a table +description: Learn how to calculate the ratio of empty or zero values in every column of a ClickHouse table to optimize sparse column serialization. date: 2023-05-18 --- @@ -6,6 +8,8 @@ date: 2023-05-18 If a column is sparse (empty or contains mostly zeros), ClickHouse can encode it in a sparse format and automatically optimize calculations - the data does not require full decompression during queries. In fact, if you know how sparse a column is, you can define its ratio using the [`ratio_of_defaults_for_sparse_serialization` setting](https://clickhouse.com/docs/en/operations/settings/merge-tree-settings#ratio_of_defaults_for_sparse_serialization) to optimize serialization. + + This handy query can take a while, but it analyzes every row in your table and determines the ratio of values that are zero (or the default) in every column in the specified table: ```sql diff --git a/knowledgebase/cannot-append-data-to-parquet-format.md b/knowledgebase/cannot-append-data-to-parquet-format.md index 91175a66df9..6009073adc8 100644 --- a/knowledgebase/cannot-append-data-to-parquet-format.md +++ b/knowledgebase/cannot-append-data-to-parquet-format.md @@ -1,10 +1,21 @@ --- +title: Resolving "Cannot Append Data in Parquet Format" Error in ClickHouse +description: Understand and resolve the "Cannot append data in format Parquet to file" error in ClickHouse by enabling the setting to allow multiple files for non-appendable formats like Parquet. date: 2023-03-25 --- -# DB::Exception: Cannot append data in format Parquet to file, because this format doesn't support appends. (CANNOT_APPEND_TO_FILE) +# Resolving "Cannot Append Data in Parquet Format" Error in ClickHouse + +Are you getting the error "Cannot append data in format Parquet to file" error in ClickHouse? + + + +Typically the error is reported as: + +`DB::Exception: Cannot append data in format Parquet to file, because this format doesn't support appends. (CANNOT_APPEND_TO_FILE)` + +Suppose you create a table that uses the `File` table engine with the Parquet format. -Suppose you create a table that uses the `File` table engine with the Parquet format: ```sql CREATE TABLE parquet_test diff --git a/knowledgebase/certificate_verify_failed_error.md b/knowledgebase/certificate_verify_failed_error.md index 58f600e0adb..92911a274fc 100644 --- a/knowledgebase/certificate_verify_failed_error.md +++ b/knowledgebase/certificate_verify_failed_error.md @@ -1,8 +1,16 @@ --- +title: Resolving SSL Certificate Verify Error in ClickHouse +description: Learn how to resolve the SSL Exception `CERTIFICATE_VERIFY_FAILED` error date: 2023-05-02 --- -# DB::NetException: SSL Exception: error:1000007d:SSL routines:OPENSSL_internal:CERTIFICATE_VERIFY_FAILED +# Resolving SSL Certificate Verify Error in ClickHouse + +The error is typically reported as: + +`DB::NetException: SSL Exception: error:1000007d:SSL routines:OPENSSL_internal:CERTIFICATE_VERIFY_FAILED` + + ## Cause of the Error diff --git a/knowledgebase/change-billing-email.md b/knowledgebase/change-billing-email.md index 34b69926f2d..463ae199829 100644 --- a/knowledgebase/change-billing-email.md +++ b/knowledgebase/change-billing-email.md @@ -5,6 +5,8 @@ title: How do I change my Billing Contact in ClickHouse Cloud? # How do I change my Billing Contact in ClickHouse Cloud? + + ## Question How do I change my Billing Contact in ClickHouse Cloud? diff --git a/knowledgebase/change-the-prompt-in-clickhouse-client.md b/knowledgebase/change-the-prompt-in-clickhouse-client.md index 41b095604a0..e65f52c7688 100644 --- a/knowledgebase/change-the-prompt-in-clickhouse-client.md +++ b/knowledgebase/change-the-prompt-in-clickhouse-client.md @@ -6,6 +6,10 @@ date: 2023-11-16 # Change the prompt in `clickhouse client` +Want to change your ClickHouse Client prompt? Read on. + + + ## Background If you don't like how `clickhouse client` displays the prompt in your terminal window, it's possible to change it by creating a single XML file. This article explains how to change the prompt to whatever you want. diff --git a/knowledgebase/check-query-cache-in-use.md b/knowledgebase/check-query-cache-in-use.md index 213047cbc99..4bab4d4c4c0 100644 --- a/knowledgebase/check-query-cache-in-use.md +++ b/knowledgebase/check-query-cache-in-use.md @@ -1,14 +1,17 @@ --- +title: How to Verify Query Cache Usage in ClickHouse +description: Learn how to check if query cache is being utilized in ClickHouse using `clickhouse-client` trace logs or SQL commands. date: 2023-06-07 --- # How can I check that query cache is being used in my query? + + See this example using [clickhouse client](https://clickhouse.com/docs/en/interfaces/cli) and ClickHouse Cloud service. create a `query_cache_test` table - ## Using clickhouse client ```sql diff --git a/knowledgebase/check_query_processing_time_only.md b/knowledgebase/check_query_processing_time_only.md index 62e3a5e5e83..073204c7e8a 100644 --- a/knowledgebase/check_query_processing_time_only.md +++ b/knowledgebase/check_query_processing_time_only.md @@ -1,4 +1,6 @@ --- +title: How to Measure Query Processing Time Without Returning Rows +description: Learn how to use the `FORMAT Null` option in ClickHouse to measure query processing time without returning any rows to the client. date: 2023-06-07 --- @@ -8,6 +10,8 @@ date: 2023-06-07 My query is returning many rows but I'm only interested in the query processing time. How do I omit the query output and check for query processing time? + + ## Answer Append `FORMAT Null` to your query to configure the output format to `Null`. This prevents data from being transmitted to the client. diff --git a/knowledgebase/check_users_roles.md b/knowledgebase/check_users_roles.md index 4a6d9935af1..50caaad9505 100644 --- a/knowledgebase/check_users_roles.md +++ b/knowledgebase/check_users_roles.md @@ -1,6 +1,6 @@ --- -sidebar_position: 1 -title: How to check users assigned to roles and viceversa? +title: How to Check Users Assigned to Roles and Vice Versa +description: Learn how to query ClickHouse's `system.role_grants` to find users assigned to roles and roles assigned to specific users. date: 2023-09-07 --- @@ -8,6 +8,7 @@ date: 2023-09-07 How to check users assigned to roles and viceversa? + ### Answer diff --git a/knowledgebase/clickhouse_cloud_api_usage.md b/knowledgebase/clickhouse_cloud_api_usage.md index c8bc916570d..af42e6163dd 100644 --- a/knowledgebase/clickhouse_cloud_api_usage.md +++ b/knowledgebase/clickhouse_cloud_api_usage.md @@ -1,9 +1,13 @@ --- +title: Managing ClickHouse Cloud Service with API and cURL +description: Learn how to start, stop, and resume a ClickHouse Cloud service using API endpoints and cURL commands. date: 2023-06-07 --- # How to start, stop and resume a Cloud service using the ClickHouse API and cURL + + ## Question How can a ClickHouse Cloud service be started, stopped and resumed using API endpoints? diff --git a/knowledgebase/columnar-database.md b/knowledgebase/columnar-database.md index 23e71418ff4..26d89e9df11 100644 --- a/knowledgebase/columnar-database.md +++ b/knowledgebase/columnar-database.md @@ -8,6 +8,8 @@ date: 2021-09-01 A columnar database stores the data of each column independently. This allows reading data from disk only for those columns used in any given query. The cost is that operations that affect whole rows become proportionally more expensive. A columnar database is a synonym for a column-oriented database management system. ClickHouse is a typical example of such a system. + + ## Advantages of a columnar database Key columnar database advantages are: diff --git a/knowledgebase/common-rbac-queries.md b/knowledgebase/common-rbac-queries.md index 4dcc1e258e7..fb24a3bb5bc 100644 --- a/knowledgebase/common-rbac-queries.md +++ b/knowledgebase/common-rbac-queries.md @@ -6,6 +6,10 @@ date: 2023-09-28 # Common role-based access control (RBAC) queries +The following are common queries to help grant specific permissions to users. + + + ## How do I grant the same permissions as the current user to another user? ```sql diff --git a/knowledgebase/compare_resultsets.md b/knowledgebase/compare_resultsets.md index 9fa611798e2..ad6fcc9cc41 100644 --- a/knowledgebase/compare_resultsets.md +++ b/knowledgebase/compare_resultsets.md @@ -1,4 +1,6 @@ --- +title: How to Validate if Two Queries Return the Same Resultsets +description: Learn how to validate that two ClickHouse queries produce identical resultsets using hash functions and comparison techniques. date: 2023-05-04 --- @@ -8,6 +10,8 @@ date: 2023-05-04 How can I validate that two queries return the same resultsets? + + ## Answer You can use the below approach: diff --git a/knowledgebase/comparing-metrics-between-queries.md b/knowledgebase/comparing-metrics-between-queries.md index a27071c03ec..3d702fdcc41 100644 --- a/knowledgebase/comparing-metrics-between-queries.md +++ b/knowledgebase/comparing-metrics-between-queries.md @@ -9,6 +9,8 @@ keywords: [compare queries, compare metrics, query performance] To compare metrics between two queries, you must first obtain the `query_id` for both queries. + + You can then run the following query: ```sql diff --git a/knowledgebase/configure-a-user-setting.md b/knowledgebase/configure-a-user-setting.md index a6e4e9bdd11..1e48c514448 100644 --- a/knowledgebase/configure-a-user-setting.md +++ b/knowledgebase/configure-a-user-setting.md @@ -1,4 +1,6 @@ --- +title: How to Configure Settings for a User in ClickHouse +description: Learn how to define settings in ClickHouse for individual queries, client sessions, or specific users using `SET` and `ALTER USER` commands. date: 2023-03-01 --- @@ -6,6 +8,8 @@ date: 2023-03-01 There are several ways to define a setting for a user in ClickHouse, depending on the use case and how long you want the setting to be configured. Let's look at a few scenarios... + + ## Configure a setting for a single query A `SELECT` query can contain a `SETTINGS` clause where you can define any number of settings. The settings are only applied for that particular query. For example: diff --git a/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker.md b/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker.md index d68f98d7bda..f293702d0fd 100644 --- a/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker.md +++ b/knowledgebase/configure_cap_ipc_lock_and_cap_sys_nice_in_docker.md @@ -1,4 +1,6 @@ --- +title: Configuring CAP_IPC_LOCK and CAP_SYS_NICE Capabilities in Docker +description: Learn how to resolve Docker capability warnings for `CAP_IPC_LOCK` and `CAP_SYS_NICE` when running ClickHouse in a container. date: 2023-06-07 --- @@ -8,6 +10,8 @@ date: 2023-06-07 When running ClickHouse in Docker, Docker is complaining about the lack of `CAP_IPC_LOCK` and `CAP_SYS_NICE` capabilities in the system. How can I resolve it? + + Here is what the no `CAP_SYS_NICE` or `CAP_SYS_NICE` capability log messages look like: ```bash diff --git a/knowledgebase/connection_timeout_remote_remoteSecure.md b/knowledgebase/connection_timeout_remote_remoteSecure.md index d9fa8db1912..3effa9b188c 100644 --- a/knowledgebase/connection_timeout_remote_remoteSecure.md +++ b/knowledgebase/connection_timeout_remote_remoteSecure.md @@ -1,5 +1,6 @@ --- -description: "When using the `remote` or `remoteSecure` table functions on a node that is located more than 100ms (latency wise) away from the remote node, it is common to encounter the following timeout error." +title: Resolving Timeout Errors with `remote` and `remoteSecure` Table Functions +description: Learn how to fix timeout errors when using `remote` or `remoteSecure` table functions in ClickHouse by adjusting the connection timeout settings. date: 2023-03-20 --- @@ -8,7 +9,11 @@ date: 2023-03-20 **Problem** [`remote()` or `remoteSecure()`](https://clickhouse.com/docs/en/sql-reference/table-functions/remote/) table function allows the access of remote table from another ClickHouse node. -When using these functions on a node that is located more than 100ms (latency wise) away from the remote node, it is common to encounter the following timeout error. +When using these functions on a node that is located more than 100ms (latency wise) away from the remote node, it is common to encounter a timeout error. + + + +For example: ``` 4776d4bd8190 :) SELECT * FROM remoteSecure('HOSTNAME.us-east-2.aws.clickhouse.cloud', DATABASE, TABLE, 'USER', 'USER_PASSWORD') diff --git a/knowledgebase/dbms-naming.md b/knowledgebase/dbms-naming.md index adb64e91362..be83ebb4358 100644 --- a/knowledgebase/dbms-naming.md +++ b/knowledgebase/dbms-naming.md @@ -8,6 +8,8 @@ date: 2021-09-01 It’s a combination of "**Click**stream" and "Data ware**House**". It comes from the original use case at Yandex.Metrica, where ClickHouse was supposed to keep records of all clicks by people from all over the Internet, and it still does the job. You can read more about this use case on [ClickHouse history](https://clickhouse.com/docs/en/about-us/history) page. + + This two-part meaning has two consequences: - The only correct way to write Click**H**ouse is with capital H. diff --git a/knowledgebase/delete-old-data.md b/knowledgebase/delete-old-data.md index 32c50635fe5..f02c90d3a54 100644 --- a/knowledgebase/delete-old-data.md +++ b/knowledgebase/delete-old-data.md @@ -8,6 +8,8 @@ date: 2022-10-19 The short answer is "yes". ClickHouse has multiple mechanisms that allow freeing up disk space by removing old data. Each mechanism is aimed for different scenarios. + + ## TTL {#ttl} ClickHouse allows to automatically drop values when some condition happens. This condition is configured as an expression based on any columns, usually just static offset for any timestamp column. diff --git a/knowledgebase/dictionaries-consistent-state.md b/knowledgebase/dictionaries-consistent-state.md index 1b0d720df24..da2dbfcfb43 100644 --- a/knowledgebase/dictionaries-consistent-state.md +++ b/knowledgebase/dictionaries-consistent-state.md @@ -8,6 +8,8 @@ date: 2024-04-12 Dictionaries created in ClickHouse Cloud may experience inconsistency during the initial creation phase. This means that you may not see any data in the dictionary right after creation. However, after several retries, the creation query may land on different replicas, and data will be visible. + + This sometimes occurs because the dictionary was created before the part reached the server. As an example: ``` diff --git a/knowledgebase/dictionary_using_strings.md b/knowledgebase/dictionary_using_strings.md index 9ddb4ee1aaf..25fdafc6b74 100644 --- a/knowledgebase/dictionary_using_strings.md +++ b/knowledgebase/dictionary_using_strings.md @@ -1,13 +1,15 @@ --- -title: How to create a dictionary with String type keys and values +title: How to Create a ClickHouse Dictionary with String Keys and Values +description: Learn how to create a ClickHouse dictionary using string keys and values from a MergeTree table as the source, with examples of setup and usage. date: 2023-07-10 --- - ### Question How to create a ClickHouse dictionary using string keys and string values from a MergeTree table source + + ### Answer - Create the source table for the dictionary diff --git a/knowledgebase/difference_between_official_builds_and_3rd_party.md b/knowledgebase/difference_between_official_builds_and_3rd_party.md index 3551ea7fb1e..35bb4c118a1 100644 --- a/knowledgebase/difference_between_official_builds_and_3rd_party.md +++ b/knowledgebase/difference_between_official_builds_and_3rd_party.md @@ -1,4 +1,6 @@ --- +title: Differences Between Official and 3rd-Party ClickHouse Builds +description: Understand the key differences between official ClickHouse builds and 3rd-party builds, including updates, compatibility, and security considerations. date: 2023-06-26 --- @@ -8,6 +10,8 @@ date: 2023-06-26 I see other vendors providing their own builds of ClickHouse. What is the difference between official ClickHouse builds and these 3rd-party builds? + + ## Answer Here are some of the differences we have observed with other builds: diff --git a/knowledgebase/enabling-ssl-with-lets-encrypt.md b/knowledgebase/enabling-ssl-with-lets-encrypt.md index 5e5ef55424c..d74fd2e46e5 100644 --- a/knowledgebase/enabling-ssl-with-lets-encrypt.md +++ b/knowledgebase/enabling-ssl-with-lets-encrypt.md @@ -1,10 +1,13 @@ --- -title: How to enable SSL with Let's Encrypt on a single ClickHouse server +title: How to Enable SSL with Let's Encrypt on a Single ClickHouse Server +description: Learn how to set up SSL for a single ClickHouse server using Let's Encrypt, including certificate issuance, configuration, and validation. date: 2024-12-11 --- The following steps can be used to enable SSL for a single ClickHouse Server using [Let's Encrypt](https://letsencrypt.org/), a free, automated, and open Certificate Authority (CA) designed to make it easy for anyone to secure their websites with HTTPS. By automating the certificate issuance and renewal process, Let's Encrypt ensures websites remain secure without requiring manual intervention. + + :::note We assume ClickHouse has been installed at the standard package locations in the following guide. We use the domain `product-test-server.clickhouse-dev.com` for all examples. Substitute your domain accordingly. ::: diff --git a/knowledgebase/exception-too-many-parts.md b/knowledgebase/exception-too-many-parts.md index 92ac442ffa2..5c1e2009afd 100644 --- a/knowledgebase/exception-too-many-parts.md +++ b/knowledgebase/exception-too-many-parts.md @@ -1,10 +1,16 @@ --- +title: Resolving "Too Many Parts" Error in ClickHouse +description: Learn how to address the "Too many parts" error in ClickHouse by optimizing insert rates, configuring MergeTree settings, and managing partitions effectively. date: 2023-03-20 --- # DB::Exception: Too many parts (600). Merges are processing significantly slower than inserts -You reached the `parts_to_throw_insert` setting on a MergeTree table. You can monitor the number of active parts for a given table with: +You reached the `parts_to_throw_insert` setting on a MergeTree table. + + + +You can monitor the number of active parts for a given table with: ```sql select count(*) from system.parts where table = '' and active == 1 diff --git a/knowledgebase/exchangeStatementToSwitchTables.md b/knowledgebase/exchangeStatementToSwitchTables.md index 30cf7c72e32..ea32e60f5dc 100644 --- a/knowledgebase/exchangeStatementToSwitchTables.md +++ b/knowledgebase/exchangeStatementToSwitchTables.md @@ -10,6 +10,8 @@ date: 2024-11-21 How do I use the [`EXCHANGE`](/docs/en/sql-reference/statements/exchange) command to switch table names? + + ### Answer The `EXCHANGE` command is useful when you need to switch a current table with another table that is temporary where possibly Primary Keys or other settings were updated. diff --git a/knowledgebase/execute-system-queries-in-cloud.md b/knowledgebase/execute-system-queries-in-cloud.md index bd9c1f6e4ee..e93a532f779 100644 --- a/knowledgebase/execute-system-queries-in-cloud.md +++ b/knowledgebase/execute-system-queries-in-cloud.md @@ -1,4 +1,6 @@ --- +title: Execute SYSTEM Statements on All Nodes in ClickHouse Cloud +description: Learn how to use `ON CLUSTER` and `clusterAllReplicas` to execute SYSTEM statements and queries across all nodes in a ClickHouse Cloud service. date: 2023-03-01 --- @@ -6,6 +8,8 @@ date: 2023-03-01 In order to execute the same query on all nodes of a ClickHouse Cloud service, we can use [clusterAllReplicas](https://clickhouse.com/docs/en/sql-reference/table-functions/cluster/). + + For example, in order to get entries from a (node-local) system table from all nodes, you can use: ```sql SELECT ... FROM clusterAllReplicas(default, system.TABLE) ...; diff --git a/knowledgebase/file-export.md b/knowledgebase/file-export.md index 09eaa9144ff..ba303dcf1a1 100644 --- a/knowledgebase/file-export.md +++ b/knowledgebase/file-export.md @@ -1,7 +1,6 @@ --- -title: How do I export data from ClickHouse to a file? -toc_hidden: true -toc_priority: 10 +title: How to Export Data from ClickHouse to a File +description: Learn various methods to export data from ClickHouse, including `INTO OUTFILE`, the File table engine, and command-line redirection. date: 2023-03-22 --- @@ -11,6 +10,8 @@ date: 2023-03-22 Add an [INTO OUTFILE](https://clickhouse.com/docs/en/sql-reference/statements/select/into-outfile) clause to your query. + + For example: ``` sql diff --git a/knowledgebase/filtered-aggregates.md b/knowledgebase/filtered-aggregates.md index ace06700119..340174abeb2 100644 --- a/knowledgebase/filtered-aggregates.md +++ b/knowledgebase/filtered-aggregates.md @@ -1,11 +1,16 @@ --- +title: Using Filtered Aggregates in ClickHouse +description: Learn how to use filtered aggregates in ClickHouse with `-If` and `-Distinct` aggregate combinators to simplify query syntax and enhance analytics. date: 2023-03-01 --- # Filtered aggregates in ClickHouse +ClickHouse provides a simple and intuitive way to write _filtered aggregates_. -ClickHouse provides a simple and intuitive way to write _filtered aggregates_. For example, compare the standard SQL way to write filtered aggregates (which work fine in ClickHouse) with the shorthand syntax using the `-If` [aggregate function combinator](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/), which can be appended to any aggregate function: + + + For example, compare the standard SQL way to write filtered aggregates (which work fine in ClickHouse) with the shorthand syntax using the `-If` [aggregate function combinator](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/), which can be appended to any aggregate function: ```sql --standard SQL diff --git a/knowledgebase/find-expensive-queries.md b/knowledgebase/find-expensive-queries.md index 676a65e6370..2f22e5847a0 100644 --- a/knowledgebase/find-expensive-queries.md +++ b/knowledgebase/find-expensive-queries.md @@ -1,4 +1,6 @@ --- +title: How to Identify the Most Expensive Queries in ClickHouse +description: Learn how to use the `query_log` table in ClickHouse to identify the most memory and CPU-intensive queries across distributed nodes. date: 2023-03-26 --- @@ -9,6 +11,8 @@ The `query_log` table in the `system` database keeps track of all your queries, - how much memory the query consumed, and - how much CPU time was needed + + The following query returns the top 10 queries, where "top" means the queries that used the most memory: ```sql diff --git a/knowledgebase/finding_expensive_queries_by_memory_usage.md b/knowledgebase/finding_expensive_queries_by_memory_usage.md index de1408cc13e..75392a96385 100644 --- a/knowledgebase/finding_expensive_queries_by_memory_usage.md +++ b/knowledgebase/finding_expensive_queries_by_memory_usage.md @@ -1,10 +1,16 @@ --- +title: Identifying Expensive Queries by Memory Usage in ClickHouse +description: Learn how to use the `system.query_log` table to find the most memory-intensive queries in ClickHouse, with examples for clustered and standalone setups. date: 2023-06-07 --- # Expensive queries by memory usage -The following useful query shows which of your executed queries used the most memory. A couple of comments about this query: +The following useful query shows which of your executed queries used the most memory. + + + +A couple of comments about this query: - the results are computed from the past day (`now() - toIntervalDay(1))`) but you can easily modify the time interval - it assumes you have a cluster named `default`, which is the name of your cluster in [ClickHouse Cloud](https://clickhouse.cloud). Change `default` to the name of your cluster diff --git a/knowledgebase/fix-developer-verification-error-in-macos.md b/knowledgebase/fix-developer-verification-error-in-macos.md index b94b1e0fdb7..8c4e2ba9f1b 100644 --- a/knowledgebase/fix-developer-verification-error-in-macos.md +++ b/knowledgebase/fix-developer-verification-error-in-macos.md @@ -1,11 +1,16 @@ --- -title: "Fix the developer verification error in MacOS" +title: Fix the Developer Verification Error in MacOS +description: Learn how to resolve the MacOS developer verification error when running ClickHouse commands, using either System Settings or the terminal. date: 2024-01-09 --- # Fix the developer verification error in MacOS -If you install ClickHouse using `brew`, you may encounter an error from MacOS. By default, MacOS will not run applications or tools created by a developer who cannot be verified. When attempting to run any `clickhouse` command, you may see this error: +If you install ClickHouse using `brew`, you may encounter an error from MacOS. By default, MacOS will not run applications or tools created by a developer who cannot be verified. + + + +When attempting to run any `clickhouse` command, you may see this error: ![MacOS showing a developer verification error.](./images/fix-the-developer-verification-error-in-macos/dev-verification-error.png) diff --git a/knowledgebase/how-do-i-contribute-code-to-clickhouse.md b/knowledgebase/how-do-i-contribute-code-to-clickhouse.md index 208d10e8411..6b45da9a17e 100644 --- a/knowledgebase/how-do-i-contribute-code-to-clickhouse.md +++ b/knowledgebase/how-do-i-contribute-code-to-clickhouse.md @@ -8,6 +8,8 @@ date: 2021-09-01 ClickHouse is an open-source project [developed on GitHub](https://github.com/ClickHouse/ClickHouse). + + As customary, contribution instructions are published in [CONTRIBUTING](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md) file in the root of the source code repository. If you want to suggest a substantial change to ClickHouse, consider [opening a GitHub issue](https://github.com/ClickHouse/ClickHouse/issues/new/choose) explaining what you want to do, to discuss it with maintainers and community first. [Examples of such RFC issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc). diff --git a/knowledgebase/how-to-check-my-clickhouse-cloud-sevice-state.md b/knowledgebase/how-to-check-my-clickhouse-cloud-sevice-state.md index cf2b8dbf6f3..c23bf9d980c 100644 --- a/knowledgebase/how-to-check-my-clickhouse-cloud-sevice-state.md +++ b/knowledgebase/how-to-check-my-clickhouse-cloud-sevice-state.md @@ -1,5 +1,6 @@ --- -title: How to check my ClickHouse Cloud Service state +title: How to Check Your ClickHouse Cloud Service State +description: Learn how to use the ClickHouse Cloud API to check if your service is stopped, idle, or running without waking it up. date: 2023-11-16 --- @@ -7,6 +8,8 @@ date: 2023-11-16 How do I check my ClickHouse Cloud Service state? I want to check if the Service is stopped, idle, or running, but I don't want to wake the Service up in doing so. + + ## Answer The [ClickHouse Cloud API](/docs/en/cloud/manage/api/api-overview) is great for checking the status of a cloud service. You need to create an API Key in your service before you can use the Cloud API. You can do this in ClickHouse Cloud [clickhouse.cloud](https://clickhouse.cloud): diff --git a/knowledgebase/how-to-connect-to-ch-cloud-using-ssh-keys.md b/knowledgebase/how-to-connect-to-ch-cloud-using-ssh-keys.md index 505bea1fadd..925f8b3b782 100644 --- a/knowledgebase/how-to-connect-to-ch-cloud-using-ssh-keys.md +++ b/knowledgebase/how-to-connect-to-ch-cloud-using-ssh-keys.md @@ -10,6 +10,8 @@ date: 2024-07-25 How can I connect to ClickHouse using SSH Key Authentication? + + :::note We use ClickHouse Cloud as an example here, but this example should also work on oss ClickHouse. ::: diff --git a/knowledgebase/how-to-create-table-to-query-multiple-remote-clusters.md b/knowledgebase/how-to-create-table-to-query-multiple-remote-clusters.md index c8842b07310..76e0cc9217b 100644 --- a/knowledgebase/how-to-create-table-to-query-multiple-remote-clusters.md +++ b/knowledgebase/how-to-create-table-to-query-multiple-remote-clusters.md @@ -9,6 +9,8 @@ date: 2024-07-03 ### Question How do I create a table that can query other clusters or instances? + + ### Answer Below is a simple example to test functionality. diff --git a/knowledgebase/how-to-filter-a-clickhouse-table-by-an-array-column.md b/knowledgebase/how-to-filter-a-clickhouse-table-by-an-array-column.md index cf500886d46..60b9416f7ba 100644 --- a/knowledgebase/how-to-filter-a-clickhouse-table-by-an-array-column.md +++ b/knowledgebase/how-to-filter-a-clickhouse-table-by-an-array-column.md @@ -8,6 +8,8 @@ date: 2024-12-17 Filtering a ClickHouse table by an array-column is a common task and the product offers a lot of [functions](/docs/en/sql-reference/functions/array-functions) to work with array-columns. + + In this article, we're going to focus on filtering a table by an array-column, but the video below covers a lot of other array-related functions: diff --git a/knowledgebase/how-to-increase-thread-pool-size.md b/knowledgebase/how-to-increase-thread-pool-size.md index c22f436a7b7..3c02b365ed0 100644 --- a/knowledgebase/how-to-increase-thread-pool-size.md +++ b/knowledgebase/how-to-increase-thread-pool-size.md @@ -1,4 +1,6 @@ --- +title: How to Increase the Number of Threads in ClickHouse +description: Learn how to configure the Global Thread pool in ClickHouse by adjusting settings like `max_thread_pool_size`, `thread_pool_queue_size`, and `max_thread_pool_free_size`. date: 2023-03-01 --- @@ -6,6 +8,8 @@ date: 2023-03-01 ClickHouse uses threads from the **Global Thread pool** to process queries and also perform background operations like merges and mutations. If there is no idle thread to process a query, then a new thread is created in the pool. + + The maximum size of the global thread pool is determined by the `max_thread_pool_size` setting, which defaults to 10,000. You can modify this value in your config - here we set it to 20,000: ```xml diff --git a/knowledgebase/how-to-insert-all-rows-from-another-table.md b/knowledgebase/how-to-insert-all-rows-from-another-table.md index 7b359f2a9d5..4076f60b437 100644 --- a/knowledgebase/how-to-insert-all-rows-from-another-table.md +++ b/knowledgebase/how-to-insert-all-rows-from-another-table.md @@ -10,6 +10,8 @@ Sometimes you need to reingest all the data from one table to another. For example, you might want to reingest data from a staging table to a production table. This article shows how to do this using the `INSERT INTO` statement. + + ## Example Below is a simple example on how it works and how to test: diff --git a/knowledgebase/how-to-set-up-ch-on-docker-odbc-connect-mssql.md b/knowledgebase/how-to-set-up-ch-on-docker-odbc-connect-mssql.md index 0feae917fef..15720631319 100644 --- a/knowledgebase/how-to-set-up-ch-on-docker-odbc-connect-mssql.md +++ b/knowledgebase/how-to-set-up-ch-on-docker-odbc-connect-mssql.md @@ -10,6 +10,8 @@ date: 2024-05-29 How do I set up ClickHouse with a Docker image to connect to Microsoft SQL Server? + + ### Answer ***Notes on this example*** diff --git a/knowledgebase/how_to_display_queries_using_MV.md b/knowledgebase/how_to_display_queries_using_MV.md index 0a8b74f1745..9ce4f3263d0 100644 --- a/knowledgebase/how_to_display_queries_using_MV.md +++ b/knowledgebase/how_to_display_queries_using_MV.md @@ -1,4 +1,6 @@ --- +title: How to Identify Queries Using Materialized Views in ClickHouse +description: Learn how to query ClickHouse logs to identify all queries involving Materialized Views within a specified time range. date: 2023-03-24 --- @@ -6,6 +8,8 @@ date: 2023-03-24 **Question:** How do I show all queries involving materialized views in the last 60m? + + **Answer:** This query will display all queries directed towards Materialized Views considering that: diff --git a/knowledgebase/how_to_use_parametrised_views.md b/knowledgebase/how_to_use_parametrised_views.md index f6cc7489e0f..9441a9b903d 100644 --- a/knowledgebase/how_to_use_parametrised_views.md +++ b/knowledgebase/how_to_use_parametrised_views.md @@ -1,4 +1,6 @@ --- +title: How to Use Parameterized Views in ClickHouse +description: Learn how to create and query parameterized views in ClickHouse for dynamic data slicing based on query-time parameters. date: 2023-08-01 --- @@ -6,6 +8,8 @@ date: 2023-08-01 Parameterized views can be handy to slice and dice data on the fly based on some parameters that can be fed at query execution time. + + See this basic example: 1) create a table diff --git a/knowledgebase/ignoring-incorrect-settings.md b/knowledgebase/ignoring-incorrect-settings.md index 71c74f47f38..4a93d59fcd6 100644 --- a/knowledgebase/ignoring-incorrect-settings.md +++ b/knowledgebase/ignoring-incorrect-settings.md @@ -1,10 +1,14 @@ --- +title: How to Ignore Incorrect Settings in ClickHouse +description: Learn how to use the `skip_check_for_incorrect_settings` option to allow ClickHouse to start even when user-level settings are specified incorrectly. date: 2023-03-01 --- # Ignoring incorrect settings -When a user-level setting is specified in the wrong place, the server won't start and an exception message is sent to the log. However, you can tell ClickHouse to ignore the incorrect setting using the `skip_check_for_incorrect_settings` setting: +When a user-level setting is specified in the wrong place, the server won't start and an exception message is sent to the log. However, you can tell ClickHouse to ignore the incorrect setting using the `skip_check_for_incorrect_settings` setting. + + Add the following to `config.xml`: diff --git a/knowledgebase/importing-geojason-with-nested-object-array.md b/knowledgebase/importing-geojason-with-nested-object-array.md index a7978038f91..1c9635031f5 100644 --- a/knowledgebase/importing-geojason-with-nested-object-array.md +++ b/knowledgebase/importing-geojason-with-nested-object-array.md @@ -10,6 +10,8 @@ date: 2024-12-18 How do I import GeoJSON with a nested object array? + + ### Answer For this tutorial, we will use open data publicly available [here](https://opendata.esri.es/datasets/ComunidadSIG::municipios-ign/explore?location=39.536006%2C-0.303882%2C6.57). A copy can be found [here](https://datasets-documentation.s3.eu-west-3.amazonaws.com/geoJSON/Municipios.geojson). diff --git a/knowledgebase/importing_and_working_with_JSON_array_objects.md b/knowledgebase/importing_and_working_with_JSON_array_objects.md index 6c500bb59a6..242176f7e38 100644 --- a/knowledgebase/importing_and_working_with_JSON_array_objects.md +++ b/knowledgebase/importing_and_working_with_JSON_array_objects.md @@ -1,4 +1,6 @@ --- +title: Importing and Querying JSON Array Objects in ClickHouse +description: Learn how to import JSON array objects into ClickHouse and perform advanced queries using JSON functions and array operations. date: 2023-03-24 --- @@ -6,6 +8,8 @@ date: 2023-03-24 **Question:** How do I import JSON arrays and how can I query the inner objects? + + **Answer:** Dump this 1 line JSON array to `sample.json` diff --git a/knowledgebase/improve-map-performance.md b/knowledgebase/improve-map-performance.md index 685a7a26709..f2a7050efb9 100644 --- a/knowledgebase/improve-map-performance.md +++ b/knowledgebase/improve-map-performance.md @@ -1,6 +1,6 @@ --- -sidebar_position: 1 -description: "Map lookups such as `a['key']' works with linear complexity (mentioned [here](https://clickhouse.com/docs/en/sql-reference/data-types/map)) and can be inefficient." +title: Improving Map Lookup Performance in ClickHouse +description: Learn how to optimize Map column lookups in ClickHouse for better query performance by materializing specific keys as standalone columns. date: 2022-10-30 --- @@ -10,6 +10,8 @@ date: 2022-10-30 Map lookup such as `a['key']` works with linear complexity (mentioned [here](https://clickhouse.com/docs/en/sql-reference/data-types/map)) and can be inefficient. This is because selecting a value with a specific key from a table would require iterating through all keys (~M) across all rows (N) in the Map column, resulting in ~MxN lookups. + + A lookup using Map can be 10x slower than a String column. The experiment below also shows ~10x slowdown for cold query, and difference in multiple magnitudes of data processed (7.21 MB vs 5.65 GB). ```sql diff --git a/knowledgebase/ingest-failures-23-9-release.md b/knowledgebase/ingest-failures-23-9-release.md index 41a069de3af..ff65312e3bd 100644 --- a/knowledgebase/ingest-failures-23-9-release.md +++ b/knowledgebase/ingest-failures-23-9-release.md @@ -1,11 +1,13 @@ --- -sidebar_position: 1 -title: Possible ingest failures after 23.9 release +title: Resolving Ingest Failures After ClickHouse 23.9 Release +description: "Learn how to resolve ingest failures caused by stricter grant checking introduced in ClickHouse 23.9 for tables using `async_inserts`. Update grants to fix errors." date: 2022-11-17 --- The root cause of these permissions errors has been identified and is due to more strict checking of grants in the new release for [`async_inserts`](/en/optimize/asynchronous-inserts). + + To fix this issue, grants must be updated for your service to work. Check the error message indicating the missing grants and add those needed grants manually. The required additional grants for the tables using `async_inserts` will either be the `SELECT` or `dictGet` grant. ## Add necessary GRANTs to affected tables and dictionaries diff --git a/knowledgebase/ingest-parquet-files-in-s3.md b/knowledgebase/ingest-parquet-files-in-s3.md index 326fb840801..ca282830241 100644 --- a/knowledgebase/ingest-parquet-files-in-s3.md +++ b/knowledgebase/ingest-parquet-files-in-s3.md @@ -1,4 +1,6 @@ --- +title: How to ingest Parquet files from an S3 bucket +description: "Learn the basics of using the S3 table engine in ClickHouse to ingest and query Parquet files from an S3 bucket, including setup, access permissions, and data import examples." date: 2023-03-22 --- @@ -6,6 +8,8 @@ date: 2023-03-22 Below are some basics of using the S3 table engine to read parquet files. + + - create access and secret keys for an IAM service user. normal login users usually don't work since they may have been configured with an MFA policy. diff --git a/knowledgebase/install-clickhouse-windows10.md b/knowledgebase/install-clickhouse-windows10.md index 0ae976d7aa3..71f321fab98 100644 --- a/knowledgebase/install-clickhouse-windows10.md +++ b/knowledgebase/install-clickhouse-windows10.md @@ -1,10 +1,14 @@ --- +title: How to Install ClickHouse on Windows 10 +description: "Learn how to install and test ClickHouse on Windows 10 using WSL 2. Includes setup, troubleshooting, and running a test environment." date: 2023-04-24 --- # How to Install ClickHouse on Windows 10 -How to install and test ClickHouse on Microsoft Windows +How to install and test ClickHouse on Microsoft Windows. + + When ClickHouse installing on Windows 10 you may receive errors when inserting data, for example: ``` diff --git a/knowledgebase/json-import.md b/knowledgebase/json-import.md index c545dacc9f7..afa140d146f 100644 --- a/knowledgebase/json-import.md +++ b/knowledgebase/json-import.md @@ -8,6 +8,8 @@ date: 2023-03-20 ClickHouse supports a wide range of [data formats for input and output](https://clickhouse.com/docs/en/interfaces/formats/). There are multiple JSON variations among them, but the most commonly used for data ingestion is [JSONEachRow](https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow). It expects one JSON object per row, each object separated by a newline. + + ## Examples {#examples} Using [HTTP interface](https://clickhouse.com/docs/en/interfaces/http/): diff --git a/knowledgebase/json_extract_example.md b/knowledgebase/json_extract_example.md index 78c3abf5bfd..24088e33f5b 100644 --- a/knowledgebase/json_extract_example.md +++ b/knowledgebase/json_extract_example.md @@ -8,6 +8,8 @@ date: 2021-09-01 This is just a short example that illustrates the use of [JSONExtract](https://clickhouse.com/docs/en/sql-reference/functions/json-functions) functions. + + Create a table: ```sql diff --git a/knowledgebase/json_simple_example.md b/knowledgebase/json_simple_example.md index 5f445a9aec0..6fd93fb923f 100644 --- a/knowledgebase/json_simple_example.md +++ b/knowledgebase/json_simple_example.md @@ -11,6 +11,8 @@ date: 2024-05-20 How do I work with JSON message using a source or landing table to extract with a Materialized View? How do I work with JSON without the experimental JSON Object? + + ### Answer A common pattern to work with JSON data is to send the data to a landing table and use JSONExtract functions to pull the data onto a new table using a Materialized View trigger. diff --git a/knowledgebase/kafka-clickhouse-json.md b/knowledgebase/kafka-clickhouse-json.md index e7abb5deab3..245c1c32e76 100644 --- a/knowledgebase/kafka-clickhouse-json.md +++ b/knowledgebase/kafka-clickhouse-json.md @@ -1,6 +1,7 @@ --- +title: Kafka and the JSON Data Type +description: "Learn how to load JSON messages from Apache Kafka directly into a single JSON column in ClickHouse using the Kafka table engine and JSON data type." date: 2024-11-06 -title: Kafka and the JSON data type --- # Kafka and the JSON data type @@ -8,6 +9,8 @@ title: Kafka and the JSON data type With the introduction of the new [`JSON`](/docs/en/sql-reference/data-types/newjson) data type, ClickHouse is now a good choice of [database for doing JSON analytics](https://clickhouse.com/engineering-resources/json-database). In this guide, we're going to learn how to load JSON messages from Apache Kafka directly into a single `JSON` column in ClickHouse. + + ## Setup Kafka Let's start by running a Kafka broker on our machine. We're also going to map port 9092 to port 9092 on our host operating system so that it's easier to interact with Kafka: diff --git a/knowledgebase/kafka-to-clickhouse-setup.md b/knowledgebase/kafka-to-clickhouse-setup.md index 9630ea94742..f2a6ab83a5a 100644 --- a/knowledgebase/kafka-to-clickhouse-setup.md +++ b/knowledgebase/kafka-to-clickhouse-setup.md @@ -1,16 +1,18 @@ --- +title: How to Ingest Data from Kafka into ClickHouse +description: "Learn how to ingest data from a Kafka topic into ClickHouse using the Kafka table engine, materialized views, and MergeTree tables." date: 2024-04-27 --- # How to ingest data from Kafka into ClickHouse - **Overview:** This article walks through the process of sending data from a Kafka topic to a ClickHouse table. We’ll use the Wiki recent changes feed, which provides a [stream of events](https://stream.wikimedia.org/v2/stream/recentchange) that represent changes made to various Wikimedia properties. The steps include: 1. How to setup Kafka on Ubuntu 2. Ingest a stream of data into a Kakfa topic 3. Create a ClickHouse table that subscribes to the topic + # 1. Setup Kafka on Ubuntu diff --git a/knowledgebase/key-value.md b/knowledgebase/key-value.md index f6bae747b69..9d5dfcff6b0 100644 --- a/knowledgebase/key-value.md +++ b/knowledgebase/key-value.md @@ -8,6 +8,8 @@ date: 2021-09-01 The short answer is **"no"**. The key-value workload is among top positions in the list of cases when **NOT** to use ClickHouse. It’s an [OLAP](https://clickhouse.com/docs/en/faq/general/olap) system after all, while there are many excellent key-value storage systems out there. + + However, there might be situations where it still makes sense to use ClickHouse for key-value-like queries. Usually, it’s some low-budget products where the main workload is analytical in nature and fits ClickHouse well, but there’s also some secondary process that needs a key-value pattern with not so high request throughput and without strict latency requirements. If you had an unlimited budget, you would have installed a secondary key-value database for this secondary workload, but in reality, there’s an additional cost of maintaining one more storage system (monitoring, backups, etc.) which might be desirable to avoid. If you decide to go against recommendations and run some key-value-like queries against ClickHouse, here are some tips: diff --git a/knowledgebase/llvm-clang-up-to-date.md b/knowledgebase/llvm-clang-up-to-date.md index 79aacad1fa1..b6f06a89fac 100644 --- a/knowledgebase/llvm-clang-up-to-date.md +++ b/knowledgebase/llvm-clang-up-to-date.md @@ -9,6 +9,8 @@ keywords: [contributing, llvm, clang] In order to build and contribute to ClickHouse, you must use [LLVM](https://llvm.org/) and [Clang](https://clang.llvm.org/). + + These are the commands to build the latest version of LLVM and Clang on Linux: ```bash diff --git a/knowledgebase/mapping-of-system-metrics-to-prometheus-metrics.md b/knowledgebase/mapping-of-system-metrics-to-prometheus-metrics.md index cf50c057758..8c4446004c6 100644 --- a/knowledgebase/mapping-of-system-metrics-to-prometheus-metrics.md +++ b/knowledgebase/mapping-of-system-metrics-to-prometheus-metrics.md @@ -9,6 +9,8 @@ date: 2024-07-23 The table below provides the mapping for the metrics used in `system.dashboards` to Prometheus metrics in `system.custom_metrics`. This is useful for customers who want to monitor for the same metrics found in `system.dashboards`. + + ### Mapping table for metrics in system.dashboards to Prometheus metrics in system.custom_metrics Dashboard | Title | Prometheus Metric Name (system.custom_metrics) diff --git a/knowledgebase/mapreduce.md b/knowledgebase/mapreduce.md index db486837c42..9a6a5c7a330 100644 --- a/knowledgebase/mapreduce.md +++ b/knowledgebase/mapreduce.md @@ -8,6 +8,8 @@ date: 2021-09-01 We can refer to systems like MapReduce as distributed computing systems in which the reduce operation is based on distributed sorting. The most common open-source solution in this class is [Apache Hadoop](http://hadoop.apache.org). + + These systems aren’t appropriate for online queries due to their high latency. In other words, they can’t be used as the back-end for a web interface. These types of systems aren’t useful for real-time data updates. Distributed sorting isn’t the best way to perform reduce operations if the result of the operation and all the intermediate results (if there are any) are located in the RAM of a single server, which is usually the case for online queries. In such a case, a hash table is an optimal way to perform reduce operations. A common approach to optimizing map-reduce tasks is pre-aggregation (partial reduce) using a hash table in RAM. The user performs this optimization manually. Distributed sorting is one of the main causes of reduced performance when running simple map-reduce tasks. Most MapReduce implementations allow you to execute arbitrary code on a cluster. But a declarative query language is better suited to OLAP to run experiments quickly. For example, Hadoop has Hive and Pig. Also consider Cloudera Impala or Shark (outdated) for Spark, as well as Spark SQL, Presto, and Apache Drill. Performance when running such tasks is highly sub-optimal compared to specialized systems, but relatively high latency makes it unrealistic to use these systems as the backend for a web interface. diff --git a/knowledgebase/marketplace.md b/knowledgebase/marketplace.md index 22f215f5119..89d44eaaee4 100644 --- a/knowledgebase/marketplace.md +++ b/knowledgebase/marketplace.md @@ -1,8 +1,13 @@ --- title: Marketplace +description: "Learn how to use CSP marketplaces to subscribe to ClickHouse Cloud." date: 2022-12-06 --- import Content from '@site/docs/en/cloud/manage/billing/marketplace/index.md'; +Details on using CSP marketplaces to subscribe to ClickHouse Cloud. + + + diff --git a/knowledgebase/maximum_number_of_tables_and_databases.md b/knowledgebase/maximum_number_of_tables_and_databases.md index ad5230b2ba7..e4ecc93f361 100644 --- a/knowledgebase/maximum_number_of_tables_and_databases.md +++ b/knowledgebase/maximum_number_of_tables_and_databases.md @@ -1,9 +1,15 @@ --- +title: Recommended Maximum Databases, Tables, Partitions, and Parts in ClickHouse +description: "Learn the recommended maximum limits for databases, tables, partitions, and parts in a ClickHouse cluster to ensure optimal performance." date: 2021-09-01 --- # How many maximum databases, tables, partitions, or parts are recommended in a ClickHouse cluster? -We recommend having a maximum of 1000 databases and 5000 tables, 50000 partitions, and 100000 parts across all databases for a service. Databases in ClickHouse are more of a _namespace_ and have no performance impact; 1000 databases is a loose guideline. However, the number of tables would affect the service startup time, so we recommend limiting the number of tables or partitions. ClickHouse provides a warning if these thresholds are hit. +We recommend having a maximum of 1000 databases and 5000 tables, 50000 partitions, and 100000 parts across all databases for a service. + + + +Databases in ClickHouse are more of a _namespace_ and have no performance impact; 1000 databases is a loose guideline. However, the number of tables would affect the service startup time, so we recommend limiting the number of tables or partitions. ClickHouse provides a warning if these thresholds are hit. These limits are also applicable to ClickHouse Cloud. diff --git a/knowledgebase/multi-region-replication.md b/knowledgebase/multi-region-replication.md index ebe67fff796..efb1ad93830 100644 --- a/knowledgebase/multi-region-replication.md +++ b/knowledgebase/multi-region-replication.md @@ -8,6 +8,8 @@ date: 2022-04-22 The short answer is "yes". However, we recommend keeping latency between all regions/datacenters in two-digit range, otherwise write performance will suffer as it goes through distributed consensus protocol. For example, replication between US coasts will likely work fine, but between the US and Europe won't. + + Configuration-wise there's no difference compared to single-region replication, simply use hosts that are located in different locations for replicas. For more information, see [full article on data replication](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication). diff --git a/knowledgebase/mysql-to-parquet-csv-json.md b/knowledgebase/mysql-to-parquet-csv-json.md index 750fe255443..c7c4cc9961c 100644 --- a/knowledgebase/mysql-to-parquet-csv-json.md +++ b/knowledgebase/mysql-to-parquet-csv-json.md @@ -1,4 +1,6 @@ --- +title: Export MySQL Data to Parquet, CSV, or JSON Using ClickHouse +description: "Learn how to use the `clickhouse-local` tool to export MySQL data into formats like Parquet, CSV, or JSON quickly and efficiently." date: 2023-03-21 --- @@ -9,6 +11,8 @@ The `clickhouse-local` tool makes it quick and easy to read data from MySQL and - Use the [`mysql` table function](https://clickhouse.com/docs/en/sql-reference/table-functions/mysql) to read the data - Use the `INTO OUTFILE _filename_ FORMAT` clause and specify the desired output format + + The `clickhouse-local` tool is a part of the ClickHouse binary. Download it using the following: ```bash diff --git a/knowledgebase/node-js-example.md b/knowledgebase/node-js-example.md index 67c0acd7dfe..f9e12c92bea 100644 --- a/knowledgebase/node-js-example.md +++ b/knowledgebase/node-js-example.md @@ -1,11 +1,14 @@ --- +title: A Simple NodeJS Example Using @clickhouse/client +description: "Learn how to use @clickhouse/client in a Node.js application to interact with ClickHouse and perform queries." date: 2023-11-22 -title: A simple nodeJS example using @clickhouse/client --- # A basic NodeJS example using @clickhouse/client -Here is a basic code snippet file`main.ts`. +Here is a basic code snippet file `main.ts`. + + `Package.json` (place it under `./`): diff --git a/knowledgebase/olap.md b/knowledgebase/olap.md index 71caf4014a6..6744713253f 100644 --- a/knowledgebase/olap.md +++ b/knowledgebase/olap.md @@ -6,7 +6,11 @@ date: 2021-09-01 # What Is OLAP? {#what-is-olap} -[OLAP](https://en.wikipedia.org/wiki/Online_analytical_processing) stands for Online Analytical Processing. It is a broad term that can be looked at from two perspectives: technical and business. But at the very high level, you can just read these words backward: +[OLAP](https://en.wikipedia.org/wiki/Online_analytical_processing) stands for Online Analytical Processing. It is a broad term that can be looked at from two perspectives: technical and business. + + + +But at the very high level, you can just read these words backward: Processing : Some source data is processed… diff --git a/knowledgebase/oracle-odbc.md b/knowledgebase/oracle-odbc.md index 80156a803bf..dd2354b5ae2 100644 --- a/knowledgebase/oracle-odbc.md +++ b/knowledgebase/oracle-odbc.md @@ -6,7 +6,11 @@ date: 2022-04-22 # What If I Have a Problem with Encodings When Using Oracle Via ODBC? {#oracle-odbc-encodings} -If you use Oracle as a source of ClickHouse external dictionaries via Oracle ODBC driver, you need to set the correct value for the `NLS_LANG` environment variable in `/etc/default/clickhouse`. For more information, see the [Oracle NLS_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). +If you use Oracle as a source of ClickHouse external dictionaries via Oracle ODBC driver, you need to set the correct value for the `NLS_LANG` environment variable in `/etc/default/clickhouse`. + + + +For more information, see the [Oracle NLS_LANG FAQ](https://www.oracle.com/technetwork/products/globalization/nls-lang-099431.html). **Example** diff --git a/knowledgebase/outputSendLogsLevelTracesToFile.md b/knowledgebase/outputSendLogsLevelTracesToFile.md index ad13a40d809..f186653f7c7 100644 --- a/knowledgebase/outputSendLogsLevelTracesToFile.md +++ b/knowledgebase/outputSendLogsLevelTracesToFile.md @@ -10,6 +10,8 @@ date: 2024-11-21 How do I output the send_logs_level output to a file using the ClickHouse Client for multiple statements and multiple lines? + + ### Answer - Create a SQL file with the statements, for example, `send_logs_level_example.sql`: diff --git a/knowledgebase/parquet-to-csv-json.md b/knowledgebase/parquet-to-csv-json.md index 03dddbf166e..ac9c465192a 100644 --- a/knowledgebase/parquet-to-csv-json.md +++ b/knowledgebase/parquet-to-csv-json.md @@ -1,4 +1,6 @@ --- +title: Converting Files from Parquet to CSV or JSON +description: "Learn how to use ClickHouse's `clickhouse-local` tool to easily convert Parquet files to CSV or JSON formats." date: 2023-03-22 --- @@ -6,6 +8,8 @@ date: 2023-03-22 You can use `clickhouse-local` to convert files between any of the [input and output formats](https://clickhouse.com/docs/en/interfaces/formats) that ClickHouse supports (which is over 70 different formats!). In this article, we are convert a Parquet file in S3 into a CSV and JSON file. + + Let's start at the beginning. ClickHouse has a collection of [table functions](https://clickhouse.com/docs/en/sql-reference/table-functions/) that read from files, databases and other resoures and converts the data to a table. To demonstrate, suppose we have a Parquet file in S3. We will use the `s3` table function to read it (ClickHouse knows it's a Parquet file based on the filename). But first, let's download the `clickhouse` binary: diff --git a/knowledgebase/part_intersects_previous_part.md b/knowledgebase/part_intersects_previous_part.md index a3f3c2de3de..6901e07e615 100644 --- a/knowledgebase/part_intersects_previous_part.md +++ b/knowledgebase/part_intersects_previous_part.md @@ -1,10 +1,16 @@ --- +title: "DB::Exception: Part XXXXX intersects previous part YYYYY. It is a bug or a result of manual intervention in the ZooKeeper data." date: 2023-05-02 +description: "This article explains how to resolve the DB::Exception error related to intersecting parts in ClickHouse, often caused by a race condition or manual intervention in the ZooKeeper data." --- # DB::Exception: Part XXXXX intersects previous part YYYYY. It is a bug or a result of manual intervention in the ZooKeeper data. -When this error occurs, a table shows as readonly and the error states intersecting parts. You can see the error in the logs or by +When this error occurs, a table shows as readonly and the error states intersecting parts. + + + +You can see the error in the logs or by ```sql SELECT * diff --git a/knowledgebase/pivot.md b/knowledgebase/pivot.md index 18f47864817..68381ac83ec 100644 --- a/knowledgebase/pivot.md +++ b/knowledgebase/pivot.md @@ -6,6 +6,8 @@ date: 2024-12-11 ClickHouse has no pivot operator, but we can achieve similar behavior using [aggregate function combinators](/docs/en/sql-reference/aggregate-functions/combinators) and, in particular, the ones with the [`-Map` suffix](/docs/en/sql-reference/aggregate-functions/combinators#-map). + + In this article, we're going to learn how to do that. There is also a video covering the same material, which you can see below: diff --git a/knowledgebase/postgresql-to-parquet-csv-json.md b/knowledgebase/postgresql-to-parquet-csv-json.md index 8ba7931f22b..4e8d5ca2e80 100644 --- a/knowledgebase/postgresql-to-parquet-csv-json.md +++ b/knowledgebase/postgresql-to-parquet-csv-json.md @@ -1,5 +1,7 @@ --- +title: "Export PostgreSQL data to Parquet, CSV or JSON" date: 2023-03-22 +description: "Learn how to export PostgreSQL data to Parquet, CSV, or JSON formats using `clickhouse-local` with various examples." --- # Export PostgreSQL data to Parquet, CSV or JSON @@ -11,6 +13,8 @@ This one is easy with `clickhouse-local`: The output format can be any of the supported [ouput formats](https://clickhouse.com/docs/en/interfaces/formats) in ClickHouse. Let's look at a few examples... + + These examples use `clickhouse-local`, which is a part of the ClickHouse binary. Download it using the following: ```bash diff --git a/knowledgebase/production.md b/knowledgebase/production.md index 9046a70029c..03e370203c1 100644 --- a/knowledgebase/production.md +++ b/knowledgebase/production.md @@ -6,7 +6,11 @@ date: 2021-09-01 # Which ClickHouse Version to Use in Production? {#which-clickhouse-version-to-use-in-production} -First of all, let’s discuss why people ask this question in the first place. There are two key reasons: +First of all, let’s discuss why people ask this question in the first place. + + + +There are two key reasons: 1. ClickHouse is developed with pretty high velocity, and usually there are 10+ stable releases per year. That makes a wide range of releases to choose from, which is not so trivial of a choice. 2. Some users want to avoid spending time figuring out which version works best for their use case and just follow someone else’s advice. @@ -33,7 +37,7 @@ Here are some key points to get reasonable fidelity in a pre-production environm The second area to invest in is **automated testing infrastructure**. Don’t assume that if some kind of query has executed successfully once, it’ll continue to do so forever. It’s OK to have some unit tests where ClickHouse is mocked, but make sure your product has a reasonable set of automated tests that are run against real ClickHouse and check that all important use cases are still working as expected. -An extra step forward could be contributing those automated tests to [ClickHouse’s open-source test infrastructure](https://github.com/ClickHouse/ClickHouse/tree/master/tests) that are continuously used in its day-to-day development. It definitely will take some additional time and effort to learn [how to run it](@site/docs/en/development/tests.md) and then how to adapt your tests to this framework, but it’ll pay off by ensuring that ClickHouse releases are already tested against them when they are announced stable, instead of repeatedly losing time on reporting the issue after the fact and then waiting for a bugfix to be implemented, backported and released. Some companies even have such test contributions to infrastructure by its use as an internal policy, (called [Beyonce’s Rule](https://www.oreilly.com/library/view/software-engineering-at/9781492082781/ch01.html#policies_that_scale_well) at Google). +An extra step forward could be contributing those automated tests to [ClickHouse’s open-source test infrastructure](https://github.com/ClickHouse/ClickHouse/tree/master/tests) that are continuously used in its day-to-day development. It definitely will take some additional time and effort to learn [how to run it](/docs/en/development/tests) and then how to adapt your tests to this framework, but it’ll pay off by ensuring that ClickHouse releases are already tested against them when they are announced stable, instead of repeatedly losing time on reporting the issue after the fact and then waiting for a bugfix to be implemented, backported and released. Some companies even have such test contributions to infrastructure by its use as an internal policy, (called [Beyonce’s Rule](https://www.oreilly.com/library/view/software-engineering-at/9781492082781/ch01.html#policies_that_scale_well) at Google). When you have your pre-production environment and testing infrastructure in place, choosing the best version is straightforward: @@ -61,5 +65,5 @@ Here is some guidance on how to choose between them: Many teams who initially think that `lts` is the way to go often switch to `stable` anyway because of some recent feature that’s important for their product. :::warning -One more thing to keep in mind when upgrading ClickHouse: we’re always keeping an eye on compatibility across releases, but sometimes it’s not reasonable to keep and some minor details might change. So make sure you check the [changelog](@site/docs/en/whats-new/changelog/index.md) before upgrading to see if there are any notes about backward-incompatible changes. +One more thing to keep in mind when upgrading ClickHouse: we’re always keeping an eye on compatibility across releases, but sometimes it’s not reasonable to keep and some minor details might change. So make sure you check the [changelog](/docs/en/whats-new/changelog) before upgrading to see if there are any notes about backward-incompatible changes. ::: diff --git a/knowledgebase/profiling-clickhouse-with-llvm-xray.md b/knowledgebase/profiling-clickhouse-with-llvm-xray.md index 21e07f506c7..4dafc5ec3ee 100644 --- a/knowledgebase/profiling-clickhouse-with-llvm-xray.md +++ b/knowledgebase/profiling-clickhouse-with-llvm-xray.md @@ -1,5 +1,7 @@ --- +title: "Profiling ClickHouse with LLVM's XRay" date: 2024-11-13 +description: "Learn how to profile ClickHouse using LLVM's XRay instrumentation profiler, visualize traces, and analyze performance." --- # Profiling ClickHouse with LLVM's XRay @@ -8,11 +10,12 @@ date: 2024-11-13 LLVM already includes a tool that instruments the code that allows us to do [instrumentation profiling](https://en.wikipedia.org/wiki/Profiling_(computer_programming)#Instrumentation). As -opposed to [sampling or statistical -profiling](https://en.wikipedia.org/wiki/Profiling_(computer_programming)#Statistical_profilers), +opposed to [sampling or statistical profiling](https://en.wikipedia.org/wiki/Profiling_(computer_programming)#Statistical_profilers), it's very precise without losing any calls, at the expense of needing to instrument the code and be more resource expensive. + + In a few words, an instrumentation profiler introduces new code to track the call to all functions. Statistical profilers allow us to run the code without requiring any changes, taking snapshots periodically to see the state of the application. So, only the functions running while the snapshot diff --git a/knowledgebase/projection_example.md b/knowledgebase/projection_example.md index f2dd9a50c8f..9364fd8e82e 100644 --- a/knowledgebase/projection_example.md +++ b/knowledgebase/projection_example.md @@ -1,12 +1,15 @@ --- -title: How to confirm if a Projection is used by the query? +title: "How to confirm if a Projection is used by the query?" date: 2022-07-10 +description: "Learn how to check if a projection is used in ClickHouse queries by testing with sample data and using EXPLAIN to verify projection usage." --- ### Question How can I tell if a projection is used? + + ### Answer 1. Create a sample database diff --git a/knowledgebase/python-clickhouse-connect-example.md b/knowledgebase/python-clickhouse-connect-example.md index bcfb57d9fcc..145ab5c3066 100644 --- a/knowledgebase/python-clickhouse-connect-example.md +++ b/knowledgebase/python-clickhouse-connect-example.md @@ -1,11 +1,15 @@ --- -Date: 2023-05-08 +date: 2023-05-08 +title: "A Python client working example for connecting to ClickHouse Cloud Service" +description: "Learn how to connect to ClickHouse Cloud Service using Python with a step-by-step example using the clickhouse-connect driver." --- # A Python client working example for connecting to ClickHouse Cloud Service This is a step by step example on how to start using Python with ClickHouse Cloud service. + + :::note Keep in mind that Python versions and libraries dependencies are constantly evolving. Make also sure to use the latest supported versions of both the driver and Python environment when trying this. diff --git a/knowledgebase/python_http_requests.md b/knowledgebase/python_http_requests.md index a155f6d0a22..f8487743271 100644 --- a/knowledgebase/python_http_requests.md +++ b/knowledgebase/python_http_requests.md @@ -8,6 +8,7 @@ date: 2022-07-10 Can I just run HTTP requests to ClickHouse server using _requests_ module? + # Answer diff --git a/knowledgebase/query_max_execution_time.md b/knowledgebase/query_max_execution_time.md index 697b164b278..f8541ed78da 100644 --- a/knowledgebase/query_max_execution_time.md +++ b/knowledgebase/query_max_execution_time.md @@ -8,6 +8,7 @@ date: 2022-07-10 How do I enforce a time limit on my queries? + # Answer diff --git a/knowledgebase/read_consistency.md b/knowledgebase/read_consistency.md index 5f8a1cd0838..94f02f91676 100644 --- a/knowledgebase/read_consistency.md +++ b/knowledgebase/read_consistency.md @@ -1,5 +1,7 @@ --- date: 2024-02-14 +title: "How to achieve data read consistency in ClickHouse?" +description: "Learn how to ensure data consistency when reading from ClickHouse, whether you're connected to the same node or a random node." --- # How to achieve data read consistency in ClickHouse? @@ -8,6 +10,8 @@ date: 2024-02-14 I'm writing data into ClickHouse cloud and need to be able ,when reading data, to guarantee that I'm getting the latest complete information. + + ## Answer ### Talking to same node diff --git a/knowledgebase/recreate_table_across_terminals.md b/knowledgebase/recreate_table_across_terminals.md index c05f647ce00..cc1ae0e29fc 100644 --- a/knowledgebase/recreate_table_across_terminals.md +++ b/knowledgebase/recreate_table_across_terminals.md @@ -1,14 +1,18 @@ --- date: 2023-03-24 +title: "How to quickly recreate a small table across different terminals" +description: "Learn how to quickly recreate a small table and its data across different terminals using copy/paste for development environments." --- # How to quickly recreate a small table across different terminals -**Question:** +## Question How can I quickly recreate a table and its data using just copy/paste across different terminals? -**Answer:** + + +## Answer This is NOT a recommended practice to migrate data from one database to another and it should NOT be used for production data migration. diff --git a/knowledgebase/row-column-policy.md b/knowledgebase/row-column-policy.md index 0ae932c28bf..19b4294326c 100644 --- a/knowledgebase/row-column-policy.md +++ b/knowledgebase/row-column-policy.md @@ -1,11 +1,15 @@ --- date: 2023-08-13 +title: "Does ClickHouse support row-level and column-level security?" +description: "Learn about row-level and column-level access restrictions in ClickHouse and ClickHouse Cloud, and how to implement role-based access control (RBAC) with policies." --- # Does ClickHouse support row-level and column-level security? ClickHouse and ClickHouse Cloud both support row and column-level access restrictions for read-only users necessary to achieve a role-based access control (RBAC) model. + + [Row Policies](/docs/en/operations/access-rights#row-policy-management) can be used to specify which rows will be returned to a read-only user when they query a table. ClickHouse Cloud is configured to enable the SQL-driven workflow by default. To leverage this workflow [CREATE](/docs/en/sql-reference/statements/create/user) a user, [GRANT](/docs/en/sql-reference/statements/grant) the user privileges on a table, then set the appropriate [ROW POLICY](/docs/en/operations/access-rights#row-policy-management). When the user performs a `SELECT * FROM table`, only rows allowed by the policy will be displayed. Column-level restrictions may be specified directly using the [GRANT statement](/docs/en/sql-reference/statements/grant) to enable table-level access for users and roles. Users *may only* include columns for which they have access in a query. Selecting restricted columns from a table in which the user does not have access to all the specified columns, such as in the query `SELECT * FROM table`, will return an error stating the user has insufficient permissions. diff --git a/knowledgebase/s3_export_data_year_month_folders.md b/knowledgebase/s3_export_data_year_month_folders.md index 2f284ced6ac..9e1705c676f 100644 --- a/knowledgebase/s3_export_data_year_month_folders.md +++ b/knowledgebase/s3_export_data_year_month_folders.md @@ -1,5 +1,7 @@ --- date: 2023-03-24 +title: "How can I do partitioned writes by year and month on S3?" +description: "Learn how to write partitioned data by year and month to an S3 bucket in ClickHouse, using a custom path structure for organizing the data." --- # How can I do partitioned writes by year and month on S3? @@ -19,8 +21,9 @@ I want to export data segregating the path in S3 bucket to follow a structure li and so on ... + -### Answer +## Answer Considering the ClickHouse table: diff --git a/knowledgebase/schema_migration_tools.md b/knowledgebase/schema_migration_tools.md index cb9e008b272..f144e675641 100644 --- a/knowledgebase/schema_migration_tools.md +++ b/knowledgebase/schema_migration_tools.md @@ -1,10 +1,16 @@ --- date: 2023-06-07 +title: "Automatic schema migration tools for ClickHouse" +description: "Learn about automatic schema migration tools for ClickHouse and how to manage changing database schemas over time." --- # Automatic schema migration tools for ClickHouse -We often get asked about a good schema migration tool for ClickHouse and what is the best practice to manage database schemas in ClickHouse that might change over time? There is no standard schema migration tool for ClickHouse, but we have compiled the following list (in no particular order) of automatic schema migration tools with support for ClickHouse that we know: +We often get asked about a good schema migration tool for ClickHouse and what is the best practice to manage database schemas in ClickHouse that might change over time? + + + +There is no standard schema migration tool for ClickHouse, but we have compiled the following list (in no particular order) of automatic schema migration tools with support for ClickHouse that we know: - [Goose](https://github.com/pressly/goose) - [Atlas](https://atlasgo.io/guides/clickhouse?utm_source=clickhouse&utm_term=knowledge) diff --git a/knowledgebase/search-across-node-for-tables-with-a-wildcard.md b/knowledgebase/search-across-node-for-tables-with-a-wildcard.md index a179c79fb02..5d847ffdea3 100644 --- a/knowledgebase/search-across-node-for-tables-with-a-wildcard.md +++ b/knowledgebase/search-across-node-for-tables-with-a-wildcard.md @@ -6,9 +6,11 @@ keywords: [search nodes, query_log, wildcard, table prefix] # How do I search across the cluster for data with from different tables which have same prefix? - This is useful when there are tables that have similar naming conventions and similar columns but are not replicated. An example is searching the system database for entries in the query log tables. + + + The `query_log` table is not replicated, and only queries that are executed on a specific node get logged. Data may also roll to a different table For example, data may be inserted into `query_log_0`, `query_log_1`, etc. Since one node may roll at a different time than others, it is useful to try to find the data we're looking for in tables that are not exactly named the same. In essence, we need to do something like this, but in ClickHouse syntax: diff --git a/knowledgebase/send_logs_level.md b/knowledgebase/send_logs_level.md index 15cfdb4a8f0..33e7afa8861 100644 --- a/knowledgebase/send_logs_level.md +++ b/knowledgebase/send_logs_level.md @@ -1,11 +1,15 @@ --- date: 2023-03-01 +title: "Capturing server logs of queries at the client" +description: "Learn how to capture server logs at the client level, even with different log settings, using the `send_logs_level` client setting." --- # Capturing server logs of queries at the client A client can view the server logs - even at a different level than what the server log level is configured to - by setting the `send_logs_level` client setting. + + For example, suppose the client runs: ```sql diff --git a/knowledgebase/terraform_example.md b/knowledgebase/terraform_example.md index 8fe517b9c9e..e48eecb0739 100644 --- a/knowledgebase/terraform_example.md +++ b/knowledgebase/terraform_example.md @@ -4,10 +4,13 @@ description: "This covers an example of how you can use terraform to create/dele Date: 2023-09-02 --- -### How can I use API to manage clusters on ClickHouse Cloud? +## Terraform example on how to use Cloud API -### Answer +How can I use API to manage clusters on ClickHouse Cloud? + + +## Answer We will use Terraform to configure our infra and [ClickHouse Provider](https://registry.terraform.io/providers/ClickHouse/clickhouse/latest/docs) diff --git a/knowledgebase/time-series.md b/knowledgebase/time-series.md index da716ac8905..711389e0714 100644 --- a/knowledgebase/time-series.md +++ b/knowledgebase/time-series.md @@ -10,6 +10,8 @@ _Note: Please see the blog [Working with Time series data in ClickHouse](https:/ ClickHouse is a generic data storage solution for [OLAP](https://clickhouse.com/docs/en/faq/general/olap) workloads, while there are many specialized time-series database management systems. Nevertheless, ClickHouse’s [focus on query execution speed](https://clickhouse.com/docs/en/faq/general/why-clickhouse-is-so-fast) allows it to outperform specialized systems in many cases. There are many independent benchmarks on this topic out there, so we’re not going to conduct one here. Instead, let’s focus on ClickHouse features that are important to use if that’s your use case. + + First of all, there are **[specialized codecs](https://clickhouse.com/docs/en/sql-reference/statements/create/table#specialized-codecs)** which make typical time-series. Either common algorithms like `DoubleDelta` and `Gorilla` or specific to ClickHouse like `T64`. Second, time-series queries often hit only recent data, like one day or one week old. It makes sense to use servers that have both fast nVME/SSD drives and high-capacity HDD drives. ClickHouse [TTL](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/##table_engine-mergetree-multiple-volumes) feature allows to configure keeping fresh hot data on fast drives and gradually move it to slower drives as it ages. Rollup or removal of even older data is also possible if your requirements demand it. diff --git a/knowledgebase/tips-tricks-optimizing-basic-data-types-in-clickhouse.md b/knowledgebase/tips-tricks-optimizing-basic-data-types-in-clickhouse.md index 4d0b564e8f1..bd28104434b 100644 --- a/knowledgebase/tips-tricks-optimizing-basic-data-types-in-clickhouse.md +++ b/knowledgebase/tips-tricks-optimizing-basic-data-types-in-clickhouse.md @@ -10,6 +10,8 @@ date: 2024-07-02 What data types should I use in ClickHouse to optimize my queries for speed and storage? + + ### Answer Many times when using an automated conversion from another system or trying to choose a data type, users will often choose the "more is better" or "choose what's easier" or "choose the most generic" approaches. This will likely work for small datasets in the millions, maybe even billions of rows. It may not be noticeable and is acceptable for those type of sets where users' queries difference is small in their use-cases. diff --git a/knowledgebase/use-clickhouse-for-log-analytics.md b/knowledgebase/use-clickhouse-for-log-analytics.md index 82f3156cb96..00f4da79689 100644 --- a/knowledgebase/use-clickhouse-for-log-analytics.md +++ b/knowledgebase/use-clickhouse-for-log-analytics.md @@ -4,4 +4,8 @@ date: 2023-05-10 # Using ClickHouse for log analytics -ClickHouse is popular for logs and metrics analysis because of the real-time analytics capabilities provided. See [ClickHouse for log analytics](https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry) for details using the Open Telemetry OTEL collector, Fluent Bit, or Vector. +ClickHouse is popular for logs and metrics analysis because of the real-time analytics capabilities provided. + + + +See our official [Observability guide](/docs/en/observability) for further details. diff --git a/knowledgebase/useful-queries-for-troubleshooting.md b/knowledgebase/useful-queries-for-troubleshooting.md index a29daeca8f6..b625ec1d913 100644 --- a/knowledgebase/useful-queries-for-troubleshooting.md +++ b/knowledgebase/useful-queries-for-troubleshooting.md @@ -1,10 +1,16 @@ --- date: 2023-03-17 +title: Useful queries for troubleshooting +description: A collection of handy queries for troubleshooting ClickHouse, including monitoring table sizes, long-running queries, and errors. --- # Useful queries for troubleshooting -In no particular order, here are some handy queries for troubleshooting ClickHouse and figuring out what is happening. We also have a great blog with some [essential queries for monitoring ClickHouse](https://clickhouse.com/blog/monitoring-troubleshooting-select-queries-clickhouse). +In no particular order, here are some handy queries for troubleshooting ClickHouse and figuring out what is happening. + + + +We also have a great blog with some [essential queries for monitoring ClickHouse](https://clickhouse.com/blog/monitoring-troubleshooting-select-queries-clickhouse). ## View which settings have been changed from the default diff --git a/knowledgebase/using-array-join-to-extract-and-query-attributes.md b/knowledgebase/using-array-join-to-extract-and-query-attributes.md index 3ce696b8ec7..ff5ed49268d 100644 --- a/knowledgebase/using-array-join-to-extract-and-query-attributes.md +++ b/knowledgebase/using-array-join-to-extract-and-query-attributes.md @@ -10,6 +10,8 @@ date: 2024-06-21 If I have varying attributes in a column using map types, how can I extract them and use them in queries? + + ### Answer This is a basic example of extracting keys and values from a variable attributes field. diff --git a/knowledgebase/vector-search.md b/knowledgebase/vector-search.md index fb4b5e84ae8..6a8dfa62a9f 100644 --- a/knowledgebase/vector-search.md +++ b/knowledgebase/vector-search.md @@ -1,11 +1,16 @@ --- date: 2023-10-26 title: Can you use ClickHouse for vector search? +description: Learn how to use ClickHouse for vector search, including storing embeddings and searching with distance functions like cosine similarity. --- # Can ClickHouse do vector search? -Yes, ClickHouse can perform vector search. The main advantages of using ClickHouse for vector search compared to using more specialized vector databases include: +Yes, ClickHouse can perform vector search. + + + +The main advantages of using ClickHouse for vector search compared to using more specialized vector databases include: - Using ClickHouse's filtering and full-text search capabilities to refine your dataset before performing a search. - Performing analytics on your datasets. diff --git a/knowledgebase/view_number_of_active_mutations.md b/knowledgebase/view_number_of_active_mutations.md index f36ea15fc29..412e25304e1 100644 --- a/knowledgebase/view_number_of_active_mutations.md +++ b/knowledgebase/view_number_of_active_mutations.md @@ -1,5 +1,7 @@ --- date: 2023-06-07 +title: How do I view the number of active or queued mutations? +description: Monitor the number of active or queued mutations in ClickHouse, especially when performing `ALTER` or `UPDATE` operations. Use the `system.mutations` table for tracking mutations. --- # How do I view the number of active or queued mutations? @@ -8,6 +10,8 @@ date: 2023-06-07 How do I view the number of active or queued mutations? + + ## Answer Monitoring the number of active or queued mutations is important if you are performing a lot of `ALTER` or `UPDATE` statements on your tables. These queries rewrite data parts and are not atomic - they are ordered by their creation part and applied to each part in that order. You can find more details on mutations in the [docs](https://clickhouse.com/docs/en/sql-reference/statements/alter#mutations). diff --git a/knowledgebase/when_is_ttl_applied.md b/knowledgebase/when_is_ttl_applied.md index 9f6ff4c1712..245ab24f74f 100644 --- a/knowledgebase/when_is_ttl_applied.md +++ b/knowledgebase/when_is_ttl_applied.md @@ -1,11 +1,15 @@ --- date: 2023-05-02 +title: When are TTL rules applied, and do we have control over it? +description: TTL rules in ClickHouse are eventually applied, and you can control when they are executed using the `merge_with_ttl_timeout` setting. Learn how to force TTL application and manage background threads for TTL execution. --- # When are TTL rules applied, and do we have control over it? TTL is going to be ***eventually*** applied. What does that mean? The `MergeTree` table setting [`merge_with_ttl_timeout`](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#merge_with_ttl_timeout) sets the minimum delay in seconds before repeating a merge with delete TTL. The default value is 14400 seconds (4 hours). But that is just the minimum delay, it can take longer until a merge for delete TTL is triggered. + + You can view all of your current TTL settings (like `merge_with_ttl_timeout`) with this query: ```sql diff --git a/knowledgebase/which-processes-are-currently-running.md b/knowledgebase/which-processes-are-currently-running.md index c4b03fb3b19..2bad77e444f 100644 --- a/knowledgebase/which-processes-are-currently-running.md +++ b/knowledgebase/which-processes-are-currently-running.md @@ -1,10 +1,16 @@ --- date: 2023-03-24 +title: How to check what code is currently running on a server? +description: ClickHouse provides introspection tools like `system.stack_trace` for inspecting what code is currently running on each server thread, helping with debugging and performance monitoring. --- # How to check what code is currently running on a server? -ClickHouse has a built-in debugger and introspection capabilities. For example, you can get the stack traces of every server's thread at runtime by querying the `system.stack_trace` table: +ClickHouse has a built-in debugger and introspection capabilities. + + + +For example, you can get the stack traces of every server's thread at runtime by querying the `system.stack_trace` table: ```sql SELECT diff --git a/knowledgebase/who-is-using-clickhouse.md b/knowledgebase/who-is-using-clickhouse.md index 2dbb70388c6..1975a6ac95f 100644 --- a/knowledgebase/who-is-using-clickhouse.md +++ b/knowledgebase/who-is-using-clickhouse.md @@ -6,7 +6,11 @@ date: 2021-09-01 # Who Is Using ClickHouse? {#who-is-using-clickhouse} -Being an open-source product makes this question not so straightforward to answer. You do not have to tell anyone if you want to start using ClickHouse, you just go grab source code or pre-compiled packages. There’s no contract to sign and the [Apache 2.0 license](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) allows for unconstrained software distribution. +Being an open-source product makes this question not so straightforward to answer. You do not have to tell anyone if you want to start using ClickHouse, you just go grab source code or pre-compiled packages. + + + +There’s no contract to sign and the [Apache 2.0 license](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) allows for unconstrained software distribution. Also, the technology stack is often in a grey zone of what’s covered by an NDA. Some companies consider technologies they use as a competitive advantage even if they are open-source and do not allow employees to share any details publicly. Some see some PR risks and allow employees to share implementation details only with their PR department approval. diff --git a/knowledgebase/why_default_logging_verbose.md b/knowledgebase/why_default_logging_verbose.md new file mode 100644 index 00000000000..a22911e1579 --- /dev/null +++ b/knowledgebase/why_default_logging_verbose.md @@ -0,0 +1,48 @@ +--- +date: 2025-01-10 +title: Why ClickHouse default logging setting is verbose? +description: "Explain why ClickHouse developers chose to set a verbose log level by default" +--- + +# Why ClickHouse default logging setting is verbose? + +One thing that often confuses new users is that ClickHouse generates a lot of log output, even under light load. + +This is because the default log level is for historical reasons `trace` (instead of `warning` that would be the default in other databases). + +The ClickHouse developers argue that `trace` provides a lot of insight in case goes wrong. + +On the other hand, large volumes of logging means that system table `system.text_log` fills up quickly and needs to be merged in the background. + +If the database runs stable, users may re-configure the log level. + +## Change log level + +The different log level available are documented [here](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#logger) + +Edit the ClickHouse server configuration file (`/etc/clickhouse-server/config.xml`) to modify the log level. + +Default value is `trace`, change it to the desired level. + +``` + + + + trace +... Rest of the configuration file + +``` \ No newline at end of file diff --git a/knowledgebase/why_is_my_primary_key_not_used.md b/knowledgebase/why_is_my_primary_key_not_used.md index 2997ff378e7..3d8c00842c1 100644 --- a/knowledgebase/why_is_my_primary_key_not_used.md +++ b/knowledgebase/why_is_my_primary_key_not_used.md @@ -5,6 +5,7 @@ date: 2024-12-12 --- Users may see cases where their query is slower than expected, in the belief they are ordering or filtering by a primary key. In this article we show how users can confirm the key is used, highlighting common reasons its not. + ## Create table {#create-table} diff --git a/knowledgebase/why_recommend_clickhouse_keeper_over_zookeeper.md b/knowledgebase/why_recommend_clickhouse_keeper_over_zookeeper.md index 646ca79303d..756be5814fa 100644 --- a/knowledgebase/why_recommend_clickhouse_keeper_over_zookeeper.md +++ b/knowledgebase/why_recommend_clickhouse_keeper_over_zookeeper.md @@ -1,11 +1,14 @@ --- date: 2023-06-07 +title: Why is ClickHouse Keeper recommended over ZooKeeper? +description: ClickHouse Keeper improves upon ZooKeeper with features like reduced disk space usage, faster recovery, and less memory consumption, offering better performance for ClickHouse clusters. --- # Why is ClickHouse Keeper recommended over ZooKeeper? ClickHouse Keeper provides the coordination system for data replication and distributed DDL queries execution. ClickHouse Keeper is compatible with ZooKeeper, but it might not be obvious why you should use ClickHouse Keeper instead of ZooKeeper. This article discusses some of the benefits of Keeper. + ## Answer diff --git a/knowledgebase/windows_active_directory_to_ch_roles.md b/knowledgebase/windows_active_directory_to_ch_roles.md index 7547ddb223e..32dacf84193 100644 --- a/knowledgebase/windows_active_directory_to_ch_roles.md +++ b/knowledgebase/windows_active_directory_to_ch_roles.md @@ -8,6 +8,8 @@ date: 2024-05-20 This example shows how AD users that belong to different AD security groups can be given role access in ClickHouse. It also shows how a user may be added to multiple AD user groups so they can have access provided by multiple roles. + + In this environment, we have the following: - A Windows Active Directory domain: `marsnet2.local` - A ClickHouse Cluster, `cluster_1S_3R` with 3 nodes on a cluster configuration of 1 shard, 3 replicas diff --git a/package.json b/package.json index f114c853314..0c4bb92545d 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,12 @@ "write-heading-ids": "docusaurus write-heading-ids" }, "dependencies": { - "@docusaurus/core": "2.4.3", - "@docusaurus/plugin-client-redirects": "2.4.3", - "@docusaurus/preset-classic": "2.4.3", - "@docusaurus/theme-mermaid": "2.4.3", - "@mdx-js/react": "^1.6.22", - "@radix-ui/react-navigation-menu": "^1.1.4", + "@docusaurus/core": "3.7.0", + "@docusaurus/plugin-client-redirects": "3.7.0", + "@docusaurus/preset-classic": "3.7.0", + "@docusaurus/theme-mermaid": "3.7.0", + "@mdx-js/react": "^3.1.0", + "@radix-ui/react-navigation-menu": "^1.2.3", "axios": "^1.7.9", "clsx": "^2.1.0", "docusaurus-plugin-sass": "^0.2.6", @@ -35,20 +35,25 @@ "esbuild-loader": "^4.0.3", "gray-matter": "^4.0.3", "hast-util-is-element": "1.1.0", - "prism-react-renderer": "^1.3.5", - "react": "^17.0.2", - "react-dom": "^17.0.2", - "rehype-katex": "5", + "katex": "^0.16.19", + "node-fetch": "^3.3.2", + "prism-react-renderer": "^2.4.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "rehype-katex": "^7.0.1", "remark-docusaurus-tabs": "^0.2.0", - "remark-math": "3", - "sass": "^1.82.0" + "remark-link-rewrite": "^1.0.7", + "remark-math": "^6.0.0", + "sass": "^1.83.1" }, "devDependencies": { "@argos-ci/cli": "^2.5.3", "@argos-ci/playwright": "^3.9.4", - "@docusaurus/module-type-aliases": "3.6.3", + "@docusaurus/module-type-aliases": "3.7.0", "@playwright/test": "^1.49.1", - "cheerio": "^1.0.0" + "@types/react": "^19.0.4", + "cheerio": "^1.0.0", + "typescript": "^5.7.3" }, "browserslist": { "production": [ diff --git a/plugins/header.js b/plugins/header.js index 1e011fda1c1..c3c45e51d76 100644 --- a/plugins/header.js +++ b/plugins/header.js @@ -1,9 +1,9 @@ // A JavaScript function that returns an object. // `context` is provided by Docusaurus. Example: siteConfig can be accessed from context. // `opts` is the user-defined options. -const menuItems = require('./menuItems.json') -const fetch = require('node-fetch'); -async function chHeader(context, opts) { +import menuItems from './menuItems.json' with { type: 'json' }; + +export default async function chHeader(context, opts) { return { name: 'ch-header-plugin', @@ -30,5 +30,3 @@ async function chHeader(context, opts) { }, }; } - -module.exports = chHeader; diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index c9467f26b5a..e7db20c8375 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -78,6 +78,30 @@ SELECT prefix || (SELECT groupConcat(*) FROM main_content) INTO OUTFILE 'docs/en/operations/settings/settings.md' TRUNCATE FORMAT LineAsString " +# Replace issue sequences for mdx3 +if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + # allow_archive_path_syntax + sed -i '' 's#'\''\\\\ :: \\\\'\''#\` :: \`#g' docs/en/operations/settings/settings.md + # max_partitions_to_read + sed -i '' "s/<= 0 means unlimited/\<0 means unlimited/g" docs/en/operations/settings/settings.md + # optimize_time_filter_with_preimage + sed -i '' "s|toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31'|\\\`toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31'\\\`|g" docs/en/operations/settings/settings.md + # stop_refreshable_materialized_views_on_startup + sed -i '' 's@\\\\@\\@g' docs/en/operations/settings/settings.md +else + # Linux + # allow_archive_path_syntax + sed -i 's#'\''\\\\ :: \\\\'\''#\` :: \`#g' docs/en/operations/settings/settings.md + # max_partitions_to_read + sed -i "s/<= 0 means unlimited/\<0 means unlimited/g" docs/en/operations/settings/settings.md + # optimize_time_filter_with_preimage + sed -i "s|toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31'|\\\`toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31'\\\`|g" docs/en/operations/settings/settings.md + # stop_refreshable_materialized_views_on_startup + sed -i 's@\\\\@\\@g' docs/en/operations/settings/settings.md +fi + + # Delete ClickHouse if [ -f ./clickhouse ]; then echo -e "\n[$SCRIPT_NAME] Deleting ClickHouse binary" diff --git a/sidebars.js b/sidebars.js index 9ee5ba90150..93ebb89d4cb 100644 --- a/sidebars.js +++ b/sidebars.js @@ -818,9 +818,18 @@ const sidebars = { "en/integrations/data-ingestion/data-formats/arrow-avro-orc", "en/integrations/data-ingestion/data-formats/templates-regex", { - type: "doc", - id: "en/interfaces/formats", - label: "View all formats...", + type: "category", + label: "View All Formats", + link: { + type: "doc", + id: "en/interfaces/formats", + }, + items: [ + { + type: "autogenerated", + dirName: "en/interfaces/formats", + } + ] }, ], }, diff --git a/src/components/DocsCategoryDropdown/index.jsx b/src/components/DocsCategoryDropdown/index.jsx index ce37c5ee1c9..7df7e839a7e 100644 --- a/src/components/DocsCategoryDropdown/index.jsx +++ b/src/components/DocsCategoryDropdown/index.jsx @@ -1,7 +1,8 @@ import React, { useState, useRef, useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import Link from '@docusaurus/Link'; -import {useDocsSidebar} from '@docusaurus/theme-common/internal'; +import {useDocsSidebar} from '@docusaurus/plugin-content-docs/client'; + import styles from './styles.module.css'; function DocsCategoryDropdown({ dropdownCategory }) { diff --git a/src/css/custom.scss b/src/css/custom.scss index d89458db003..749848ebc48 100644 --- a/src/css/custom.scss +++ b/src/css/custom.scss @@ -1,6 +1,6 @@ -@import "colors"; -@import "default"; -@import "home"; +@use "colors"; +@use "default"; +@use "home"; html[data-theme="dark"] { background: var(--click-color-background); @@ -222,11 +222,48 @@ html { code { font-weight: bold; background-color: var(--ifm-code-background) !important; + scrollbar-width: thin; + scrollbar-color: rgb(99, 99, 99) rgb(41, 45, 62); span { font-style: normal !important; } } +/* Style individual tokens */ +code .token.keyword { + font-style: italic !important; /* Italicize keywords */ + color: rgb(86, 156, 214) !important; /* Blue for keywords */ +} + +code .token.plain { + color: rgb(156, 220, 254) !important; +} + +code .token.punctuation { + color: rgb(199, 146, 234) !important; /* Match punctuation color */ +} + +code .token.operator { + color: rgb(137, 221, 255) !important; /* Light blue for operators */ +} + +code .token.string { + color: rgb(206, 145, 120) !important; /* Orange for strings */ +} + +code .token.number { + color: rgb(181, 206, 168) !important; /* Green for numbers */ +} + +code .token.function { + color: rgb(220, 220, 170) !important; /* Bright blue for functions */ +} + +/* Ensure keywords are not italic */ +code span.token.keyword { + font-style: normal !important; +} + .markdown h1:first-child { font-size: 32px !important; } diff --git a/src/hooks/fixLinks.js b/src/hooks/fixLinks.js new file mode 100644 index 00000000000..e87130597de --- /dev/null +++ b/src/hooks/fixLinks.js @@ -0,0 +1,12 @@ +import { visit } from 'unist-util-visit'; + +export default function fixLinks() { + return (tree) => { + visit(tree, 'link', (node) => { + // e.g. fix `.md/#` → `.md#` + if (node.url.includes('.md/#')) { + node.url = node.url.replace('.md/#', '.md#'); + } + }); + }; +} diff --git a/src/theme/DocCard/index.js b/src/theme/DocCard/index.js index 7513f565b81..8d13f60a625 100644 --- a/src/theme/DocCard/index.js +++ b/src/theme/DocCard/index.js @@ -1,13 +1,11 @@ import React from 'react'; import clsx from 'clsx'; import Link from '@docusaurus/Link'; -import { - findFirstCategoryLink, - useDocById, -} from '@docusaurus/theme-common/internal'; -import isInternalUrl from '@docusaurus/isInternalUrl'; + +import {useDocById, findFirstSidebarItemLink} from '@docusaurus/plugin-content-docs/client'; import {translate} from '@docusaurus/Translate'; import styles from './styles.module.css'; + function CardContainer({href, children}) { return ( ); } + function CardLayout({href, icon, title, description}) { return ( @@ -33,12 +32,15 @@ function CardLayout({href, icon, title, description}) { ); } -function CardCategory({item}) { - const href = findFirstCategoryLink(item); + +function CardCategory({ item }) { + const href = findFirstSidebarItemLink(item); // Unexpected: categories that don't have a link have been filtered upfront if (!href) { return null; } + const itemCount = Array.isArray(item.items) ? item.items.length : 0; + return ( ); + } + function CardLink({item}) { const icon = ''; const doc = useDocById(item.docId ?? undefined); @@ -68,6 +72,7 @@ function CardLink({item}) { /> ); } + export default function DocCard({item}) { switch (item.type) { case 'link': diff --git a/src/theme/DocItem/Footer/index.js b/src/theme/DocItem/Footer/index.js index 60211fb14ca..62bc39cf6df 100644 --- a/src/theme/DocItem/Footer/index.js +++ b/src/theme/DocItem/Footer/index.js @@ -1,9 +1,8 @@ import React from 'react'; import clsx from 'clsx'; import {ThemeClassNames} from '@docusaurus/theme-common'; -import {useDoc} from '@docusaurus/theme-common/internal'; +import {useDoc} from '@docusaurus/plugin-content-docs/client'; import LastUpdated from '@theme/LastUpdated'; -import EditThisPage from '@theme/EditThisPage'; import TagsListInline from '@theme/TagsListInline'; import styles from './styles.module.css'; function TagsRow(props) { diff --git a/src/theme/DocItem/Layout/index.js b/src/theme/DocItem/Layout/index.js index 7c15c0e3cb1..d98130e13aa 100644 --- a/src/theme/DocItem/Layout/index.js +++ b/src/theme/DocItem/Layout/index.js @@ -1,7 +1,7 @@ import React from 'react'; import clsx from 'clsx'; import {useWindowSize} from '@docusaurus/theme-common'; -import {useDoc} from '@docusaurus/theme-common/internal'; +import {useDoc} from '@docusaurus/plugin-content-docs/client'; import DocItemPaginator from '@theme/DocItem/Paginator'; import DocVersionBanner from '@theme/DocVersionBanner'; import DocVersionBadge from '@theme/DocVersionBadge'; diff --git a/src/theme/DocItem/TOC/Desktop/index.js b/src/theme/DocItem/TOC/Desktop/index.js index 5fb46f071c7..bf306d6646a 100644 --- a/src/theme/DocItem/TOC/Desktop/index.js +++ b/src/theme/DocItem/TOC/Desktop/index.js @@ -1,6 +1,6 @@ import React, {useEffect, useState} from 'react'; import {ThemeClassNames} from '@docusaurus/theme-common'; -import {useDoc} from '@docusaurus/theme-common/internal'; +import {useDoc} from '@docusaurus/plugin-content-docs/client'; import TOC from '@theme/TOC'; import clsx from "clsx"; import styles from './styles.module.css' diff --git a/src/theme/DocPage/Layout/Main/index.js b/src/theme/DocRoot/Layout/Main/index.js similarity index 90% rename from src/theme/DocPage/Layout/Main/index.js rename to src/theme/DocRoot/Layout/Main/index.js index 2c705ef9d5a..c28eba9fbbf 100644 --- a/src/theme/DocPage/Layout/Main/index.js +++ b/src/theme/DocRoot/Layout/Main/index.js @@ -1,6 +1,6 @@ import React from 'react'; import clsx from 'clsx'; -import {useDocsSidebar} from '@docusaurus/theme-common/internal'; +import {useDocsSidebar} from '@docusaurus/plugin-content-docs/client'; import styles from './styles.module.css'; export default function DocPageLayoutMain({hiddenSidebarContainer, children}) { const sidebar = useDocsSidebar(); diff --git a/src/theme/DocPage/Layout/Main/styles.module.css b/src/theme/DocRoot/Layout/Main/styles.module.css similarity index 100% rename from src/theme/DocPage/Layout/Main/styles.module.css rename to src/theme/DocRoot/Layout/Main/styles.module.css diff --git a/src/theme/DocPage/Layout/index.js b/src/theme/DocRoot/Layout/index.js similarity index 81% rename from src/theme/DocPage/Layout/index.js rename to src/theme/DocRoot/Layout/index.js index b66a8baf031..ed4b93ab28f 100644 --- a/src/theme/DocPage/Layout/index.js +++ b/src/theme/DocRoot/Layout/index.js @@ -1,9 +1,9 @@ import React, {useState} from 'react'; -import {useDocsSidebar} from '@docusaurus/theme-common/internal'; +import {useDocsSidebar} from '@docusaurus/plugin-content-docs/client'; import Layout from '@theme/Layout'; import BackToTopButton from '@theme/BackToTopButton'; -import DocPageLayoutSidebar from '@theme/DocPage/Layout/Sidebar'; -import DocPageLayoutMain from '@theme/DocPage/Layout/Main'; +import DocPageLayoutSidebar from '@theme/DocRoot/Layout/Sidebar'; +import DocPageLayoutMain from '@theme/DocRoot/Layout/Main'; import styles from './styles.module.css'; export default function DocPageLayout({children}) { const sidebar = useDocsSidebar(); diff --git a/src/theme/DocPage/Layout/styles.module.css b/src/theme/DocRoot/Layout/styles.module.css similarity index 100% rename from src/theme/DocPage/Layout/styles.module.css rename to src/theme/DocRoot/Layout/styles.module.css diff --git a/src/theme/DocSidebarItem/Category/index.js b/src/theme/DocSidebarItem/Category/index.js index 720e7789272..bf5fd9512bb 100644 --- a/src/theme/DocSidebarItem/Category/index.js +++ b/src/theme/DocSidebarItem/Category/index.js @@ -8,11 +8,10 @@ import { useCollapsible, } from '@docusaurus/theme-common'; import { - isActiveSidebarItem, - findFirstCategoryLink, - useDocSidebarItemsExpandedState, isSamePath, } from '@docusaurus/theme-common/internal'; + +import { findFirstSidebarItemLink, isActiveSidebarItem, useDocSidebarItemsExpandedState } from '@docusaurus/plugin-content-docs/client'; import Link from '@docusaurus/Link'; import {translate} from '@docusaurus/Translate'; import useIsBrowser from '@docusaurus/useIsBrowser'; @@ -48,9 +47,10 @@ function useCategoryHrefWithSSRFallback(item) { if (isBrowser || !item.collapsible) { return undefined; } - return findFirstCategoryLink(item); + return findFirstSidebarItemLink(item); }, [item, isBrowser]); } + function CollapseButton({categoryLabel, onClick}) { return (