feat(new sink): add greptime log sink (#20812)

* chore: add greptime log sink * chore: add changelog * chore: fix by pr comment * chore: add new changelog * chore: add doc and modify feature greptimedb-integration-tests to greptimedb-metrics-integration-tests * chore: update changelog * chore: make clippy happy * chore: fix integration test * chore: merge greptime logs and metrics into one integration test * chore: update greptimedb sink doc and licenses * chore: add greptimedb cue * chore: update LICENSE-3rdparty.csv * chore: add authors for changelog * chore: fix changelog format issue * chore: fix greptime sink cue issue * chore: fix greptime sink cue issue * chore: update greptimedb sinks doc Co-authored-by: Stephen Wakely <[email protected]> * chore: generate greptimedb sinks doc * Update website/cue/reference/components/sinks/base/greptimedb_logs.cue Co-authored-by: Brett Blue <[email protected]> * chore: make greptimedb sinks documents more readable Co-authored-by: Brett Blue <[email protected]> * chore: update greptimedb logs sinks code doc --------- Co-authored-by: Stephen Wakely <[email protected]> Co-authored-by: Brett Blue <[email protected]>
vectordotdev · Aug 19, 2024 · 35e82bd · 35e82bd
1 parent f371bc2
commit 35e82bd
Show file tree

Hide file tree

Showing 29 changed files with 2,236 additions and 411 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -256,7 +256,7 @@ ratatui = { version = "0.27.0", optional = true, default-features = false, featu
 hex = { version = "0.4.3", default-features = false, optional = true }
 
 # GreptimeDB
-greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust.git", rev = "d21dbcff680139ed2065b62100bac3123da7c789", optional = true }
+greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d", optional = true }
 
 # External libs
 arc-swap = { version = "1.7", default-features = false, optional = true }
@@ -680,6 +680,7 @@ sinks-logs = [
   "sinks-elasticsearch",
   "sinks-file",
   "sinks-gcp",
+  "sinks-greptimedb_logs",
   "sinks-honeycomb",
   "sinks-http",
   "sinks-humio",
@@ -707,7 +708,7 @@ sinks-metrics = [
   "sinks-blackhole",
   "sinks-console",
   "sinks-datadog_metrics",
-  "sinks-greptimedb",
+  "sinks-greptimedb_metrics",
   "sinks-humio",
   "sinks-influxdb",
   "sinks-kafka",
@@ -743,7 +744,8 @@ sinks-elasticsearch = ["transforms-metric_to_log"]
 sinks-file = ["dep:async-compression"]
 sinks-gcp = ["sinks-gcp-chronicle", "dep:base64", "gcp"]
 sinks-gcp-chronicle =  ["gcp"]
-sinks-greptimedb = ["dep:greptimedb-client"]
+sinks-greptimedb_metrics = ["dep:greptimedb-ingester"]
+sinks-greptimedb_logs = []
 sinks-honeycomb = []
 sinks-http = []
 sinks-humio = ["sinks-splunk_hec", "transforms-metric_to_log"]
@@ -856,7 +858,7 @@ fluent-integration-tests = ["docker", "sources-fluent"]
 gcp-cloud-storage-integration-tests = ["sinks-gcp"]
 gcp-integration-tests = ["sinks-gcp"]
 gcp-pubsub-integration-tests = ["sinks-gcp", "sources-gcp_pubsub"]
-greptimedb-integration-tests = ["sinks-greptimedb"]
+greptimedb-integration-tests = ["sinks-greptimedb_metrics","sinks-greptimedb_logs"]
 humio-integration-tests = ["sinks-humio"]
 http-client-integration-tests = ["sources-http_client"]
 influxdb-integration-tests = ["sinks-influxdb"]

diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
@@ -173,6 +173,9 @@ der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,Rust
 deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt <[email protected]>
 derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton <[email protected]>
 derive_arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald <[email protected]>, Manish Goregaokar <[email protected]>, Andre Bogus <[email protected]>, Corey Farwell <[email protected]>"
+derive_builder,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel <[email protected]>, Pascal Hertleif <[email protected]>, Jan-Erik Rediger <[email protected]>, Ted Driggs <[email protected]>"
+derive_builder_core,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel <[email protected]>, Pascal Hertleif <[email protected]>, Jan-Erik Rediger <[email protected]>, Ted Driggs <[email protected]>"
+derive_builder_macro,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel <[email protected]>, Pascal Hertleif <[email protected]>, Jan-Erik Rediger <[email protected]>, Ted Driggs <[email protected]>"
 derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema <[email protected]>
 digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
 dirs-next,https://github.com/xdg-rs/dirs,MIT OR Apache-2.0,The @xdg-rs members
@@ -243,7 +246,7 @@ graphql_client,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,
 graphql_client_codegen,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé <[email protected]>
 graphql_query_derive,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé <[email protected]>
 greptime-proto,https://github.com/GreptimeTeam/greptime-proto,Apache-2.0,The greptime-proto Authors
-greptimedb-client,https://github.com/GreptimeTeam/greptimedb-ingester-rust,Apache-2.0,The greptimedb-client Authors
+greptimedb-ingester,https://github.com/GreptimeTeam/greptimedb-ingester-rust,Apache-2.0,The greptimedb-ingester Authors
 grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger <[email protected]>
 group,https://github.com/zkcrypto/group,MIT OR Apache-2.0,"Sean Bowe <[email protected]>, Jack Grigg <[email protected]>"
 h2,https://github.com/hyperium/h2,MIT,"Carl Lerche <[email protected]>, Sean McArthur <[email protected]>"

diff --git a/changelog.d/20812_add_greptime_log_sink.feature.md b/changelog.d/20812_add_greptime_log_sink.feature.md
@@ -0,0 +1,6 @@
+* update greptimedb-ingester version
+* add a new greptimedb_logs sink
+* Changed the original greptime code structure to make it clearer
+* rename sink name from `greptimedb` to `greptimedb_metrics`
+
+authors: GreptimeTeam
diff --git a/scripts/integration/greptimedb/test.yaml b/scripts/integration/greptimedb/test.yaml
@@ -7,11 +7,12 @@ runner:
   env:
     GREPTIMEDB_ENDPOINT: greptimedb:4001
     GREPTIMEDB_HTTP: http://greptimedb:4000
+    no_proxy: greptimedb
 
 matrix:
   # Temporarily pegging to the latest known stable release
   # since using `latest` is failing consistently.
-  version: [v0.4.4]
+  version: [v0.9.0]
 
 # changes to these files/paths will invoke the integration test in CI
 # expressions are evaluated using https://github.com/micromatch/picomatch