diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 4cb681b4b5a29..052008d0ae084 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -35,6 +35,7 @@ Benss Blaupunkt Blusens Casio +Cantarell Celkon Ceph Chromecast diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ddd4816bfe92e..bdc3e9b5b79c4 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -779,6 +779,7 @@ nullishness numbackends oahd oap +oidc OKD omfwd omitempty diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index b9d810ffbbc03..b57222b053da6 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -21,7 +21,7 @@ jobs: # Run `git checkout` - uses: actions/checkout@v3 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.36.0 + - uses: bufbuild/buf-setup-action@v1.38.0 # Perform breaking change detection against the `master` branch - uses: bufbuild/buf-breaking-action@v1.1.4 with: diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index bf5213c0674d9..f43d69d876eaf 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -419,7 +419,7 @@ jobs: submit-job: name: Submit regression job runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 60 needs: - compute-metadata - upload-baseline-image-to-ecr diff --git a/Cargo.lock b/Cargo.lock index 6a371cf014bc9..b9693d8ed1575 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -464,7 +464,7 @@ dependencies = [ "fnv", "futures-util", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.4.0", "mime", "multer", "num-traits", @@ -514,7 +514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ "bytes 1.7.1", - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_json", ] @@ -2297,7 +2297,7 @@ dependencies = [ "futures-core", "prost 0.12.6", "prost-types 0.12.6", - "tonic 0.11.0", + "tonic", "tracing-core 0.1.32", ] @@ -2320,7 +2320,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic 0.11.0", + "tonic", "tracing 0.1.40", "tracing-core 0.1.32", "tracing-subscriber", @@ -2613,6 +2613,21 @@ dependencies = [ "cipher", ] +[[package]] +name = "curl-sys" +version = "0.4.74+curl-8.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8af10b986114528fcdc4b63b6f5f021b7057618411046a4de2ba0f0149a097bf" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "windows-sys 0.52.0", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -3414,7 +3429,7 @@ dependencies = [ "flate2", "futures 0.3.30", "glob", - "indexmap 2.2.6", + "indexmap 2.4.0", "libc", "quickcheck", "scan_fmt", @@ -3835,7 +3850,7 @@ dependencies = [ "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.11.0", + "tonic", "tonic-build 0.11.0", ] @@ -3856,7 +3871,7 @@ dependencies = [ "snafu 0.7.5", "tokio", "tokio-stream", - "tonic 0.11.0", + "tonic", "tonic-build 0.9.2", "tower", ] @@ -3894,7 +3909,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.9", - "indexmap 2.2.6", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -3913,7 +3928,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -4602,9 +4617,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -5071,7 +5086,7 @@ dependencies = [ "ena", "is-terminal", "itertools 0.10.5", - "lalrpop-util", + "lalrpop-util 0.20.0", "petgraph", "regex", "regex-syntax 0.7.5", @@ -5087,6 +5102,15 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" +[[package]] +name = "lalrpop-util" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "108dc8f5dabad92c65a03523055577d847f5dcc00f3e7d3a68bc4d48e01d8fe1" +dependencies = [ + "regex-automata 0.4.4", +] + [[package]] name = "lapin" version = "2.5.0" @@ -5442,7 +5466,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb791d015f8947acf5a7f62bd28d00f289bb7ea98cfbe3ffec1d061eee12df12" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "itoa", "lockfree-object-pool", "metrics", @@ -5463,7 +5487,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.14.5", - "indexmap 2.2.6", + "indexmap 2.4.0", "metrics", "num_cpus", "ordered-float 4.2.2", @@ -6321,8 +6345,8 @@ dependencies = [ "ordered-float 4.2.2", "prost 0.12.6", "prost-build 0.12.6", - "tonic 0.10.2", - "tonic-build 0.10.2", + "tonic", + "tonic-build 0.11.0", "vector-core", "vector-lookup", "vrl", @@ -6583,7 +6607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.4.0", ] [[package]] @@ -7041,7 +7065,7 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "nom", "num_enum 0.7.3", "prost 0.12.6", @@ -7102,6 +7126,16 @@ dependencies = [ "prost-derive 0.12.6", ] +[[package]] +name = "prost" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +dependencies = [ + "bytes 1.7.1", + "prost-derive 0.13.1", +] + [[package]] name = "prost-build" version = "0.11.9" @@ -7172,15 +7206,28 @@ dependencies = [ ] [[package]] -name = "prost-reflect" +name = "prost-derive" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5eec97d5d34bdd17ad2db2219aabf46b054c6c41bd5529767c9ce55be5898f" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2 1.0.86", + "quote 1.0.37", + "syn 2.0.75", +] + +[[package]] +name = "prost-reflect" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a6a9143ae25c25fa7b6a48d6cc08b10785372060009c25140a4e7c340e95af" dependencies = [ "base64 0.22.1", "once_cell", - "prost 0.12.6", - "prost-types 0.12.6", + "prost 0.13.1", + "prost-types 0.13.1", "serde", "serde-value", ] @@ -7203,6 +7250,15 @@ dependencies = [ "prost 0.12.6", ] +[[package]] +name = "prost-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +dependencies = [ + "prost 0.13.1", +] + [[package]] name = "psl" version = "2.1.22" @@ -7563,6 +7619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", + "curl-sys", "libc", "libz-sys", "num_enum 0.5.11", @@ -8440,7 +8497,7 @@ version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "itoa", "memchr", "ryu", @@ -8538,7 +8595,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_derive", "serde_json", @@ -8588,7 +8645,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "itoa", "ryu", "serde", @@ -9575,7 +9632,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.18", ] @@ -9586,7 +9643,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.18", ] @@ -9597,7 +9654,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.18", ] @@ -9608,45 +9665,13 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_spanned", "toml_datetime", "winnow 0.6.18", ] -[[package]] -name = "tonic" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.21.7", - "bytes 1.7.1", - "flate2", - "h2 0.3.26", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.12.6", - "rustls 0.21.11", - "rustls-native-certs 0.6.3", - "rustls-pemfile 1.0.3", - "tokio", - "tokio-rustls 0.24.1", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing 0.1.40", -] - [[package]] name = "tonic" version = "0.11.0" @@ -9693,19 +9718,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "tonic-build" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" -dependencies = [ - "prettyplease 0.2.15", - "proc-macro2 1.0.86", - "prost-build 0.12.6", - "quote 1.0.37", - "syn 2.0.75", -] - [[package]] name = "tonic-build" version = "0.11.0" @@ -10326,7 +10338,7 @@ dependencies = [ "dunce", "glob", "hex", - "indexmap 2.2.6", + "indexmap 2.4.0", "indicatif", "itertools 0.13.0", "log", @@ -10428,7 +10440,7 @@ dependencies = [ "hyper 0.14.28", "hyper-openssl", "hyper-proxy", - "indexmap 2.2.6", + "indexmap 2.4.0", "indoc", "infer 0.16.0", "inventory", @@ -10513,8 +10525,8 @@ dependencies = [ "tokio-tungstenite 0.20.1", "tokio-util", "toml", - "tonic 0.10.2", - "tonic-build 0.10.2", + "tonic", + "tonic-build 0.11.0", "tower", "tower-http", "tower-test", @@ -10613,7 +10625,7 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.30", - "indexmap 2.2.6", + "indexmap 2.4.0", "metrics", "nom", "ordered-float 4.2.2", @@ -10644,7 +10656,7 @@ dependencies = [ "chrono-tz", "encoding_rs", "http 0.2.9", - "indexmap 2.2.6", + "indexmap 2.4.0", "inventory", "no-proxy", "num-traits", @@ -10715,7 +10727,7 @@ dependencies = [ "headers", "http 0.2.9", "hyper-proxy", - "indexmap 2.2.6", + "indexmap 2.4.0", "ipnet", "metrics", "metrics-tracing-context", @@ -10757,7 +10769,7 @@ dependencies = [ "tokio-test", "tokio-util", "toml", - "tonic 0.10.2", + "tonic", "tracing 0.1.40", "tracing-core 0.1.32", "tracing-subscriber", @@ -10910,8 +10922,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84451e6a979d54175932cfe49f36f31ab7a483f3feb3cfa65ee584b91ef2f858" +source = "git+https://github.com/vectordotdev/vrl?rev=bab1cd50658f8a50008902e96e9fa4e5c5a9547c#bab1cd50658f8a50008902e96e9fa4e5c5a9547c" dependencies = [ "aes", "ansi_term", @@ -10931,6 +10942,7 @@ dependencies = [ "clap", "codespan-reporting", "community-id", + "convert_case 0.6.0", "crypto_secretbox", "csv", "ctr", @@ -10947,11 +10959,12 @@ dependencies = [ "hostname 0.4.0", "iana-time-zone", "idna 0.5.0", - "indexmap 2.2.6", + "indexmap 2.4.0", "indoc", + "influxdb-line-protocol", "itertools 0.13.0", "lalrpop", - "lalrpop-util", + "lalrpop-util 0.21.0", "md-5", "mlua", "nom", @@ -10966,7 +10979,7 @@ dependencies = [ "pest_derive", "prettydiff", "prettytable-rs", - "prost 0.12.6", + "prost 0.13.1", "prost-reflect", "psl", "psl-types", diff --git a/Cargo.toml b/Cargo.toml index c02f671a1f0ae..ae4a6c2b351a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,19 +131,22 @@ members = [ [workspace.dependencies] chrono = { version = "0.4.38", default-features = false, features = ["clock", "serde"] } -clap = { version = "4.5.10", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } -indexmap = { version = "2.2.6", default-features = false, features = ["serde", "std"] } +clap = { version = "4.5.16", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +indexmap = { version = "2.4.0", default-features = false, features = ["serde", "std"] } metrics = "0.22.3" metrics-tracing-context = { version = "0.15.0", default-features = false } metrics-util = { version = "0.16.3", default-features = false, features = ["registry"] } pin-project = { version = "1.1.5", default-features = false } proptest = { version = "1.5" } proptest-derive = { version = "0.4.0" } +prost-reflect = { version = "0.14", default-features = false } serde_json = { version = "1.0.125", default-features = false, features = ["raw_value", "std"] } -serde = { version = "1.0.206", default-features = false, features = ["alloc", "derive", "rc"] } +serde = { version = "1.0.209", default-features = false, features = ["alloc", "derive", "rc"] } toml = { version = "0.8.19", default-features = false, features = ["display", "parse"] } +tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } +tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } uuid = { version = "1.10.0", features = ["v4", "v7", "serde"] } -vrl = { version = "0.17.0", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl", rev = "bab1cd50658f8a50008902e96e9fa4e5c5a9547c", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] pin-project.workspace = true @@ -166,7 +169,7 @@ loki-logproto = { path = "lib/loki-logproto", optional = true } async-stream = { version = "0.3.5", default-features = false } async-trait = { version = "0.1.81", default-features = false } futures = { version = "0.3.30", default-features = false, features = ["compat", "io-compat"], package = "futures" } -tokio = { version = "1.39.2", default-features = false, features = ["full"] } +tokio = { version = "1.39.3", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.4", default-features = false } tokio-stream = { version = "0.1.15", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } @@ -233,7 +236,7 @@ rmpv = { version = "1.3.0", default-features = false, features = ["with-serde"], # Prost / Protocol Buffers prost = { version = "0.12", default-features = false, features = ["std"] } -prost-reflect = { version = "0.13", default-features = false, optional = true } +prost-reflect = { workspace = true, optional = true } prost-types = { version = "0.12", default-features = false, optional = true } # GCP @@ -282,12 +285,12 @@ dyn-clone = { version = "1.0.17", default-features = false } encoding_rs = { version = "0.8.34", default-features = false, features = ["serde"] } enum_dispatch = { version = "0.3.13", default-features = false } exitcode = { version = "1.1.2", default-features = false } -flate2 = { version = "1.0.31", default-features = false, features = ["default"] } +flate2 = { version = "1.0.32", default-features = false, features = ["default"] } futures-util = { version = "0.3.29", default-features = false } glob = { version = "0.3.1", default-features = false } governor = { version = "0.6.3", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } grok = { version = "2.0.0", default-features = false, optional = true } -h2 = { version = "0.4.5", default-features = false, optional = true } +h2 = { version = "0.4.6", default-features = false, optional = true } hash_hasher = { version = "2.0.0", default-features = false } hashbrown = { version = "0.14.5", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.9", default-features = false } @@ -326,7 +329,7 @@ postgres-openssl = { version = "0.5.0", default-features = false, features = ["r pulsar = { version = "6.3.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } rand_distr = { version = "0.4.3", default-features = false } -rdkafka = { version = "0.35.0", default-features = false, features = ["tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.35.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.24.0", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex = { version = "1.10.6", default-features = false, features = ["std", "perf"] } roaring = { version = "0.10.6", default-features = false, features = ["std"], optional = true } @@ -344,7 +347,7 @@ tikv-jemallocator = { version = "0.6.0", default-features = false, features = [" tokio-postgres = { version = "0.7.11", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = {version = "0.20.1", default-features = false, features = ["connect"], optional = true} toml.workspace = true -tonic = { version = "0.10", optional = true, default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } +tonic = { workspace = true, optional = true } hickory-proto = { version = "0.24.1", default-features = false, features = ["dnssec"], optional = true } typetag = { version = "0.2.18", default-features = false } url = { version = "2.5.2", default-features = false, features = ["serde"] } @@ -367,7 +370,7 @@ nix = { version = "0.26.2", default-features = false, features = ["socket", "sig [build-dependencies] prost-build = { version = "0.12", default-features = false, optional = true } -tonic-build = { version = "0.10", default-features = false, features = ["transport", "prost"], optional = true } +tonic-build = { workspace = true, optional = true } # update 'openssl_version' in website/config.toml whenever version changes openssl-src = { version = "300", default-features = false, features = ["force-engine", "legacy"] } @@ -382,7 +385,7 @@ azure_storage = { version = "0.17", default-features = false } base64 = "0.22.1" criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.13.0", default-features = false, features = ["use_alloc"] } -libc = "0.2.156" +libc = "0.2.158" similar-asserts = "1.5.0" proptest.workspace = true quickcheck = "1.0.3" @@ -390,7 +393,7 @@ reqwest = { version = "0.11", features = ["json"] } rstest = {version = "0.22.0"} tempfile = "3.12.0" test-generator = "0.3.1" -tokio = { version = "1.39.2", features = ["test-util"] } +tokio = { version = "1.39.3", features = ["test-util"] } tokio-test = "0.4.4" tower-test = "0.4.0" vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl", "test"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ea585e64fa881..c15a7d3a023ce 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -3,6 +3,7 @@ Inflector,https://github.com/whatisinternet/inflector,BSD-2-Clause,Josh Teeter addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line Authors adler,https://github.com/jonas-schievink/adler,0BSD OR MIT OR Apache-2.0,Jonas Schievink +adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " adler32,https://github.com/remram44/adler32-rs,Zlib,Remi Rampin aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers aes,https://github.com/RustCrypto/block-ciphers,MIT OR Apache-2.0,RustCrypto Developers @@ -160,6 +161,7 @@ crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers +curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs diff --git a/changelog.d/20994_honeycomb_sink_support_configurable_endpoint.feature.md b/changelog.d/20994_honeycomb_sink_support_configurable_endpoint.feature.md new file mode 100644 index 0000000000000..cd61225913ac2 --- /dev/null +++ b/changelog.d/20994_honeycomb_sink_support_configurable_endpoint.feature.md @@ -0,0 +1,3 @@ +Adds support for configuring the Honeycomb.io endpoint, allowing users to send data to the EU endpoint in addition to the default US endpoint. + +authors: raytung diff --git a/changelog.d/21067_reduce.fix.md b/changelog.d/21067_reduce.fix.md deleted file mode 100644 index 67af1e4e16374..0000000000000 --- a/changelog.d/21067_reduce.fix.md +++ /dev/null @@ -1 +0,0 @@ -Fixes a Vector v0.40.0 regression where the `reduce` transform would not group top level objects correctly. diff --git a/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md b/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md new file mode 100644 index 0000000000000..3b834393df8bf --- /dev/null +++ b/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md @@ -0,0 +1,3 @@ +The `kafka` sink now supports OIDC authentication through the exposed `librdkafka_options`. + +authors: zapdos26 diff --git a/changelog.d/21129_suppress_warnings_for_kafka_sink.fix.md b/changelog.d/21129_suppress_warnings_for_kafka_sink.fix.md new file mode 100644 index 0000000000000..843296f7f0540 --- /dev/null +++ b/changelog.d/21129_suppress_warnings_for_kafka_sink.fix.md @@ -0,0 +1,3 @@ +The `kafka` sink no longer emits warnings due to applying rdkafka options to a consumer used for the health check. Now it uses the producer client for the health check. + +authors: belltoy diff --git a/changelog.d/21141_fix_crash_when_handling_metrics.fix.md b/changelog.d/21141_fix_crash_when_handling_metrics.fix.md new file mode 100644 index 0000000000000..1e26d861728b4 --- /dev/null +++ b/changelog.d/21141_fix_crash_when_handling_metrics.fix.md @@ -0,0 +1,3 @@ +The `amqp` source no longer panics when deserializing metrics (e.g. via the `native_json` codec). + +authors: kghost diff --git a/changelog.d/file-sources-exclude-patterns-not-match-some-files.fix.md b/changelog.d/file-sources-exclude-patterns-not-match-some-files.fix.md new file mode 100644 index 0000000000000..be2d139430c60 --- /dev/null +++ b/changelog.d/file-sources-exclude-patterns-not-match-some-files.fix.md @@ -0,0 +1,3 @@ +The `file` source now properly handle exclude patterns with multiple slashes when matching files. + +authors: suikammd diff --git a/changelog.d/support-config-watch-for-windows.enhancement.md b/changelog.d/support-config-watch-for-windows.enhancement.md new file mode 100644 index 0000000000000..f49cbb60c7315 --- /dev/null +++ b/changelog.d/support-config-watch-for-windows.enhancement.md @@ -0,0 +1,4 @@ +Windows now supports the -w command line parameter just like every other +platform and will reload the configuration files on any change + +authors: darklajid diff --git a/distribution/install.sh b/distribution/install.sh index 58bfe4e0cd13a..b1298945c8d58 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.40.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.40.1"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index ea028bc772dab..b668665d1c59b 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -22,7 +22,7 @@ memchr = { version = "2", default-features = false } once_cell = { version = "1.19", default-features = false } ordered-float = { version = "4.2.2", default-features = false } prost = { version = "0.12.6", default-features = false, features = ["std"] } -prost-reflect = { version = "0.13", default-features = false, features = ["serde"] } +prost-reflect = { workspace = true, features = ["serde"] } regex = { version = "1.10.6", default-features = false, features = ["std", "perf"] } serde.workspace = true serde_json.workspace = true diff --git a/lib/codecs/src/decoding/framing/character_delimited.rs b/lib/codecs/src/decoding/framing/character_delimited.rs index 1a6da25ff5883..bea46b49d0c43 100644 --- a/lib/codecs/src/decoding/framing/character_delimited.rs +++ b/lib/codecs/src/decoding/framing/character_delimited.rs @@ -39,6 +39,7 @@ impl CharacterDelimitedDecoderConfig { #[derive(Clone, Debug, PartialEq, Eq)] pub struct CharacterDelimitedDecoderOptions { /// The character that delimits byte sequences. + #[configurable(metadata(docs::type_override = "ascii_char"))] #[serde(with = "vector_core::serde::ascii_char")] pub delimiter: u8, diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index 1313124d0bcb7..1835540eaef20 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -1,7 +1,6 @@ use crate::encoding::BuildError; use bytes::BytesMut; -use prost::Message; -use prost_reflect::MessageDescriptor; +use prost_reflect::{prost::Message as _, MessageDescriptor}; use std::path::PathBuf; use tokio_util::codec::Encoder; use vector_core::{ diff --git a/lib/codecs/src/encoding/framing/character_delimited.rs b/lib/codecs/src/encoding/framing/character_delimited.rs index 2e536976618b8..257a5b4319d64 100644 --- a/lib/codecs/src/encoding/framing/character_delimited.rs +++ b/lib/codecs/src/encoding/framing/character_delimited.rs @@ -31,6 +31,7 @@ impl CharacterDelimitedEncoderConfig { #[derive(Clone, Debug, Eq, PartialEq)] pub struct CharacterDelimitedEncoderOptions { /// The ASCII (7-bit) character that delimits byte sequences. + #[configurable(metadata(docs::type_override = "ascii_char"))] #[serde(with = "vector_core::serde::ascii_char")] pub delimiter: u8, } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index ce46a5b0f6267..bb7dcab6ce6ee 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -69,7 +69,7 @@ default-features = false features = [] [dependencies.tokio] -version = "1.39.2" +version = "1.39.3" default-features = false features = ["full"] diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index 8672b2d6e826c..cdb2fb90c5bbc 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,7 +14,7 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.11.26", features = ["json"] } serde_json.workspace = true -tokio = { version = "1.39.2", features = ["full"] } +tokio = { version = "1.39.3", features = ["full"] } indoc = "2.0.5" env_logger = "0.11" tracing = { version = "0.1", features = ["log"] } diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index 2db5e9a2486b7..e6dae00f0acbb 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json.workspace = true tempfile = "3" -tokio = { version = "1.39.2", features = ["full"] } +tokio = { version = "1.39.3", features = ["full"] } log = "0.4" diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index d18b084879418..02c1b93272d4b 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -7,7 +7,7 @@ publish = false [build-dependencies] prost-build = { version = "0.12", default-features = false} -tonic-build = { version = "0.10", default-features = false, features = ["prost", "transport"] } +tonic-build.workspace = true [dependencies] bytes = { version = "1.7.1", default-features = false, features = ["serde"] } @@ -16,6 +16,6 @@ hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } ordered-float = { version = "4.2.2", default-features = false } prost = { version = "0.12", default-features = false, features = ["std"] } -tonic = { version = "0.10", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } +tonic.workspace = true vrl.workspace = true vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 5fc2cb1463357..10e663e3df4ac 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -17,7 +17,7 @@ anyhow = { version = "1.0.86", default-features = false, features = ["std"] } # Tokio / Futures futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } -tokio = { version = "1.39.2", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.39.3", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.15", default-features = false, features = ["sync"] } # GraphQL diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index a16f58a6e07b5..e5b8b9124c89d 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -22,11 +22,11 @@ metrics.workspace = true num-traits = { version = "0.2.19", default-features = false } paste = "1.0.15" pin-project.workspace = true -rkyv = { version = "0.7.44", default-features = false, features = ["size_32", "std", "strict", "validation"] } +rkyv = { version = "0.7.45", default-features = false, features = ["size_32", "std", "strict", "validation"] } serde.workspace = true snafu = { version = "0.7.5", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.39.2", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.39.3", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index dce292de31f1e..a239317e52e16 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -58,7 +58,7 @@ serde_json.workspace = true smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.2", default-features = false } -tokio = { version = "1.39.2", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.39.3", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } vrl.workspace = true vector-config = { path = "../vector-config" } @@ -67,6 +67,6 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] futures = { version = "0.3.30", default-features = false, features = ["async-await", "std"] } -tokio = { version = "1.39.2", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.39.3", default-features = false, features = ["rt", "time"] } quickcheck = "1" quickcheck_macros = "1" diff --git a/lib/vector-config/src/external/vrl.rs b/lib/vector-config/src/external/vrl.rs index 45303c52ce887..b84b591efc3b9 100644 --- a/lib/vector-config/src/external/vrl.rs +++ b/lib/vector-config/src/external/vrl.rs @@ -1,7 +1,7 @@ use std::cell::RefCell; use serde_json::Value; -use vrl::compiler::VrlRuntime; +use vrl::{compiler::VrlRuntime, datadog_search_syntax::QueryNode}; use crate::{ schema::{generate_string_schema, SchemaGenerator, SchemaObject}, @@ -27,3 +27,12 @@ impl ToValue for VrlRuntime { }) } } + +impl Configurable for QueryNode { + fn generate_schema(_gen: &RefCell) -> Result + where + Self: Sized, + { + Ok(generate_string_schema()) + } +} diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index be58347614ee6..4ad4ad17559dd 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -48,12 +48,12 @@ serde_with = { version = "3.9.0", default-features = false, features = ["std", " smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu = { version = "0.7.5", default-features = false } socket2 = { version = "0.5.7", default-features = false } -tokio = { version = "1.39.2", default-features = false, features = ["net"] } +tokio = { version = "1.39.3", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.4", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } toml.workspace = true -tonic = { version = "0.10", default-features = false, features = ["transport"] } +tonic.workspace = true tracing = { version = "0.1.34", default-features = false } tracing-core = { version = "0.1.26", default-features = false } tracing-subscriber = { version = "0.3.18", default-features = false, features = ["std"] } @@ -88,7 +88,7 @@ proptest = "1.5" similar-asserts = "1.5.0" tokio-test = "0.4.4" toml.workspace = true -ndarray = "0.16.0" +ndarray = "0.16.1" ndarray-stats = "0.5.1" noisy_float = "0.2.0" rand = "0.8.5" diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 398cf876f3381..297fbccd46ebc 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -10,7 +10,7 @@ async-stream = { version = "0.3.5", default-features = false } futures = { version = "0.3.30", default-features = false, features = ["std"] } futures-util = { version = "0.3.29", default-features = false, features = ["std"] } pin-project.workspace = true -tokio = { version = "1.39.2", default-features = false, features = ["net"] } +tokio = { version = "1.39.3", default-features = false, features = ["net"] } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } tower = { version = "0.4", default-features = false, features = ["util"] } tracing = { version = "0.1.34", default-features = false } diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index ccd8b35a5d42c..7a2cb90ad4b8d 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -10,7 +10,7 @@ license = "MPL-2.0" colored = { version = "2.1.0", default-features = false } serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.7.5", default-features = false } -tokio = { version = "1.39.2", default-features = false, features = ["time"] } +tokio = { version = "1.39.3", default-features = false, features = ["time"] } tokio-stream = { version = "0.1.15", default-features = false, features = ["sync"] } tokio-tungstenite = { version = "0.20.1", default-features = false } tracing = { version = "0.1.34", default-features = false } @@ -22,7 +22,7 @@ futures-util = "0.3.30" chrono = { workspace = true } portpicker = { path = "../portpicker" } serde_json = { workspace = true } -tokio = { version = "1.39.2", default-features = false, features = ["test-util"] } +tokio = { version = "1.39.3", default-features = false, features = ["test-util"] } diff --git a/lib/vector-vrl/web-playground/public/assets/dd_icon_rgb.svg b/lib/vector-vrl/web-playground/public/assets/dd_icon_rgb.svg new file mode 100644 index 0000000000000..f39a6ac1b0bc6 --- /dev/null +++ b/lib/vector-vrl/web-playground/public/assets/dd_icon_rgb.svg @@ -0,0 +1,41 @@ + + + + + + diff --git a/lib/vector-vrl/web-playground/public/index.css b/lib/vector-vrl/web-playground/public/index.css index 296b067889097..caaacc2cd0d3a 100644 --- a/lib/vector-vrl/web-playground/public/index.css +++ b/lib/vector-vrl/web-playground/public/index.css @@ -1,329 +1,375 @@ -body { - margin-right: 2vw; - margin-left: 2vw; - font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, - Helvetica Neue, Arial, Noto Sans, sans-serif, Apple Color Emoji, Segoe UI Emoji, - Segoe UI Symbol, Noto Color Emoji; +:root { + --datadog-purple-lightest: #f8f6ff; + --datadog-purple-light: #e1d8ff; + --datadog-purple: #632ca6; + --datadog-purple-vibrant: #7b40f1; + --datadog-purple-dark: #451b78; + --datadog-gray: #252525; + --datadog-gray-light: #b6b6b6; + --datadog-gray-lighter: #eaeaea; + --datadog-background: #fafbfc; + --light-grey: #f5f5f5; } -table { - width: 100%; - border-spacing: 10px; +body { + margin: 0; + padding: 0; + font-family: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Helvetica Neue", sans-serif; + background-color: var(--datadog-background); + color: var(--datadog-gray); } -td { - text-align: left; - font-weight: bold; +.top-bar-wrapper { + border-bottom: 1px solid var(--datadog-purple-light); + padding: 10px 0; } - .headers-grid { display: grid; - grid-template-columns: 3fr 2fr 7fr; - grid-template-rows: 90px; - gap: 10px; - width: 100%; - height: 100%; - margin: auto; + grid-template-columns: auto 1fr auto; + gap: 20px; + padding: 0 15px; + max-width: 1600px; + margin: 0 auto; + align-items: stretch; } .headers-grid-item { - background-color: #dfd8ec; - padding: 5px 10px; - border-radius: 4px; - border: none; - display: grid; - align-items: center; + background-color: var(--datadog-purple-lightest); + padding: 16px; + border-radius: 12px; + border: 1px solid var(--datadog-purple-light); + display: flex; + flex-direction: column; justify-content: center; - height: 100%; +} + +.title-section { + flex-direction: row; + align-items: center; + gap: 8px; +} + +#logo { + width: 20px; + height: 20px; +} + +.headers-grid-item h2 { + font-size: 22px; + font-weight: 600; + color: var(--datadog-purple); + margin: 0; } #description-cell { - grid-column: 3; - display: grid; + grid-column: 2; +} + +#description-cell p { + font-size: 13px; + line-height: 1.4; + margin: 0; + color: var(--datadog-gray); +} + +#description-cell p, +.version-info td { + color: var(--datadog-gray); +} + +.version-info { + justify-content: center; +} + +.version-info table { + width: 100%; + border-spacing: 0; +} + +.version-info tr { + display: flex; + justify-content: space-between; + align-items: baseline; +} + +.version-info td { + padding: 2px 0; + font-weight: normal; + font-size: 12px; + color: var(--datadog-gray); +} + +.version-info td:first-child { + font-weight: 600; +} + +#description-cell a:hover, +.version-info a:hover { + text-decoration: underline; } div#App { - padding-top: 5px; display: grid; - width: 100%; - height: 100%; - grid-template-columns: repeat(2, 1fr); - grid-template-rows: 1fr 18fr; + width: calc(100% - 40px); + max-width: 1600px; + height: calc(100vh - 240px); + grid-template-columns: minmax(45%, 1fr) minmax(0, 1fr); + grid-template-rows: auto 1fr; grid-gap: 1rem; - resize: both; - overflow: hidden; + padding: 20px 20px; + margin: 0 auto; } div#toolbar-section { - padding-top: 30px; - display: grid; + padding: 10px 0; + display: flex; + flex-wrap: wrap; grid-row: 1; - grid-column: 1 / span 2; - grid-template-columns: 2fr 2fr 6fr; - grid-gap: 1rem; + grid-column: 1 / -1; + gap: 1rem; align-items: center; } - #toolbar-section #run-code-btn { - grid-column: 1; - height: 40px; +#toolbar-section #run-code-btn, +#toolbar-section #share-code-btn { + height: 36px; + flex: 0 1 auto; + min-width: 120px; +} + +div#input-section, +div#output-section { + border: 1px solid var(--datadog-gray-lighter); + border-radius: 4px; +} + +div#input-section { + display: grid; + grid-column: 1; + grid-row: 2; +} + +div#output-section { + display: grid; + grid-column: 2; + grid-row: 2; + grid-template-rows: 1fr 1fr; + gap: 1rem; +} + +#input-section #cell, +#output-section #event-cell, +#output-section #output-cell { + display: grid; + grid-template-rows: auto 1fr; +} + +.cell-title { + font-weight: 600; + font-size: 14px; + color: var(--datadog-gray); + margin: 0; + padding: 8px 12px; + background-color: var(--datadog-gray-lighter); + border-bottom: 1px solid var(--datadog-gray-light); +} + +/* BUTTONS */ +.btn { + display: inline-block; + outline: 0; + border: none; + cursor: pointer; + border-radius: 4px; + font-size: 14px; + height: 36px; + padding: 0 16px; + font-weight: 500; + transition: all 0.2s ease; +} + +.btn:active { + transform: translateY(1px); +} +.btn-primary { + background-color: var(--datadog-purple); + color: var(--datadog-background); +} + +.btn-primary:hover { + background-color: var(--datadog-purple-dark); +} + +.btn-secondary { + background-color: var(--datadog-gray-lighter); + color: var(--datadog-gray); + border: 1px solid var(--datadog-gray-light); +} + +.btn-secondary:hover { + background-color: var(--datadog-gray-light); +} + +/* Media Queries for Responsiveness */ +@media only screen and (min-width: 1601px) { + .headers-grid, + div#App { + max-width: 1800px; } - #toolbar-section #share-code-btn { - grid-column: 2; - height: 40px; + .headers-grid { + padding: 20px 30px; } - /* input pane */ - div#input-section { - display: grid; - grid-column: 1; - grid-row: 2; - overflow: hidden; + div#App { + padding: 30px 30px; } - #input-section #cell { - display: grid; - grid-template-rows: 4% 96%; - overflow: hidden; + .headers-grid-item h2 { + font-size: 24px; } - #input-section #cell #input-cell-title { - height: 100%; - grid-row: 1; - font-weight: bold; + #description-cell p { + font-size: 16px; } +} - #input-section #cell #container-program { - display: grid; - height: 100%; +@media only screen and (min-width: 1201px) and (max-width: 1600px) { + .headers-grid, + div#App { + max-width: 1400px; } - div#output-section { - display: grid; - grid-column: 2; - grid-row: 2; - grid-template-rows: 30% 60%; + .headers-grid { + padding: 18px 25px; } - /* event pane */ - #output-section #event-cell { - display: grid; - /* this is double of input-cell-title row - because we are working with half the space */ - grid-template-rows: 12% 88%; - grid-row: 1; - height: 100%; + div#App { + padding: 25px 25px; } - #output-section #event-cell #event-cell-title { - display: grid; - grid-row: 1; - font-weight: bold; + .headers-grid-item h2 { + font-size: 22px; } - #output-section #event-cell #container-event { - display: grid; - grid-row: 2; - height: 100%; + #description-cell p { + font-size: 15px; } +} - /* output pane */ - #output-section #output-cell { - display: grid; - grid-template-rows: 6% 94%; - grid-row: 2; - height: 100%; +@media only screen and (max-width: 1200px) { + .headers-grid, + div#App { + max-width: 1000px; + } +} + +@media only screen and (max-width: 1024px) { + .headers-grid { + grid-template-columns: 1fr; + gap: 10px; + } + + .headers-grid-item { + width: 100%; + box-sizing: border-box; + } + + #description-cell { + grid-column: 1; } - #output-section #output-cell #output-cell-title { - display: grid; - grid-row: 1; - font-weight: bold; - height: 50px; + div#App { + grid-template-columns: 1fr; + height: auto; + min-height: calc(100vh - 180px); } - #output-section #output-cell #container-output { - display: grid; + div#input-section { + grid-column: 1; grid-row: 2; - height: 100%; + height: 40vh; } -/* BUTTONS */ -.btn { - display: inline-block; - outline: 0; - border: none; - cursor: pointer; - border-radius: 8px; - font-size: 13px; - height: 30px; - padding: 0 20px; -} + div#output-section { + grid-column: 1; + grid-row: 3; + grid-template-rows: auto auto; + grid-template-columns: 1fr; + height: auto; + } -.btn:active { - box-shadow: 0 4px #666; - transform: translateY(2px); + #output-section #event-cell, + #output-section #output-cell { + height: 30vh; + } } -.btn-primary { - background-color: #9147ff; - color: white; -} +@media only screen and (max-width: 768px) { + .headers-grid, + div#App { + width: calc(100% - 20px); + padding-left: 10px; + padding-right: 10px; + } -.btn-primary:hover { - background-color: #6a1ae1; + .headers-grid { + grid-template-columns: 1fr; + gap: 10px; + } + + .headers-grid-item h2 { + font-size: 18px; + } + + #toolbar-section #run-code-btn, + #toolbar-section #share-code-btn { + flex: 1 1 auto; + } } -.btn-secondary { - background-color: #0000001a; - color: #000000; +@media only screen and (max-width: 480px) { + .headers-grid, + div#App { + width: calc(100% - 16px); + padding-left: 8px; + padding-right: 8px; + } + + .headers-grid-item h2 { + font-size: 16px; + } + + #description-cell p { + font-size: 12px; + } + + div#toolbar-section { + padding: 10px 0; + } + + #toolbar-section #run-code-btn, + #toolbar-section #share-code-btn { + width: 100%; + margin-bottom: 8px; + } } -.btn-secondary:hover { - background-color: #c0bdbd; -} -/* END OF BUTTONS */ - -/* Portrait and Landscape */ -@media only screen - and (min-width: 200px) - and (max-width: 1000px) { - div#App { - display: grid; - width: 100%; - height: 100%; - /* the app will have multiple rows - stacking each section on top of each other */ - grid-template-rows: 20vh 10vh 50vh; - grid-template-columns: 100%; - } - - div#toolbar-section { - display: grid; - grid-row: 1; - grid-column: 1; - grid-template-columns: 100%; - grid-template-rows: repeat(2, 1fr); - } - - #toolbar-section #run-code-btn { - display: grid; - grid-row: 1; - grid-column: 1; - } - - #toolbar-section #share-code-btn { - display: grid; - grid-row: 2; - grid-column: 1; - } - - /* input pane */ - div#input-section { - display: grid; - grid-column: 1; - grid-row: 2; - overflow: hidden; - } - - #input-section #cell { - display: grid; - grid-column: 1; - grid-template-rows: 23% 77%; - overflow: hidden; - } - - #input-section #cell #input-cell-title { - height: 100%; - grid-column: 1; - grid-row: 1; - } - - #input-section #cell #container-program { - display: grid; - height: 100%; - } - - div#output-section { - display: grid; - grid-row: 4 / 5; - grid-column: 1; - grid-template-rows: 50% 50%; - overflow: hidden; - } - - /* event pane */ - #output-section #event-cell { - display: grid; - grid-template-rows: 20% 80%; - grid-row: 1; - height: 100%; - } - - #output-section #event-cell #event-cell-title { - display: grid; - grid-row: 1; - } - - #output-section #event-cell #container-event { - display: grid; - height: 100%; - } - - /* output pane */ - #output-section #output-cell { - display: grid; - grid-template-rows: 20% 80%; - grid-row: 2; - height: 100%; - } - - #output-section #output-cell #output-cell-title { - display: grid; - grid-row: 1; - } - - #output-section #output-cell #container-output { - display: grid; - grid-row: 2; - height: 100%; - } - - .btn-primary { - display: inline-block; - outline: 0; - border: none; - cursor: pointer; - border-radius: 4px; - font-size: 13px; - height: 30px; - background-color: #9147ff; - color: white; - padding: 0 20px; - align-items: center; - } - - .btn-primary:hover { - background-color: #772ce8; - } - - .btn-secondary { - display: inline-block; - outline: 0; - border: none; - cursor: pointer; - border-radius: 4px; - font-size: 13px; - height: 30px; - background-color: #0000001a; - color: #000000; - padding: 0 20px; - align-items: center; - } - - .btn-secondary:hover { - background-color: #dcdcdc; - } +@media only screen and (max-width: 320px) { + .headers-grid, + div#App { + width: calc(100% - 10px); + padding-left: 12px; + padding-right: 12px; + } + + .headers-grid-item h2 { + font-size: 14px; + } + + #description-cell p { + font-size: 11px; + } } diff --git a/lib/vector-vrl/web-playground/public/index.html b/lib/vector-vrl/web-playground/public/index.html index 8fd0d464d88b1..9799dcd4a4e76 100644 --- a/lib/vector-vrl/web-playground/public/index.html +++ b/lib/vector-vrl/web-playground/public/index.html @@ -22,33 +22,34 @@ -
-
-

VRL Playground

-
-
- - - - - - - - - -
Vector Version:
VRL Version:
-
-
-

+

+
+
+ +

VRL Playground

+
+
+

Vector Remap Language (VRL) is an expression-oriented language designed for transforming observability data. This playground lets you write a program, run it against an event or events, share it, and see how the events are transformed.

+
+
+ + + + + + + + + +
Vector Version:
VRL Version:
+
- -
@@ -59,7 +60,6 @@

VRL Playground

Program

-
diff --git a/scripts/generate-component-docs.rb b/scripts/generate-component-docs.rb index 8f55d3e048a50..9972b897528b1 100755 --- a/scripts/generate-component-docs.rb +++ b/scripts/generate-component-docs.rb @@ -753,7 +753,11 @@ def resolve_schema(root_schema, schema) type_override = get_schema_metadata(schema, 'docs::type_override') if !type_override.nil? if type_override == 'ascii_char' - resolved = { 'type' => { type_override.to_s => { 'default' => schema['default'].chr } } } + if !schema['default'].nil? + resolved = { 'type' => { type_override.to_s => { 'default' => schema['default'].chr } } } + else + resolved = { 'type' => { type_override.to_s => { } } } + end else resolved = { 'type' => { type_override.to_s => {} } } end diff --git a/scripts/integration/aws/compose.yaml b/scripts/integration/aws/compose.yaml index 07de953aaed52..c45cbe7f2e37b 100644 --- a/scripts/integration/aws/compose.yaml +++ b/scripts/integration/aws/compose.yaml @@ -6,9 +6,7 @@ services: mock-localstack: image: docker.io/localstack/localstack:3 environment: - - SERVICES=kinesis,s3,cloudwatch,es,firehose,sqs,sns - mock-watchlogs: - image: docker.io/luciofranco/mockwatchlogs:latest + - SERVICES=kinesis,s3,cloudwatch,es,firehose,sqs,sns,logs mock-ecs: image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest volumes: diff --git a/scripts/integration/aws/test.yaml b/scripts/integration/aws/test.yaml index 706efe5ba6dd4..71ff65cae11f8 100644 --- a/scripts/integration/aws/test.yaml +++ b/scripts/integration/aws/test.yaml @@ -14,7 +14,6 @@ env: S3_ADDRESS: http://mock-localstack:4566 SQS_ADDRESS: http://mock-localstack:4566 SNS_ADDRESS: http://mock-localstack:4566 - WATCHLOGS_ADDRESS: http://mock-watchlogs:6000 matrix: version: [latest] diff --git a/src/app.rs b/src/app.rs index b526f1fb0f686..e27d68e5a6cab 100644 --- a/src/app.rs +++ b/src/app.rs @@ -476,12 +476,15 @@ pub async fn load_configs( if watch_config { // Start listening for config changes immediately. - config::watcher::spawn_thread(config_paths.iter().map(Into::into), None).map_err( - |error| { - error!(message = "Unable to start config watcher.", %error); - exitcode::CONFIG - }, - )?; + config::watcher::spawn_thread( + signal_handler.clone_tx(), + config_paths.iter().map(Into::into), + None, + ) + .map_err(|error| { + error!(message = "Unable to start config watcher.", %error); + exitcode::CONFIG + })?; } info!( diff --git a/src/components/validation/runner/io.rs b/src/components/validation/runner/io.rs index 9e4e51e85651d..d9a49943d32d8 100644 --- a/src/components/validation/runner/io.rs +++ b/src/components/validation/runner/io.rs @@ -5,7 +5,8 @@ use hyper::Body; use tokio::{pin, select, sync::mpsc}; use tonic::{ body::BoxBody, - transport::{Channel, Endpoint, NamedService}, + server::NamedService, + transport::{Channel, Endpoint}, Status, }; use tower::Service; diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index d192f9b910436..3556783f0d546 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -3,28 +3,32 @@ use std::borrow::Cow; use bytes::Bytes; use vector_lib::configurable::configurable_component; use vector_lib::event::{Event, LogEvent, Value}; -use vrl::datadog_filter::{ - build_matcher, - regex::{wildcard_regex, word_regex}, - Filter, Matcher, Resolver, Run, -}; -use vrl::datadog_search_syntax::parse; -use vrl::datadog_search_syntax::{Comparison, ComparisonValue, Field}; +use vrl::datadog_filter::regex::{wildcard_regex, word_regex}; +use vrl::datadog_filter::{build_matcher, Filter, Matcher, Resolver, Run}; +use vrl::datadog_search_syntax::{Comparison, ComparisonValue, Field, QueryNode}; use crate::conditions::{Condition, Conditional, ConditionalConfig}; /// A condition that uses the [Datadog Search](https://docs.datadoghq.com/logs/explorer/search_syntax/) query syntax against an event. #[configurable_component] -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct DatadogSearchConfig { /// The query string. - source: String, + source: QueryNode, +} + +impl Default for DatadogSearchConfig { + fn default() -> Self { + Self { + source: QueryNode::MatchAllDocs, + } + } } impl_generate_config_from_default!(DatadogSearchConfig); /// Runner that contains the boxed `Matcher` function to check whether an `Event` matches -/// a Datadog Search Syntax query. +/// a [Datadog Search Syntax query](https://docs.datadoghq.com/logs/explorer/search_syntax/). #[derive(Debug, Clone)] pub struct DatadogSearchRunner { matcher: Box>, @@ -42,8 +46,7 @@ impl ConditionalConfig for DatadogSearchConfig { &self, _enrichment_tables: &vector_lib::enrichment::TableRegistry, ) -> crate::Result { - let node = parse(&self.source)?; - let matcher = as_log(build_matcher(&node, &EventFilter)); + let matcher = as_log(build_matcher(&self.source, &EventFilter)); Ok(Condition::DatadogSearch(DatadogSearchRunner { matcher })) } @@ -77,7 +80,7 @@ impl Filter for EventFilter { Field::Reserved(field) if field == "tags" => { any_string_match("tags", move |value| value == field) } - Field::Default(f) | Field::Facet(f) | Field::Reserved(f) => { + Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => { Run::boxed(move |log: &LogEvent| { log.parse_path_and_get_value(f.as_str()) .ok() @@ -111,7 +114,7 @@ impl Filter for EventFilter { array_match("tags", move |values| values.contains(&value_bytes)) } // Everything else is matched by string equality. - Field::Reserved(field) | Field::Facet(field) => { + Field::Reserved(field) | Field::Attribute(field) => { let to_match = to_match.to_owned(); string_match(field, move |value| value == to_match) @@ -134,7 +137,7 @@ impl Filter for EventFilter { any_string_match("tags", move |value| value.starts_with(&starts_with)) } // All other field types are compared by complete value. - Field::Reserved(field) | Field::Facet(field) => { + Field::Reserved(field) | Field::Attribute(field) => { let prefix = prefix.to_owned(); string_match(field, move |value| value.starts_with(&prefix)) @@ -154,7 +157,7 @@ impl Filter for EventFilter { any_string_match("tags", move |value| re.is_match(&value)) } - Field::Reserved(field) | Field::Facet(field) => { + Field::Reserved(field) | Field::Attribute(field) => { let re = wildcard_regex(wildcard); string_match(field, move |value| re.is_match(&value)) @@ -171,8 +174,8 @@ impl Filter for EventFilter { let rhs = Cow::from(comparison_value.to_string()); match field { - // Facets are compared numerically if the value is numeric, or as strings otherwise. - Field::Facet(f) => { + // Attributes are compared numerically if the value is numeric, or as strings otherwise. + Field::Attribute(f) => { Run::boxed(move |log: &LogEvent| { match ( log.parse_path_and_get_value(f.as_str()).ok().flatten(), @@ -327,10 +330,9 @@ where #[cfg(test)] mod test { - use serde_json::json; use vector_lib::event::Event; use vrl::datadog_filter::{build_matcher, Filter, Resolver}; - use vrl::datadog_search_syntax::parse; + use vrl::datadog_search_syntax::QueryNode; use super::*; use crate::log_event; @@ -359,23 +361,23 @@ mod test { log_event!["tags" => vec!["b:foo"]], log_event!["tags" => vec!["a:foo"]], ), - // Facet exists. + // Attribute exists. ( "_exists_:@b", - log_event!["custom" => json!({"b": "foo"})], - log_event!["custom" => json!({"a": "foo"})], + log_event!["b" => "foo"], + log_event!["a" => "foo"], ), - // Facet exists (negate). + // Attribute exists (negate). ( "NOT _exists_:@b", - log_event!["custom" => json!({"a": "foo"})], - log_event!["custom" => json!({"b": "foo"})], + log_event!["a" => "foo"], + log_event!["b" => "foo"], ), - // Facet exists (negate w/-). + // Attribute exists (negate w/-). ( "-_exists_:@b", - log_event!["custom" => json!({"a": "foo"})], - log_event!["custom" => json!({"b": "foo"})], + log_event!["a" => "foo"], + log_event!["b" => "foo"], ), // Tag doesn't exist. ( @@ -395,23 +397,23 @@ mod test { log_event!["tags" => vec!["a:foo"]], log_event![], ), - // Facet doesn't exist. + // Attribute doesn't exist. ( "_missing_:@b", - log_event!["custom" => json!({"a": "foo"})], - log_event!["custom" => json!({"b": "foo"})], + log_event!["a" => "foo"], + log_event!["b" => "foo"], ), - // Facet doesn't exist (negate). + // Attribute doesn't exist (negate). ( "NOT _missing_:@b", - log_event!["custom" => json!({"b": "foo"})], - log_event!["custom" => json!({"a": "foo"})], + log_event!["b" => "foo"], + log_event!["a" => "foo"], ), - // Facet doesn't exist (negate w/-). + // Attribute doesn't exist (negate w/-). ( "-_missing_:@b", - log_event!["custom" => json!({"b": "foo"})], - log_event!["custom" => json!({"a": "foo"})], + log_event!["b" => "foo"], + log_event!["a" => "foo"], ), // Keyword. ("bla", log_event!["message" => "bla"], log_event![]), @@ -543,55 +545,55 @@ mod test { ( r#"a:"bla""#, log_event!["tags" => vec!["a:bla"]], - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], ), // Quoted tag match (negate). ( r#"NOT a:"bla""#, - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], log_event!["tags" => vec!["a:bla"]], ), // Quoted tag match (negate w/-). ( r#"-a:"bla""#, - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], log_event!["tags" => vec!["a:bla"]], ), - // Facet match. + // Attribute match. ( "@a:bla", - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], log_event!["tags" => vec!["a:bla"]], ), - // Facet match (negate). + // Attribute match (negate). ( "NOT @a:bla", log_event!["tags" => vec!["a:bla"]], - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], ), - // Facet match (negate w/-). + // Attribute match (negate w/-). ( "-@a:bla", log_event!["tags" => vec!["a:bla"]], - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], ), - // Quoted facet match. + // Quoted attribute match. ( r#"@a:"bla""#, - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], log_event!["tags" => vec!["a:bla"]], ), - // Quoted facet match (negate). + // Quoted attribute match (negate). ( r#"NOT @a:"bla""#, log_event!["tags" => vec!["a:bla"]], - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], ), - // Quoted facet match (negate w/-). + // Quoted attribute match (negate w/-). ( r#"-@a:"bla""#, log_event!["tags" => vec!["a:bla"]], - log_event!["custom" => json!({"a": "bla"})], + log_event!["a" => "bla"], ), // Wildcard prefix. ( @@ -630,23 +632,15 @@ mod test { log_event!["message" => "blafoo"], ), // Multiple wildcards. - ( - "*b*la*", - log_event!["custom" => json!({"title": "foobla"})], - log_event![], - ), + ("*b*la*", log_event!["message" => "foobla"], log_event![]), // Multiple wildcards (negate). ( "NOT *b*la*", log_event![], - log_event!["custom" => json!({"title": "foobla"})], + log_event!["message" => "foobla"], ), // Multiple wildcards (negate w/-). - ( - "-*b*la*", - log_event![], - log_event!["custom" => json!({"title": "foobla"})], - ), + ("-*b*la*", log_event![], log_event!["message" => "foobla"]), // Wildcard prefix - tag. ( "a:*bla", @@ -701,59 +695,59 @@ mod test { log_event!["custom" => r#"{"title": "foobla"}"#], log_event!["tags" => vec!["c:foobla"]], ), - // Wildcard prefix - facet. + // Wildcard prefix - attribute. ( "@a:*bla", - log_event!["custom" => json!({"a": "foobla"})], + log_event!["a" => "foobla"], log_event!["tags" => vec!["a:foobla"]], ), - // Wildcard prefix - facet (negate). + // Wildcard prefix - attribute (negate). ( "NOT @a:*bla", log_event!["tags" => vec!["a:foobla"]], - log_event!["custom" => json!({"a": "foobla"})], + log_event!["a" => "foobla"], ), - // Wildcard prefix - facet (negate w/-). + // Wildcard prefix - attribute (negate w/-). ( "-@a:*bla", log_event!["tags" => vec!["a:foobla"]], - log_event!["custom" => json!({"a": "foobla"})], + log_event!["a" => "foobla"], ), - // Wildcard suffix - facet. + // Wildcard suffix - attribute. ( "@b:bla*", - log_event!["custom" => json!({"b": "blabop"})], + log_event!["b" => "blabop"], log_event!["tags" => vec!["b:blabop"]], ), - // Wildcard suffix - facet (negate). + // Wildcard suffix - attribute (negate). ( "NOT @b:bla*", log_event!["tags" => vec!["b:blabop"]], - log_event!["custom" => json!({"b": "blabop"})], + log_event!["b" => "blabop"], ), - // Wildcard suffix - facet (negate w/-). + // Wildcard suffix - attribute (negate w/-). ( "-@b:bla*", log_event!["tags" => vec!["b:blabop"]], - log_event!["custom" => json!({"b": "blabop"})], + log_event!["b" => "blabop"], ), - // Multiple wildcards - facet. + // Multiple wildcards - attribute. ( "@c:*b*la*", - log_event!["custom" => json!({"c": "foobla"})], + log_event!["c" => "foobla"], log_event!["tags" => vec!["c:foobla"]], ), - // Multiple wildcards - facet (negate). + // Multiple wildcards - attribute (negate). ( "NOT @c:*b*la*", log_event!["tags" => vec!["c:foobla"]], - log_event!["custom" => json!({"c": "foobla"})], + log_event!["c" => "foobla"], ), - // Multiple wildcards - facet (negate w/-). + // Multiple wildcards - attribute (negate w/-). ( "-@c:*b*la*", log_event!["tags" => vec!["c:foobla"]], - log_event!["custom" => json!({"c": "foobla"})], + log_event!["c" => "foobla"], ), // Special case for tags. ( @@ -905,62 +899,50 @@ mod test { log_event!["tags" => vec!["b:test"]], log_event!["tags" => vec!["a:test"]], ), - // Range - numeric, inclusive, facet. - ( - "@b:[1 TO 10]", - log_event!["custom" => json!({"b": 5})], - log_event!["custom" => json!({"b": 11})], - ), + // Range - numeric, inclusive, attribute. + ("@b:[1 TO 10]", log_event!["b" => 5], log_event!["b" => 11]), ( "@b:[1 TO 100]", - log_event!["custom" => json!({"b": "10"})], - log_event!["custom" => json!({"b": "2"})], + log_event!["b" => "10"], + log_event!["b" => "2"], ), - // Range - numeric, inclusive, facet (negate). + // Range - numeric, inclusive, attribute (negate). ( "NOT @b:[1 TO 10]", - log_event!["custom" => json!({"b": 11})], - log_event!["custom" => json!({"b": 5})], + log_event!["b" => 11], + log_event!["b" => 5], ), ( "NOT @b:[1 TO 100]", - log_event!["custom" => json!({"b": "2"})], - log_event!["custom" => json!({"b": "10"})], - ), - // Range - numeric, inclusive, facet (negate w/-). - ( - "-@b:[1 TO 10]", - log_event!["custom" => json!({"b": 11})], - log_event!["custom" => json!({"b": 5})], + log_event!["b" => "2"], + log_event!["b" => "10"], ), + // Range - numeric, inclusive, attribute (negate w/-). + ("-@b:[1 TO 10]", log_event!["b" => 11], log_event!["b" => 5]), ( "NOT @b:[1 TO 100]", - log_event!["custom" => json!({"b": "2"})], - log_event!["custom" => json!({"b": "10"})], + log_event!["b" => "2"], + log_event!["b" => "10"], ), - // Range - alpha, inclusive, facet. - ( - "@b:[a TO z]", - log_event!["custom" => json!({"b": "c"})], - log_event!["custom" => json!({"b": 5})], - ), - // Range - alphanumeric, inclusive, facet. + // Range - alpha, inclusive, attribute. + ("@b:[a TO z]", log_event!["b" => "c"], log_event!["b" => 5]), + // Range - alphanumeric, inclusive, attribute. ( r#"@b:["1" TO "100"]"#, - log_event!["custom" => json!({"b": "10"})], - log_event!["custom" => json!({"b": "2"})], + log_event!["b" => "10"], + log_event!["b" => "2"], ), - // Range - alphanumeric, inclusive, facet (negate). + // Range - alphanumeric, inclusive, attribute (negate). ( r#"NOT @b:["1" TO "100"]"#, - log_event!["custom" => json!({"b": "2"})], - log_event!["custom" => json!({"b": "10"})], + log_event!["b" => "2"], + log_event!["b" => "10"], ), - // Range - alphanumeric, inclusive, facet (negate). + // Range - alphanumeric, inclusive, attribute (negate). ( r#"-@b:["1" TO "100"]"#, - log_event!["custom" => json!({"b": "2"})], - log_event!["custom" => json!({"b": "10"})], + log_event!["b" => "2"], + log_event!["b" => "10"], ), // Range - tag, exclusive. ( @@ -995,38 +977,34 @@ mod test { log_event!["tags" => vec!["f:100"]], log_event!["tags" => vec!["f:10"]], ), - // Range - facet, exclusive. - ( - "@f:{1 TO 100}", - log_event!["custom" => json!({"f": 50})], - log_event!["custom" => json!({"f": 1})], - ), + // Range - attribute, exclusive. + ("@f:{1 TO 100}", log_event!["f" => 50], log_event!["f" => 1]), ( "@f:{1 TO 100}", - log_event!["custom" => json!({"f": 50})], - log_event!["custom" => json!({"f": 100})], + log_event!["f" => 50], + log_event!["f" => 100], ), - // Range - facet, exclusive (negate). + // Range - attribute, exclusive (negate). ( "NOT @f:{1 TO 100}", - log_event!["custom" => json!({"f": 1})], - log_event!["custom" => json!({"f": 50})], + log_event!["f" => 1], + log_event!["f" => 50], ), ( "NOT @f:{1 TO 100}", - log_event!["custom" => json!({"f": 100})], - log_event!["custom" => json!({"f": 50})], + log_event!["f" => 100], + log_event!["f" => 50], ), - // Range - facet, exclusive (negate w/-). + // Range - attribute, exclusive (negate w/-). ( "-@f:{1 TO 100}", - log_event!["custom" => json!({"f": 1})], - log_event!["custom" => json!({"f": 50})], + log_event!["f" => 1], + log_event!["f" => 50], ), ( "-@f:{1 TO 100}", - log_event!["custom" => json!({"f": 100})], - log_event!["custom" => json!({"f": 50})], + log_event!["f" => 100], + log_event!["f" => 50], ), ] } @@ -1043,7 +1021,7 @@ mod test { let checks = get_checks(); for (source, pass, fail) in checks { - let node = parse(source).unwrap(); + let node: QueryNode = source.parse().unwrap(); let matcher = build_matcher(&node, &filter); assert!(matcher.run(&processor(pass))); @@ -1066,7 +1044,7 @@ mod test { fn check_datadog() { for (source, pass, fail) in get_checks() { let config = DatadogSearchConfig { - source: source.to_owned(), + source: source.parse().unwrap(), }; // Every query should build successfully. @@ -1076,16 +1054,16 @@ mod test { assert!( cond.check_with_context(pass.clone()).0.is_ok(), - "should pass: {}\nevent: {:?}", + "should pass: {}\nevent: {}", source, - pass.as_log() + serde_json::to_string(&pass.as_log()).unwrap(), ); assert!( cond.check_with_context(fail.clone()).0.is_err(), - "should fail: {}\nevent: {:?}", + "should fail: {}\nevent: {}", source, - fail.as_log() + serde_json::to_string(&fail.as_log()).unwrap(), ); } } diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 64dcbd0d99fc2..584dcc3bd64fe 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -1,11 +1,9 @@ use std::{path::PathBuf, time::Duration}; -#[cfg(unix)] use std::{ sync::mpsc::{channel, Receiver}, thread, }; -#[cfg(unix)] use notify::{recommended_watcher, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; use crate::Error; @@ -17,18 +15,16 @@ use crate::Error; /// - Invalid config, caused either by user or by data race. /// - Frequent changes, caused by user/editor modifying/saving file in small chunks. /// so we can use smaller, more responsive delay. -#[cfg(unix)] const CONFIG_WATCH_DELAY: std::time::Duration = std::time::Duration::from_secs(1); -#[cfg(unix)] const RETRY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); -/// Triggers SIGHUP when file on config_path changes. +/// Sends a ReloadFromDisk on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of /// this function until the main thread stops. -#[cfg(unix)] pub fn spawn_thread<'a>( + signal_tx: crate::signal::SignalTx, config_paths: impl IntoIterator + 'a, delay: impl Into>, ) -> Result<(), Error> { @@ -65,7 +61,9 @@ pub fn spawn_thread<'a>( debug!(message = "Reloaded paths."); info!("Configuration file changed."); - raise_sighup(); + _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk).map_err(|error| { + error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) + }); } else { debug!(message = "Ignoring event.", event = ?event) } @@ -83,31 +81,15 @@ pub fn spawn_thread<'a>( // so for a good measure raise SIGHUP and let reload logic // determine if anything changed. info!("Speculating that configuration files have changed."); - raise_sighup(); + _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk).map_err(|error| { + error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) + }); } }); Ok(()) } -#[cfg(windows)] -/// Errors on Windows. -pub fn spawn_thread<'a>( - _config_paths: impl IntoIterator + 'a, - _delay: impl Into>, -) -> Result<(), Error> { - Err("Reloading config on Windows isn't currently supported. Related issue https://github.com/vectordotdev/vector/issues/938 .".into()) -} - -#[cfg(unix)] -fn raise_sighup() { - use nix::sys::signal; - _ = signal::raise(signal::Signal::SIGHUP).map_err(|error| { - error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) - }); -} - -#[cfg(unix)] fn create_watcher( config_paths: &[PathBuf], ) -> Result< @@ -124,7 +106,6 @@ fn create_watcher( Ok((watcher, receiver)) } -#[cfg(unix)] fn add_paths(watcher: &mut RecommendedWatcher, config_paths: &[PathBuf]) -> Result<(), Error> { for path in config_paths { watcher.watch(path, RecursiveMode::Recursive)?; @@ -134,20 +115,22 @@ fn add_paths(watcher: &mut RecommendedWatcher, config_paths: &[PathBuf]) -> Resu #[cfg(all(test, unix, not(target_os = "macos")))] // https://github.com/vectordotdev/vector/issues/5000 mod tests { - use std::{fs::File, io::Write, time::Duration}; - - use tokio::signal::unix::{signal, SignalKind}; - use super::*; - use crate::test_util::{temp_dir, temp_file, trace_init}; - - async fn test(file: &mut File, timeout: Duration) -> bool { - let mut signal = signal(SignalKind::hangup()).expect("Signal handlers should not panic."); + use crate::{ + signal::SignalRx, + test_util::{temp_dir, temp_file, trace_init}, + }; + use std::{fs::File, io::Write, time::Duration}; + use tokio::sync::broadcast; + async fn test(file: &mut File, timeout: Duration, mut receiver: SignalRx) -> bool { file.write_all(&[0]).unwrap(); file.sync_all().unwrap(); - tokio::time::timeout(timeout, signal.recv()).await.is_ok() + matches!( + tokio::time::timeout(timeout, receiver.recv()).await, + Ok(Ok(crate::signal::SignalTo::ReloadFromDisk)) + ) } #[tokio::test] @@ -161,9 +144,10 @@ mod tests { std::fs::create_dir(&dir).unwrap(); let mut file = File::create(&file_path).unwrap(); - spawn_thread(&[dir], delay).unwrap(); + let (signal_tx, signal_rx) = broadcast::channel(128); + spawn_thread(signal_tx, &[dir], delay).unwrap(); - if !test(&mut file, delay * 5).await { + if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); } } @@ -176,14 +160,16 @@ mod tests { let file_path = temp_file(); let mut file = File::create(&file_path).unwrap(); - spawn_thread(&[file_path], delay).unwrap(); + let (signal_tx, signal_rx) = broadcast::channel(128); + spawn_thread(signal_tx, &[file_path], delay).unwrap(); - if !test(&mut file, delay * 5).await { + if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); } } #[tokio::test] + #[cfg(unix)] async fn sym_file_update() { trace_init(); @@ -193,9 +179,10 @@ mod tests { let mut file = File::create(&file_path).unwrap(); std::os::unix::fs::symlink(&file_path, &sym_file).unwrap(); - spawn_thread(&[sym_file], delay).unwrap(); + let (signal_tx, signal_rx) = broadcast::channel(128); + spawn_thread(signal_tx, &[sym_file], delay).unwrap(); - if !test(&mut file, delay * 5).await { + if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); } } @@ -212,9 +199,10 @@ mod tests { std::fs::create_dir_all(&sub_dir).unwrap(); let mut file = File::create(&file_path).unwrap(); - spawn_thread(&[sub_dir], delay).unwrap(); + let (signal_tx, signal_rx) = broadcast::channel(128); + spawn_thread(signal_tx, &[sub_dir], delay).unwrap(); - if !test(&mut file, delay * 5).await { + if !test(&mut file, delay * 5, signal_rx).await { panic!("Test timed out"); } } diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs index c87a7949c8d8a..4e80493adaf88 100644 --- a/src/sinks/aws_cloudwatch_logs/integration_tests.rs +++ b/src/sinks/aws_cloudwatch_logs/integration_tests.rs @@ -25,8 +25,8 @@ use crate::{ const GROUP_NAME: &str = "vector-cw"; -fn watchlogs_address() -> String { - std::env::var("WATCHLOGS_ADDRESS").unwrap_or_else(|_| "http://localhost:6000".into()) +fn cloudwatch_address() -> String { + std::env::var("CLOUDWATCH_ADDRESS").unwrap_or_else(|_| "http://localhost:4566".into()) } #[tokio::test] @@ -39,7 +39,7 @@ async fn cloudwatch_insert_log_event() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -57,7 +57,7 @@ async fn cloudwatch_insert_log_event() { let timestamp = chrono::Utc::now(); - let (input_lines, events) = random_lines_with_stream(100, 11, None); + let (mut input_lines, events) = random_lines_with_stream(100, 11, None); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; let response = create_client_test() @@ -72,12 +72,12 @@ async fn cloudwatch_insert_log_event() { let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - assert_eq!(output_lines, input_lines); + assert_eq!(output_lines.sort(), input_lines.sort()); } #[tokio::test] @@ -90,7 +90,7 @@ async fn cloudwatch_insert_log_events_sorted() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -145,7 +145,7 @@ async fn cloudwatch_insert_log_events_sorted() { let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); @@ -153,7 +153,7 @@ async fn cloudwatch_insert_log_events_sorted() { // readjust input_lines in the same way we have readjusted timestamps. let first = input_lines.remove(0); input_lines.push(first); - assert_eq!(output_lines, input_lines); + assert_eq!(output_lines.sort(), input_lines.sort()); } #[tokio::test] @@ -166,7 +166,7 @@ async fn cloudwatch_insert_out_of_range_timestamp() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -225,12 +225,12 @@ async fn cloudwatch_insert_out_of_range_timestamp() { let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - assert_eq!(output_lines, lines); + assert_eq!(output_lines.sort(), lines.sort()); } #[tokio::test] @@ -243,7 +243,7 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(group_name.as_str()).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -261,7 +261,7 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { let timestamp = chrono::Utc::now(); - let (input_lines, events) = random_lines_with_stream(100, 11, None); + let (mut input_lines, events) = random_lines_with_stream(100, 11, None); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; let response = create_client_test() @@ -276,12 +276,12 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - assert_eq!(output_lines, input_lines); + assert_eq!(output_lines.sort(), input_lines.sort()); } #[tokio::test] @@ -299,7 +299,7 @@ async fn cloudwatch_insert_log_event_batched() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(group_name.as_str()).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -317,7 +317,7 @@ async fn cloudwatch_insert_log_event_batched() { let timestamp = chrono::Utc::now(); - let (input_lines, events) = random_lines_with_stream(100, 11, None); + let (mut input_lines, events) = random_lines_with_stream(100, 11, None); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; let response = create_client_test() @@ -332,12 +332,12 @@ async fn cloudwatch_insert_log_event_batched() { let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - assert_eq!(output_lines, input_lines); + assert_eq!(output_lines.sort(), input_lines.sort()); } #[tokio::test] @@ -350,7 +350,7 @@ async fn cloudwatch_insert_log_event_partitioned() { let config = CloudwatchLogsSinkConfig { group_name: Template::try_from(GROUP_NAME).unwrap(), stream_name: Template::try_from(format!("{}-{{{{key}}}}", stream_name)).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -393,11 +393,11 @@ async fn cloudwatch_insert_log_event_partitioned() { .await .unwrap(); let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - let expected_output = input_lines + let mut expected_output = input_lines .clone() .into_iter() .enumerate() @@ -405,7 +405,7 @@ async fn cloudwatch_insert_log_event_partitioned() { .map(|(_, e)| e) .collect::>(); - assert_eq!(output_lines, expected_output); + assert_eq!(output_lines.sort(), expected_output.sort()); let response = create_client_test() .await @@ -418,11 +418,11 @@ async fn cloudwatch_insert_log_event_partitioned() { .unwrap(); let events = response.events.unwrap(); - let output_lines = events + let mut output_lines = events .into_iter() .map(|e| e.message.unwrap()) .collect::>(); - let expected_output = input_lines + let mut expected_output = input_lines .clone() .into_iter() .enumerate() @@ -430,7 +430,7 @@ async fn cloudwatch_insert_log_event_partitioned() { .map(|(_, e)| e) .collect::>(); - assert_eq!(output_lines, expected_output); + assert_eq!(output_lines.sort(), expected_output.sort()); } #[tokio::test] @@ -443,7 +443,7 @@ async fn cloudwatch_healthcheck() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from("test-stream").unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), - region: RegionOrEndpoint::with_both("us-east-1", watchlogs_address().as_str()), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, create_missing_stream: true, @@ -464,7 +464,7 @@ async fn cloudwatch_healthcheck() { async fn create_client_test() -> CloudwatchLogsClient { let auth = AwsAuthentication::test_auth(); let region = Some(Region::new("us-east-1")); - let endpoint = Some(watchlogs_address()); + let endpoint = Some(cloudwatch_address()); let proxy = ProxyConfig::default(); create_client::(&auth, region, endpoint, &proxy, &None, &None) diff --git a/src/sinks/honeycomb/config.rs b/src/sinks/honeycomb/config.rs index c5e4bbe749253..27a4849d480d0 100644 --- a/src/sinks/honeycomb/config.rs +++ b/src/sinks/honeycomb/config.rs @@ -3,6 +3,7 @@ use bytes::Bytes; use futures::FutureExt; use http::{Request, StatusCode, Uri}; +use vector_lib::configurable::configurable_component; use vector_lib::sensitive_string::SensitiveString; use vrl::value::Kind; @@ -28,8 +29,13 @@ pub(super) const HTTP_HEADER_HONEYCOMB: &str = "X-Honeycomb-Team"; #[configurable_component(sink("honeycomb", "Deliver log events to Honeycomb."))] #[derive(Clone, Debug)] pub struct HoneycombConfig { - // This endpoint is not user-configurable and only exists for testing purposes - #[serde(skip, default = "default_endpoint")] + /// Honeycomb's endpoint to send logs to + #[serde(default = "default_endpoint")] + #[configurable(metadata( + docs::examples = "https://api.honeycomb.io", + docs::examples = "https://api.eu1.honeycomb.io", + ))] + #[configurable(validation(format = "uri"))] pub(super) endpoint: String, /// The API key that is used to authenticate against Honeycomb. @@ -65,7 +71,7 @@ pub struct HoneycombConfig { } fn default_endpoint() -> String { - "https://api.honeycomb.io/1/batch".to_string() + "https://api.honeycomb.io".to_string() } #[derive(Clone, Copy, Debug, Default)] @@ -136,7 +142,11 @@ impl SinkConfig for HoneycombConfig { impl HoneycombConfig { fn build_uri(&self) -> crate::Result { - let uri = format!("{}/{}", self.endpoint, self.dataset); + let uri = format!( + "{}/1/batch/{}", + self.endpoint.trim_end_matches('/'), + self.dataset + ); uri.parse::().map_err(Into::into) } } diff --git a/src/sinks/kafka/config.rs b/src/sinks/kafka/config.rs index 5860fa1d291b0..7e42e7ab85d21 100644 --- a/src/sinks/kafka/config.rs +++ b/src/sinks/kafka/config.rs @@ -142,16 +142,8 @@ fn example_librdkafka_options() -> HashMap { ]) } -/// Used to determine the options to set in configs, since both Kafka consumers and producers have -/// unique options, they use the same struct, and the error if given the wrong options. -#[derive(Debug, PartialOrd, PartialEq, Eq)] -pub enum KafkaRole { - Consumer, - Producer, -} - impl KafkaSinkConfig { - pub(crate) fn to_rdkafka(&self, kafka_role: KafkaRole) -> crate::Result { + pub(crate) fn to_rdkafka(&self) -> crate::Result { let mut client_config = ClientConfig::new(); client_config .set("bootstrap.servers", &self.bootstrap_servers) @@ -164,73 +156,71 @@ impl KafkaSinkConfig { self.auth.apply(&mut client_config)?; // All batch options are producer only. - if kafka_role == KafkaRole::Producer { - client_config - .set("compression.codec", &to_string(self.compression)) - .set( - "message.timeout.ms", - &self.message_timeout_ms.as_millis().to_string(), - ); - - if let Some(value) = self.batch.timeout_secs { - // Delay in milliseconds to wait for messages in the producer queue to accumulate before - // constructing message batches (MessageSets) to transmit to brokers. A higher value - // allows larger and more effective (less overhead, improved compression) batches of - // messages to accumulate at the expense of increased message delivery latency. - // Type: float - let key = "queue.buffering.max.ms"; - if let Some(val) = self.librdkafka_options.get(key) { - return Err(format!("Batching setting `batch.timeout_secs` sets `librdkafka_options.{}={}`.\ - The config already sets this as `librdkafka_options.queue.buffering.max.ms={}`.\ - Please delete one.", key, value, val).into()); - } - debug!( - librdkafka_option = key, - batch_option = "timeout_secs", - value, - "Applying batch option as librdkafka option." - ); - client_config.set(key, &((value * 1000.0).round().to_string())); + client_config + .set("compression.codec", &to_string(self.compression)) + .set( + "message.timeout.ms", + &self.message_timeout_ms.as_millis().to_string(), + ); + + if let Some(value) = self.batch.timeout_secs { + // Delay in milliseconds to wait for messages in the producer queue to accumulate before + // constructing message batches (MessageSets) to transmit to brokers. A higher value + // allows larger and more effective (less overhead, improved compression) batches of + // messages to accumulate at the expense of increased message delivery latency. + // Type: float + let key = "queue.buffering.max.ms"; + if let Some(val) = self.librdkafka_options.get(key) { + return Err(format!("Batching setting `batch.timeout_secs` sets `librdkafka_options.{key}={value}`.\ + The config already sets this as `librdkafka_options.queue.buffering.max.ms={val}`.\ + Please delete one.").into()); } - if let Some(value) = self.batch.max_events { - // Maximum number of messages batched in one MessageSet. The total MessageSet size is - // also limited by batch.size and message.max.bytes. - // Type: integer - let key = "batch.num.messages"; - if let Some(val) = self.librdkafka_options.get(key) { - return Err(format!("Batching setting `batch.max_events` sets `librdkafka_options.{}={}`.\ - The config already sets this as `librdkafka_options.batch.num.messages={}`.\ - Please delete one.", key, value, val).into()); - } - debug!( - librdkafka_option = key, - batch_option = "max_events", - value, - "Applying batch option as librdkafka option." - ); - client_config.set(key, &value.to_string()); + debug!( + librdkafka_option = key, + batch_option = "timeout_secs", + value, + "Applying batch option as librdkafka option." + ); + client_config.set(key, &((value * 1000.0).round().to_string())); + } + if let Some(value) = self.batch.max_events { + // Maximum number of messages batched in one MessageSet. The total MessageSet size is + // also limited by batch.size and message.max.bytes. + // Type: integer + let key = "batch.num.messages"; + if let Some(val) = self.librdkafka_options.get(key) { + return Err(format!("Batching setting `batch.max_events` sets `librdkafka_options.{key}={value}`.\ + The config already sets this as `librdkafka_options.batch.num.messages={val}`.\ + Please delete one.").into()); } - if let Some(value) = self.batch.max_bytes { - // Maximum size (in bytes) of all messages batched in one MessageSet, including protocol - // framing overhead. This limit is applied after the first message has been added to the - // batch, regardless of the first message's size, this is to ensure that messages that - // exceed batch.size are produced. The total MessageSet size is also limited by - // batch.num.messages and message.max.bytes. - // Type: integer - let key = "batch.size"; - if let Some(val) = self.librdkafka_options.get(key) { - return Err(format!("Batching setting `batch.max_bytes` sets `librdkafka_options.{}={}`.\ - The config already sets this as `librdkafka_options.batch.size={}`.\ - Please delete one.", key, value, val).into()); - } - debug!( - librdkafka_option = key, - batch_option = "max_bytes", - value, - "Applying batch option as librdkafka option." - ); - client_config.set(key, &value.to_string()); + debug!( + librdkafka_option = key, + batch_option = "max_events", + value, + "Applying batch option as librdkafka option." + ); + client_config.set(key, &value.to_string()); + } + if let Some(value) = self.batch.max_bytes { + // Maximum size (in bytes) of all messages batched in one MessageSet, including protocol + // framing overhead. This limit is applied after the first message has been added to the + // batch, regardless of the first message's size, this is to ensure that messages that + // exceed batch.size are produced. The total MessageSet size is also limited by + // batch.num.messages and message.max.bytes. + // Type: integer + let key = "batch.size"; + if let Some(val) = self.librdkafka_options.get(key) { + return Err(format!("Batching setting `batch.max_bytes` sets `librdkafka_options.{key}={value}`.\ + The config already sets this as `librdkafka_options.batch.size={val}`.\ + Please delete one.").into()); } + debug!( + librdkafka_option = key, + batch_option = "max_bytes", + value, + "Applying batch option as librdkafka option." + ); + client_config.set(key, &value.to_string()); } for (key, value) in self.librdkafka_options.iter() { diff --git a/src/sinks/kafka/sink.rs b/src/sinks/kafka/sink.rs index c908979e5c2e2..03b11abae068e 100644 --- a/src/sinks/kafka/sink.rs +++ b/src/sinks/kafka/sink.rs @@ -1,7 +1,6 @@ use rdkafka::{ - consumer::{BaseConsumer, Consumer}, error::KafkaError, - producer::FutureProducer, + producer::{BaseProducer, FutureProducer, Producer}, ClientConfig, }; use snafu::{ResultExt, Snafu}; @@ -9,7 +8,7 @@ use tokio::time::Duration; use tracing::Span; use vrl::path::OwnedTargetPath; -use super::config::{KafkaRole, KafkaSinkConfig}; +use super::config::KafkaSinkConfig; use crate::{ kafka::KafkaStatisticsContext, sinks::kafka::{request_builder::KafkaRequestBuilder, service::KafkaService}, @@ -46,7 +45,7 @@ pub(crate) fn create_producer( impl KafkaSink { pub(crate) fn new(config: KafkaSinkConfig) -> crate::Result { - let producer_config = config.to_rdkafka(KafkaRole::Producer)?; + let producer_config = config.to_rdkafka()?; let producer = create_producer(producer_config)?; let transformer = config.encoding.transformer(); let serializer = config.encoding.build()?; @@ -105,7 +104,7 @@ impl KafkaSink { pub(crate) async fn healthcheck(config: KafkaSinkConfig) -> crate::Result<()> { trace!("Healthcheck started."); - let client = config.to_rdkafka(KafkaRole::Consumer).unwrap(); + let client_config = config.to_rdkafka().unwrap(); let topic: Option = match config.healthcheck_topic { Some(topic) => Some(topic), _ => match config.topic.render_string(&LogEvent::from_str_legacy("")) { @@ -121,10 +120,11 @@ pub(crate) async fn healthcheck(config: KafkaSinkConfig) -> crate::Result<()> { }; tokio::task::spawn_blocking(move || { - let consumer: BaseConsumer = client.create().unwrap(); + let producer: BaseProducer = client_config.create().unwrap(); let topic = topic.as_ref().map(|topic| &topic[..]); - consumer + producer + .client() .fetch_metadata(topic, Duration::from_secs(3)) .map(|_| ()) }) diff --git a/src/sinks/kafka/tests.rs b/src/sinks/kafka/tests.rs index 6d36e77fd6fd2..2c327f09d42f9 100644 --- a/src/sinks/kafka/tests.rs +++ b/src/sinks/kafka/tests.rs @@ -19,11 +19,7 @@ mod integration_test { event::{BatchNotifier, BatchStatus}, }; - use super::super::{ - config::{KafkaRole, KafkaSinkConfig}, - sink::KafkaSink, - *, - }; + use super::super::{config::KafkaSinkConfig, sink::KafkaSink, *}; use crate::{ event::{ObjectMap, Value}, kafka::{KafkaAuthConfig, KafkaCompression, KafkaSaslConfig}, @@ -190,8 +186,7 @@ mod integration_test { headers_key: None, acknowledgements: Default::default(), }; - config.clone().to_rdkafka(KafkaRole::Consumer)?; - config.clone().to_rdkafka(KafkaRole::Producer)?; + config.clone().to_rdkafka()?; self::sink::healthcheck(config.clone()).await?; KafkaSink::new(config) } diff --git a/src/sources/amqp.rs b/src/sources/amqp.rs index bcafc1125f805..4f3fc7098aac0 100644 --- a/src/sources/amqp.rs +++ b/src/sources/amqp.rs @@ -27,7 +27,7 @@ use vector_lib::configurable::configurable_component; use vector_lib::lookup::{lookup_v2::OptionalValuePath, metadata_path, owned_value_path, path}; use vector_lib::{ config::{log_schema, LegacyKey, LogNamespace, SourceAcknowledgementsConfig}, - event::Event, + event::{Event, LogEvent}, EstimatedJsonEncodedSizeOf, }; use vector_lib::{ @@ -236,14 +236,12 @@ struct Keys<'a> { } /// Populates the decoded event with extra metadata. -fn populate_event( - event: &mut Event, +fn populate_log_event( + log: &mut LogEvent, timestamp: Option>, keys: &Keys<'_>, log_namespace: LogNamespace, ) { - let log = event.as_mut_log(); - log_namespace.insert_source_metadata( AmqpSourceConfig::NAME, log, @@ -348,10 +346,12 @@ async fn receive_event( )); for mut event in events { - populate_event(&mut event, - timestamp, - &keys, - log_namespace); + if let Event::Log(ref mut log) = event { + populate_log_event(log, + timestamp, + &keys, + log_namespace); + } yield event; } diff --git a/src/sources/file.rs b/src/sources/file.rs index 3bf8aa826e540..362cae59cc07b 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -511,6 +511,11 @@ pub fn file_source( return Box::pin(future::ready(Err(()))); } + let exclude_patterns = config + .exclude + .iter() + .map(|path_buf| path_buf.iter().collect::()) + .collect::>(); let ignore_before = calculate_ignore_before(config.ignore_older_secs); let glob_minimum_cooldown = config.glob_minimum_cooldown_ms; let (ignore_checkpoints, read_from) = reconcile_position_options( @@ -525,7 +530,7 @@ pub fn file_source( let paths_provider = Glob::new( &config.include, - &config.exclude, + &exclude_patterns, MatchOptions::default(), emitter.clone(), ) @@ -1407,6 +1412,52 @@ mod tests { assert_eq!(is, [n as usize; 3]); } + #[tokio::test] + async fn file_exclude_paths() { + let n = 5; + + let dir = tempdir().unwrap(); + let config = file::FileConfig { + include: vec![dir.path().join("a//b/*.log.*")], + exclude: vec![dir.path().join("a//b/test.log.*")], + ..test_default_file_config(&dir) + }; + + let path1 = dir.path().join("a//b/a.log.1"); + let path2 = dir.path().join("a//b/test.log.1"); + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + std::fs::create_dir_all(dir.path().join("a/b")).unwrap(); + let mut file1 = File::create(&path1).unwrap(); + let mut file2 = File::create(&path2).unwrap(); + + sleep_500_millis().await; // The files must be observed at their original lengths before writing to them + + for i in 0..n { + writeln!(&mut file1, "1 {}", i).unwrap(); + writeln!(&mut file2, "2 {}", i).unwrap(); + } + + sleep_500_millis().await; + }) + .await; + + let mut is = [0; 1]; + + for event in received { + let line = + event.as_log()[log_schema().message_key().unwrap().to_string()].to_string_lossy(); + let mut split = line.split(' '); + let file = split.next().unwrap().parse::().unwrap(); + assert_ne!(file, 4); + let i = split.next().unwrap().parse::().unwrap(); + + assert_eq!(is[file - 1], i); + is[file - 1] += 1; + } + + assert_eq!(is, [n as usize; 1]); + } + #[tokio::test] async fn file_key_acknowledged() { file_key(Acks).await diff --git a/src/sources/util/grpc/mod.rs b/src/sources/util/grpc/mod.rs index 1052318538bc6..0c19e9795e37b 100644 --- a/src/sources/util/grpc/mod.rs +++ b/src/sources/util/grpc/mod.rs @@ -8,10 +8,7 @@ use http::{Request, Response}; use hyper::Body; use std::{convert::Infallible, net::SocketAddr, time::Duration}; use tonic::transport::server::Routes; -use tonic::{ - body::BoxBody, - transport::server::{NamedService, Server}, -}; +use tonic::{body::BoxBody, server::NamedService, transport::server::Server}; use tower::Service; use tower_http::{ classify::{GrpcErrorsAsFailures, SharedClassifier}, diff --git a/tests/data/nats/nats-client.key b/tests/data/nats/nats-client.key index ca64030220132..f896e132da3f1 100644 --- a/tests/data/nats/nats-client.key +++ b/tests/data/nats/nats-client.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+pFV/gLpYpHnG -EspyM/DEH4a5zRz/3PYF4wXNGhxZW5D0eriH8fp9reMvgfQB+56yIk5O7FA8NLv8 -V3Gb+24usUM+1Hf2UOm/s5AqHa1lGlPRhFFU0izRBbVIE+D0Fc7Q7DKk8v/ADRfU -qPNKqPxoUyMWeDv4j88PVJFVDwupdSav4JK40OqG/hxY78HmsguvFdScoV324//g -ljYTuWA/4uTZWm/W+//JgrC4mqR+IOFTgdQ2NSVQCk0YzkUAZHxYjoH9yvT5GkSE -3N3IoyvLqhDi2UEkhCIgcDfis5vfBxw41IMN4laFgKeF3fhzs5QJ4qBqqNQ+jVjC -LfnVDa/1AgMBAAECggEAKdv/ge2L/SawLlO16/mHpiid9nIZKf0kcAp3HZ/cFarL -W0MO/ZqeO7ZQqf/Iyh6rq8VomF+/Blk15aR1p6A/gzUc5NDWPllp2dPdlcHKbuam -/Nwm148GlCbCtT7YKAAZ1znlrN5GVa8KSoGGWiBcu7N1gtuHn35EIsOY3apeDyju -ne5JHraj9XNdBu2w1Rms/45shnegypuhTkpUmMU01nVqSr9mmwef2bsGTs0qGUjh -0jwr78+386QxtPd2gcdnG8s9Wb8rmf6UXmg1P0kSwvn4jrIQjdb+bzt1cBUemtbM -/uxoW8/wWJkVP112kCz4pzOaSgzgnv7rexYRnZkdwQKBgQDKKBBBNgFavYi480+R -avFzmo6zONuj1CXiohKLzL6cC9XE4vHaM2+/01NLmpdHmCYd/hsbtAplmwPXVkME -opb7NY6sp1yv5ukfv2SoTfSosj6wjgKB8wZKF0ft6/gZfEEg+bdZiDZxuCmkyMBe -elcNEYKirs/0xhP/V5rugEDz/QKBgQDxayfTwKdO9UUY7Bs67hALBKI9uznWs9TZ -/7i8MKyaglvi3kpjbW0Z+TGix+sUxOh0gOFl/4W7V5a/43hwV5Prdv+HQcjwnk3H -f3kRqK5njy1VJbgyXkuQaJ8CIqoG9WkDXR5ie3RivydM4UnCAh9ZmjCAJEElkOPn -WIsP0f5hWQKBgQCpdBHEB9f0zwFHPzTISWOhqsHJO3TpDBDBu2deZj60oTsMccuX -Qxx+KE8fBbOfYMs6wuDIRQ+eAeWjYHyG6x8x7b/tu9B25lEtuopOhiH6ModYCYHS -mfsAqikyJ2VtP3l8gbYOR56TjnLh3SFbUKOdobrmUTCiFlN262FEYJ+RpQKBgDFf -Y87HmK/qM05sgwkjRL6sHO4ZCzQgpQhMZyB9p98Vh+s24OnCqExmjUMoNaiSuKOS -9AXjIxt0R01JHQmEr6i5NnsX9htXlLqXFFCLOdfODUxboXcTl8qZyuMp6bHwoKBi -e+NSMjQ7ubLcZDQ8MOMnSIH8aCJhpmpazl3W6XpJAoGAXdbJ2dJ4I869EsUlUw8d -qgk9fqjFsuJ1L5/KyF/JN5zIUMKyTHblEVvNMZzmqrJHM8GJEmCbLDxmH4RbBFYM -B7Kh2mznMDgUGhgjc8MZ2X0kKRVGKA0beED1gFnCvJkeyULuKDoRjpWjwy2pgaEF -Eos6evX5WdDzieJEAFoen34= +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDAQtfZUHxXgM/F +IthtVOisDo/mSWRF8IwPY237KPh/cjjq4x29LfkfCkCJrBIcVJ97bbk7cQwqcUWD +x24whcMRGhFkbcHB9PV5/doRc8NZKBq/hMLTn+77BHWqVFSpYliRmqAUZdJFevtB +dQsBCSIwgm7KfiUtk6FJ5KAquHyjiq4jR8wLCpwk64sXX6vaOJoKLv2mGJwe+7ZD +KwQmdIG7WsJvzwrYf7iuwFwJlhBdPeUqLPSToD9iRTrom8j96LUYGKuBnzWSkVco +lhZsYXWSxbeTd4pD5NoPEFYvQItjx2Hp7fdXdML+yTyHwjgWBBQCGqTG8yWOymvF +Ba7R8zWhAgMBAAECggEBAJRTvXVB+/wCO8kqXfDTfgIn5erTmlG/CD9jBVmsvTKN +682DVNE746OH5EySfRJwUu4Fs89hIvKnmurU9HTo7ZqHEv35E2/QzYXFS/UE66Qg +Jaq2AR0FmXj9cs3q67oWTqBeu7ug5n2GKMvSQaAV7NOh/VmpwoMMD3YTtWHgHQCJ +x93dvfBt2LdcquSZVO2b/el2+V2giOlOl8eq+DQh0wdQPDXVd8zn9eHYspGQ5Cl3 +v2xVCtLMxWcrFQz3+RdWfqtRp8fDSfjx8/0uqXpoa72KdNI/ui91MO0800NAdtdE +iERdMg7/JNnIMJz92r2DaesJoe/YUrwf3OF+88Jv8LECgYEA+DqClgFXyH9mg+o4 +SvUWfK7fS/cJhc3FWgAye+98Is31jfcQtv03AL5jkd1UKbIz15ujH+vTIEa+oqeK +5fffNm2yCsEBEcdhJwQMUKCvLS7vqMvEGqS3Wxcs6QmCYuHOd7tPw8f/LGMADmO/ +yb4sJG1mn60BNtB4PGZp2OPhhaUCgYEAxkfEltxEglshGX+sLaK2+EgcvS3lvlXA +YKKHmnOq5zZpB7PbnB8hzIBjFhQzKyVdzcKbHE40gC80NJR2L46k4lNKx6aaSwrZ +Hd/BqW+4EG6J67LfjKXrg/0j4iCDj7t6aIJdqfEfSCzOOequgo8ASY0ue8PS7DRb +LqtFd2nUh00CgYAumF0W512hBzkpHfg/Kc50zSahTdf9w+h9wne3h/bXWa30XiJH +FzUR8KskIFXCFe9hU5fFUDErZ9w1EBoknOjLgL/hAOXAfpqgBkSBhggcBSl7l/RR +xAWfFbSaceyFvzlQhFqut4YQwOyrxtGjfw0uMVMIOiN2NJ+9RsQrsGsO+QKBgQCy +O8sUjRWsU/xx7lk4gH6WukPTLlHJgU230AGzLexfgIJzIRL0yOtZ58KqEyhXzXP/ +xMIndIku/L+h0B5Y65F9vL+IrlMdx7eZagysoTdE1gh2bckTpgYKFF/QIcgBSNZk +4WMOoyeqelQ6raGGzMLGWvmA2WtdVkbssZp1GBWqNQKBgQC31DmKt/Vijf3Uh2e0 +OBcFfm1L/cVLsQre/msiniFsVr3nveqF3SqCjIwHzP627vJr+ACto8euWwA+qRHb +V7QQGzLP8VvNWdqhOCFEehTTjXKRFynR9tuoRlRXjWkewRCHWXXHKSk39v375Z7I +yOVuhg4SFHqVSfyi0rCKAqKRJw== -----END PRIVATE KEY----- diff --git a/tests/data/nats/nats-client.pem b/tests/data/nats/nats-client.pem index 4b3f310eed502..6c2bfeb47b122 100644 --- a/tests/data/nats/nats-client.pem +++ b/tests/data/nats/nats-client.pem @@ -1,26 +1,28 @@ -----BEGIN CERTIFICATE----- -MIIEbDCCAtSgAwIBAgIQI823IA+ACP2j4LAnkVMoaTANBgkqhkiG9w0BAQsFADBd -MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGTAXBgNVBAsMEHRvYnlA -Y29uc2lnbGllcmUxIDAeBgNVBAMMF21rY2VydCB0b2J5QGNvbnNpZ2xpZXJlMB4X -DTIyMDUxOTE3MTc1NloXDTI0MDgxOTE3MTc1NlowRDEnMCUGA1UEChMebWtjZXJ0 -IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMRkwFwYDVQQLDBB0b2J5QGNvbnNpZ2xp -ZXJlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvqRVf4C6WKR5xhLK -cjPwxB+Guc0c/9z2BeMFzRocWVuQ9Hq4h/H6fa3jL4H0AfuesiJOTuxQPDS7/Fdx -m/tuLrFDPtR39lDpv7OQKh2tZRpT0YRRVNIs0QW1SBPg9BXO0OwypPL/wA0X1Kjz -Sqj8aFMjFng7+I/PD1SRVQ8LqXUmr+CSuNDqhv4cWO/B5rILrxXUnKFd9uP/4JY2 -E7lgP+Lk2Vpv1vv/yYKwuJqkfiDhU4HUNjUlUApNGM5FAGR8WI6B/cr0+RpEhNzd -yKMry6oQ4tlBJIQiIHA34rOb3wccONSDDeJWhYCnhd34c7OUCeKgaqjUPo1Ywi35 -1Q2v9QIDAQABo4HAMIG9MA4GA1UdDwEB/wQEAwIFoDAnBgNVHSUEIDAeBggrBgEF -BQcDAgYIKwYBBQUHAwEGCCsGAQUFBwMEMB8GA1UdIwQYMBaAFABORGXvkPQ6Eodc -3kIdY+Lo4JzxMGEGA1UdEQRaMFiCCWxvY2FsaG9zdIIIbmF0cy10bHOCFG5hdHMt -dGxzLWNsaWVudC1jZXJ0gghuYXRzLWp3dIEPZW1haWxAbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABMA0GCSqGSIb3DQEBCwUAA4IBgQCmlYkeJB5DqhIqSLJnM3Bh -LrDptzGv2No2hT9ZBh58TH5APODUnaWTrUfBmscoTsApUyH3ZdgJnDSOOuj1MM4+ -Jbfo9zk2RIqAK8s99h93A9jcXLbj1nzQtqyaJ97GRo1TGVTK+CX1oB4RTZJ/wSi1 -3f/YNl7919x8VB5/EOOE8OIwRR//SUgRiWjRb1Mw5xWh5I1DGLRb5j8uQ0FlMMV4 -YGgJXjZqI3n0riVBWWpdBLM+qlhlFfvG3apRsSwDpOZVAqPoYfY4DwIZsvedWZBj -LRTJB5GlSLd+XlsFJW3vIJgVYDSJ23en8i84EuIsLOqui1qceshrwFVeqs9En9da -5HI7CmrFOLLABbOJ10UY9xCDCmZXtVBPvPA+NfPlrxS6z/x5ZJVE+1zLIlrlFiYo -4a9PzydjbdLMot0zB1BxoZqRnAXXVCSslbXHYnwyfq7shPe0p81UnCabm0mz8D/g -uZGO4XijaGHo7+DmVulzlC4SYo/FQ8h3Sty0+VYF4pE= +MIIExDCCAyygAwIBAgIQEBPAKPbaNfc7JuhyQ2wD/zANBgkqhkiG9w0BAQsFADCB +lzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMTYwNAYDVQQLDC1qZXNz +ZS5zendlZGtvQENPTVAtSjRDNFAyN0s5USAoSmVzc2UgU3p3ZWRrbykxPTA7BgNV +BAMMNG1rY2VydCBqZXNzZS5zendlZGtvQENPTVAtSjRDNFAyN0s5USAoSmVzc2Ug +U3p3ZWRrbykwHhcNMjQwODE5MjMxNDM5WhcNMjYxMTIwMDAxNDM5WjBhMScwJQYD +VQQKEx5ta2NlcnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxNjA0BgNVBAsMLWpl +c3NlLnN6d2Vka29AQ09NUC1KNEM0UDI3SzlRIChKZXNzZSBTendlZGtvKTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMBC19lQfFeAz8Ui2G1U6KwOj+ZJ +ZEXwjA9jbfso+H9yOOrjHb0t+R8KQImsEhxUn3ttuTtxDCpxRYPHbjCFwxEaEWRt +wcH09Xn92hFzw1koGr+EwtOf7vsEdapUVKliWJGaoBRl0kV6+0F1CwEJIjCCbsp+ +JS2ToUnkoCq4fKOKriNHzAsKnCTrixdfq9o4mgou/aYYnB77tkMrBCZ0gbtawm/P +Cth/uK7AXAmWEF095Sos9JOgP2JFOuibyP3otRgYq4GfNZKRVyiWFmxhdZLFt5N3 +ikPk2g8QVi9Ai2PHYent91d0wv7JPIfCOBYEFAIapMbzJY7Ka8UFrtHzNaECAwEA +AaOBwDCBvTAOBgNVHQ8BAf8EBAMCBaAwJwYDVR0lBCAwHgYIKwYBBQUHAwIGCCsG +AQUFBwMBBggrBgEFBQcDBDAfBgNVHSMEGDAWgBQORWCblOmWc/q0xTyadPzbWUsl +fjBhBgNVHREEWjBYgglsb2NhbGhvc3SCCG5hdHMtdGxzghRuYXRzLXRscy1jbGll +bnQtY2VydIIIbmF0cy1qd3SBD2VtYWlsQGxvY2FsaG9zdIcQAAAAAAAAAAAAAAAA +AAAAATANBgkqhkiG9w0BAQsFAAOCAYEAHZhAyvqsD5y/6I9jFrQmMb0rrg4LrzWK +/wLCNQT7rJO6c6N4OXIrLEfteUT5NXGwqXmTp379Kl+B+pl0i6E1QLWbmLbbX72A +ieH3nPRsf1es3Xokia55O1YhSPqX8e5oufZ6g63kD7xmU6YQyrAt8c8JLOV4mtbj +lu/zAwQxTC8NBND/z1gDtnN9roOY1uCvx2S9/Ky3Anjd0+VXrCivukVz+Sqfv103 +cReRLEvSfZPObDSoFzxfuN4j648gAkR2r8LGzHYiXO4pk5lmv0OGmTegD/FVb7vE +rF/+n+A/dn6IDLo3/hiwE6u28geQMfEChwMzprAClevtc78oC+pJ9oOqX5W/WjVJ +IZy6gDCOBl9UBTSqiM9LYoH7wCmg7q4IDxBsj0iJkSGN7d8byan1itj64YKCldXL +mBxAaC0qEBqpvchmr3sqkmNc+4va5EkXcGjEJBLdYLJw991OorY+DTC4V/eLyZ69 +BUKfRT7GgjFtT7IT+PdbC8Q7RX5ivJfP -----END CERTIFICATE----- diff --git a/tests/data/nats/nats-server.key b/tests/data/nats/nats-server.key index 63db05e0d215a..72e279d71421e 100644 --- a/tests/data/nats/nats-server.key +++ b/tests/data/nats/nats-server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC/hUmBJlo8drat -XGsTNkBVEFyPUVbNETzz3Qjttd67EGYFqFRm1zOPG79LHY8CaiaHM01o+qZijW6+ -c3yIBIYOaRQxikHnBUW+f+HfwImqd7RsYixL6pnfRbD4UFa90IUOv0JqeFLvDNMb -mb29V9ESE6le//8FKgHH777ioKjuxSvbl1K5kskXktzSpjELnaBf5x74XF32k23U -uz3LNMK/5+OIoFICqM6Fh14sRof810xN98LHPZazjMQmOsphZja8YYf7dnJAlt8o -iCuSuTmCEXCtgCJhW0OawuNsRZCe+wZhOT7XjXW07x2tCLDjyRP4ckf9e7yHRHep -ffr0D8KDAgMBAAECggEBAJ/lLJ/5oB6Y0hzcarzIb19VYKF6vChK+wNyB3FGhOTS -ZQrWJVkS1AXnmYah7/KJIw9/YmKB2QWwpMO2cUKx3CYNzoEpt6mjej34UGDZQ1Yh -yG0wbWEYRUzlx0EY+cJRY7UTMuTm/f58S4fgjEuXFIKuDZq1yXjLQF9nJovYrBFi -8NH8vyILV73x5D6f52gEqHw44TW5mBue/dFlSYjNyv3CVLgb3hauGqfmqK7M5bgf -/Mp+sc9qH8vuTcEFJFT5+K213YObqoVwrIU8czr6v2k8VGg05mF38WT2ai4yffOc -WiYiDXn8vWuSMgzyTMOTK8ag0Ui+7+cH7Ker7v6n6SkCgYEAwKxpldDDKyPavAxt -qO2PMqy++59HrMAuGkDuEJoeJzAkVbwn2lInfJnq7G2BJMv4tnwEsURhjP+EOcmZ -7aehGNK3nxbFkGwgi1EXWqhHWtVvCydVhcJh6xP2wf8IdlIRHA93UJYlRQsJ65Gs -oVJD57Wsh6XRnmQ1au4CPBaHu/UCgYEA/nfgA2fdskx6RNOexI7yJvibZxlEic+2 -8PS8dHHSkKJPIRu9lhzzAiM59mCcw4buuQfKbVAt3nPHcPa+V3RXCmKNPxVyoVd3 -qUDYOEcSJ2uWG2jX0V9kdT7TDyc2d68WXWy4cHKDukBaWro7IM90IsJXcdDPt9cM -SAdPTaUuMZcCgYEAlchy/KiiUPM6pF31EhI79GUew3X7vmxYash3SijFsClY1d3E -DZE4aUEuxEYM05JN829OJpDGVVp+bH4xT/O05RtK7D5DsBae5i9QPIX9krvB66XU -cnu+q4WFmyKsbW83Jd1QIDHZH2eJerxwu6i8aXFugrWB0+SLMml9C0Fom9ECgYBP -C+HUbQFbD5KzqqEZIeZq4LM1dY3q2VFUzFF2Qp1MPH8kI7/iz59VPKgWtrA7tMCX -Tao+BrgY8BI2Lw+1jvgyy6ZgwELZh5guRWxlHv6urCb24LyZ124XVaqufFnZF9R0 -tP8ePxRYWQE4Sb1wVcEXwxLy0hCUZQL1pP+nePY+hQKBgQCgzw+XQ+9nlGJOW5/3 -g1geSalgu1+TvqqvdRIs+Nd7tTdCZSIeycIlR1qDLLE0GAr6ZyzIoRjxyWEem/ev -N47+Kp6PzWhdnkOhKSnRXgLadppxYCHZuOGHLPGSCRfW57fuEXr9iDV04hOiDOXs -G+6AxZps1wF6HlmRLSlsyHT+pA== +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCwMoaoX7OrT2AO +2QPTfHGxfgHx2z7FSw0gnYPuzVFZDBmXaK/HkY5Ybra0agsBVQo2IOTjuHXdrb6M +RnhTgexYw+Dho+TPxDylaEonHmMD+mKU5MWueX2sCAmB4p1E4Uzl/aJkt55mJRwA +JPuaxmEp+kuEYna20G80M0Mr962Xe5mNrjT3DPyxoNAOSZIh3WQqY0BpBB2g46ty +W1bhY1fud48QpzIcqVq7cRokv2KdMH6fzBGjbSi5Jc7qYhtA56DZ4tVTzwFCqSTX +qQqF91I4Y5AI+0aU7hjLJY4AO3xUDif7PmEj4OGs1isYoAkGX2eH76DOLu6LEgbX +tBR4FeGLAgMBAAECggEAM3Es+2+lhziPdQZLJa3Ji+acLr6yHuwAkysp38i2HWd4 ++UmpOHz4sDzyvs3PFL59Urxb11h9oREe9tmEO+1M0cWunHI7VaWXhzthmvY2Nm0d +5xa72GrEFDvseq1s+TVGPEfG59zsNCpVeH1TsCaHK7hV3/rXahyEuxrUW7veRMFm +yOy/t5AHax/dDZeSTKxhrltj/H9iZNfAcAmSkHNw46fNqm0smeEi1Q+SeumIZ8nW +7hnhqeP1guG6vITulacPMhKX3ReA13XE804aLvFdrZvWxDSu1wRrdOi5EH6Kt+bg +agR/71udA3b4GY63iTOAvP+dK7FjvsTXgADRxV6OsQKBgQDppe8qsfEGVwQkC/yt +srKRmolIu35GM70LR6n0a/Mo+LG0FE/ySJ6Ab2A1hJR4++13pTFENXtcBoHdTFtq +LwXa6IMFNOg885CHiSqiNCBwW41jF+mGTyiKmJfajRU4hFskiRJk1NMLMNWB8pmt +vPg7LvlUgsCS3UFgVUy+dKg7HQKBgQDBDZ2wtGAF8FYUiRoh8vilMjVlryMCiBFx +Qv1YKAZ+35uBa+LqbWZUvfjetJxuk3N6QYfw1Y8fKB78NdvxfzIeOzRHClUe0hJS +GbYt6xeaZ20W3X1Hs7ufJQm6SKKfdCnRUKhjHhCkfBwe9bbsGwsLRGFG3zYsJ0qc +3W6rvt5GxwKBgG33cCpbF4KRX9yLW8Bw04SeBDEcNaFa5oWxk/b6+7Xz3sdPrBTv +QtJWGST3Il4SzEA8moRs24ODeImsT3YZbZsLReckcI2HYSW5VDyFZiMbo97YuUtC +ZGVezlHJhwxTlwYKEYMte6rPsB16DgRsZTSNtkGlTHeS9Rse6oDVLPytAoGAHBTb +9EE4moSMMO6Eg52xN25nQARKxmp+hwgki/c+gqQKHbhQyKLjOL5tfg+tlZQTPxxb +JtgZuoF+iFdZR/allqor/41sNlB+Y7zjuCgzRb13q9TKDW0ChLLBiHxaU3F1R8JA +zDIqX5LCEGrTkeKZJ6lpsQJ1AdUxUEIRgCACey8CgYBFDljSuTc68Qzd0xy7S+Rg +EGXTTyBJyTSWEfwRn6HZzZoQ9SAoyMa80v7JvLPnQqUJgyafBkBVRADk1Nv9PGSS +RNZkD1OfypMy6HXNM0wdNpVwSFtzq06ODK2xNHJS3lBQ3pnxfRhXNvU97DH3CBab +apojOJ9l1pr4UfEaNcTcqQ== -----END PRIVATE KEY----- diff --git a/tests/data/nats/nats-server.pem b/tests/data/nats/nats-server.pem index b1a731b0a2f2e..4c91fa8ea5c64 100644 --- a/tests/data/nats/nats-server.pem +++ b/tests/data/nats/nats-server.pem @@ -1,25 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIERzCCAq+gAwIBAgIQd6B8dtqQiVfwVmPSHbgTmjANBgkqhkiG9w0BAQsFADBd -MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGTAXBgNVBAsMEHRvYnlA -Y29uc2lnbGllcmUxIDAeBgNVBAMMF21rY2VydCB0b2J5QGNvbnNpZ2xpZXJlMB4X -DTIyMDUxOTE3MTY0OFoXDTI0MDgxOTE3MTY0OFowRDEnMCUGA1UEChMebWtjZXJ0 -IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMRkwFwYDVQQLDBB0b2J5QGNvbnNpZ2xp -ZXJlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv4VJgSZaPHa2rVxr -EzZAVRBcj1FWzRE8890I7bXeuxBmBahUZtczjxu/Sx2PAmomhzNNaPqmYo1uvnN8 -iASGDmkUMYpB5wVFvn/h38CJqne0bGIsS+qZ30Ww+FBWvdCFDr9CanhS7wzTG5m9 -vVfREhOpXv//BSoBx+++4qCo7sUr25dSuZLJF5Lc0qYxC52gX+ce+Fxd9pNt1Ls9 -yzTCv+fjiKBSAqjOhYdeLEaH/NdMTffCxz2Ws4zEJjrKYWY2vGGH+3ZyQJbfKIgr -krk5ghFwrYAiYVtDmsLjbEWQnvsGYTk+1411tO8drQiw48kT+HJH/Xu8h0R3qX36 -9A/CgwIDAQABo4GbMIGYMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEF -BQcDATAfBgNVHSMEGDAWgBQATkRl75D0OhKHXN5CHWPi6OCc8TBQBgNVHREESTBH -gglsb2NhbGhvc3SCCG5hdHMtdGxzghRuYXRzLXRscy1jbGllbnQtY2VydIIIbmF0 -cy1qd3SHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggGBAAPC47Ps -+WW66xMyWyIgpjUWhZLohWkDe5r70R+rJgKSNmYhG8p/VreZsai3Ita/INNVf8v5 -81eBdPXna6xYcHcTdoi7NVR4y4C+8/6Nmw3IAuWv5frqXEAm5tUJqvWtjlZwwP7g -imHb8IyhPxAbVRE9v3iyaQYv+1r31sYr17rYH3xP9bQvG8wXrId3xhqhvKufBenT -dtByl9gV5aKCnC3Sw83ymW9wYXXsAfsKjdE7CypIBZ+l3m+Xht/13RlRtx+YvHsp -B5fqmziQHRkaWMrTX8vg8bBYJdnUOdtK0Z42x0bEzsWI9dvFZsXiyC7aCkvDnHTZ -rArxVMsoLfN4nzoTkbpJzPUbgaS3gliscfelKopiV5eTbBt0Ju5QvxJMFJrPdujC -rau6ovdwHzmGcLNh9bd3nR3CxMF+CECf5zHxZqUdiA0SyeFcSW6egqk5VASAw0j5 -j6NGFRJB+Fzr8dqAvzKK/dZWv/YEb5Ar2DAhFj4T+qvkj+ZU9BUaxx16Ow== +MIIEnzCCAwegAwIBAgIQPlim3fviZEd9GCJB453iHzANBgkqhkiG9w0BAQsFADCB +lzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMTYwNAYDVQQLDC1qZXNz +ZS5zendlZGtvQENPTVAtSjRDNFAyN0s5USAoSmVzc2UgU3p3ZWRrbykxPTA7BgNV +BAMMNG1rY2VydCBqZXNzZS5zendlZGtvQENPTVAtSjRDNFAyN0s5USAoSmVzc2Ug +U3p3ZWRrbykwHhcNMjQwODE5MjM0NTEwWhcNMjYxMTIwMDA0NTEwWjBhMScwJQYD +VQQKEx5ta2NlcnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxNjA0BgNVBAsMLWpl +c3NlLnN6d2Vka29AQ09NUC1KNEM0UDI3SzlRIChKZXNzZSBTendlZGtvKTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALAyhqhfs6tPYA7ZA9N8cbF+AfHb +PsVLDSCdg+7NUVkMGZdor8eRjlhutrRqCwFVCjYg5OO4dd2tvoxGeFOB7FjD4OGj +5M/EPKVoSiceYwP6YpTkxa55fawICYHinUThTOX9omS3nmYlHAAk+5rGYSn6S4Ri +drbQbzQzQyv3rZd7mY2uNPcM/LGg0A5JkiHdZCpjQGkEHaDjq3JbVuFjV+53jxCn +MhypWrtxGiS/Yp0wfp/MEaNtKLklzupiG0DnoNni1VPPAUKpJNepCoX3UjhjkAj7 +RpTuGMsljgA7fFQOJ/s+YSPg4azWKxigCQZfZ4fvoM4u7osSBte0FHgV4YsCAwEA +AaOBmzCBmDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHwYD +VR0jBBgwFoAUDkVgm5TplnP6tMU8mnT821lLJX4wUAYDVR0RBEkwR4IJbG9jYWxo +b3N0gghuYXRzLXRsc4IUbmF0cy10bHMtY2xpZW50LWNlcnSCCG5hdHMtand0hxAA +AAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEBCwUAA4IBgQCLKgHUxF8uGHguOHDy +VSmzm7ZL649WN4VJm46L4nyssUKAGqkjjZTPd/zCaNZuMs+oDaKytR9ACFgTFUc/ +53/q1P+ZwCGXAAtp48MykMa09HZF7ZMNoMX8YRfjsyHD6OkpYNxRnF46OA4glhhX +rqyYnPwZutrsj3lcb5Jj/HpxyxNX89fW6b8A6Zzi9P9U+MgivTtr+Lk7/2TYjSw0 +sQ+g/Clj+OrTKYQe1KQO/iR29O8ibYJn9PM3ye+zML+Mqkbko3EGp3JIPS+nsAat +/MNDDcVu6j84BENl4yzmEl1L58qJ5AnXjCxi3siu7dAfk+MbaAPw5BMpUM2QMbza +W4jHgFqIOqNwZrkWa2lBc1RT4lsEhGCtuTj1SRLJ2w206aMv8CpZ67SvotU7HpiR +akBRnHeyDq2MDDSFNBh/6BPh9Mw3xMYNy4ujP6EIkqjbyeV+hprrGhjF6FG58O2u +cIUwATvNzat6IxeFhQvdUdvhyTIpEwyMTAnBtteIALJu4JU= -----END CERTIFICATE----- diff --git a/tests/data/nats/rootCA.pem b/tests/data/nats/rootCA.pem index e5c8a440d5c5d..2ccf5d7a4f47e 100644 --- a/tests/data/nats/rootCA.pem +++ b/tests/data/nats/rootCA.pem @@ -1,27 +1,29 @@ -----BEGIN CERTIFICATE----- -MIIEijCCAvKgAwIBAgIRANYEd0cmu9WUolBDl/xC6BAwDQYJKoZIhvcNAQELBQAw -XTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRkwFwYDVQQLDBB0b2J5 -QGNvbnNpZ2xpZXJlMSAwHgYDVQQDDBdta2NlcnQgdG9ieUBjb25zaWdsaWVyZTAe -Fw0yMjA1MTkxNzE2NDhaFw0zMjA1MTkxNzE2NDhaMF0xHjAcBgNVBAoTFW1rY2Vy -dCBkZXZlbG9wbWVudCBDQTEZMBcGA1UECwwQdG9ieUBjb25zaWdsaWVyZTEgMB4G -A1UEAwwXbWtjZXJ0IHRvYnlAY29uc2lnbGllcmUwggGiMA0GCSqGSIb3DQEBAQUA -A4IBjwAwggGKAoIBgQDXWXInef4jNMeVpn+nFYEw0C5qIKpZrXsG61MJT4fR7oXq -QKY3b7YJJpgWIzJbct/N68h3Sd16/M3/diOgA4PeQgEl5fImXsqKJk6rwT4Y9nda -YtW9ScSaH2Ljx79eDzQIAq7/AIJaTIihhGD7krmADIq0xrEDrShg68bGAz8LbNCD -E7cG0hLZjkgeapXCcS5IZW9Of7ug4DshfJbWMe5PwUdQVxvD1Kyt+vu7AGM61xft -R6EyKxJqT84Zu2IkT0ESZV6XenUUhxoxxjg9IgDFOnCV7P4a3KuI3ukYGoavAMOM -KQrj3SUPsEN3+9mjEuNi1cnTM1UkzNAxaNHsPlB6iQttOaOQsUyFhWDdo6mVpE/N -mwYAsDZnKWWFecwrBQ0QeedlbawygKhO0qcFG/1mCfq6iA3JdRg2o9BOp8XszkSw -o5K0elX5XUdNmvJ8u/z8+obapojy5pciyg2ibiE4PggGuhOOfwETq3iUjiyyDm5h -aR3pC/O6t8UXzytTbmkCAwEAAaNFMEMwDgYDVR0PAQH/BAQDAgIEMBIGA1UdEwEB -/wQIMAYBAf8CAQAwHQYDVR0OBBYEFABORGXvkPQ6Eodc3kIdY+Lo4JzxMA0GCSqG -SIb3DQEBCwUAA4IBgQC+J8f/5ESsCR2r19iQgjbTszNWywl8417UtyOnEnfyHJW5 -2kbSnFZH44oWc1IA3Chpjio4WzBMcA7xBl/eSiLEz7YPHHU+ALyxBnoFgRU1p05D -kJ5nV+DQpFlcHx+zYFVMeVZUisr8hWaFeYK7AizT4iUzl3hhweLx/AG0TU4+vFS4 -HemUUD77ZN5/vVCK8mWpuHHl4F1tppSZe30ThxLFty7JuEtSE8Ey6GXMoRMxXa6u -ESSUlOYZZkzdq+QtFIU6lDI/xIEcsklu+khhl7Z3GgosoKhzmUAeLKxwoWSna7dN -MN1KhlIc1YoMjW1G452kBSi6Gbp4bb0C6WgRAxdNS1mwPfR5GDOXtbfRJntPSlKI -MSDGSRk09gzytbbIdD03pfqucXK0gbssaWK1tluOa1sFwFm9ks3iU6ROFfDNCCPi -hzLJScytCKjhJPALsTUQwMbs3KaEs1AmMc3ZwijS2rD2qFLMAFG2QAFa2Zk5TG4k -6o9xp5+hnXojLdvP8I0= +MIIFADCCA2igAwIBAgIRAKYxAhOd9aP0FJVL6jwBQjIwDQYJKoZIhvcNAQELBQAw +gZcxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTE2MDQGA1UECwwtamVz +c2Uuc3p3ZWRrb0BDT01QLUo0QzRQMjdLOVEgKEplc3NlIFN6d2Vka28pMT0wOwYD +VQQDDDRta2NlcnQgamVzc2Uuc3p3ZWRrb0BDT01QLUo0QzRQMjdLOVEgKEplc3Nl +IFN6d2Vka28pMB4XDTI0MDgxOTIzMTMwM1oXDTM0MDgxOTIzMTMwM1owgZcxHjAc +BgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTE2MDQGA1UECwwtamVzc2Uuc3p3 +ZWRrb0BDT01QLUo0QzRQMjdLOVEgKEplc3NlIFN6d2Vka28pMT0wOwYDVQQDDDRt +a2NlcnQgamVzc2Uuc3p3ZWRrb0BDT01QLUo0QzRQMjdLOVEgKEplc3NlIFN6d2Vk +a28pMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAymeZF22eB4uciDuF +AmYsUfR8kmnn2N/ctlDxY9QgIGcyn1xpRE89S6aH4jFrvRjU1vEfMZkQie9w8yGU +G42CdibK+THBUIKxmfopRfwQjkoSgTCv16GEu14ibyQTL+MhgnGWm2sAdb0dDCNO +axNWi631Mh9TGDahVZ27/AqNTPfOUPvq47HbowWhRjVyK5assMOecGo/+tkg3em8 +TbMFd/EBknW/SvclPDyM393qb9XLPfOsm0dJ4EFrmXYrn8Ghl2sFHXpm9zAaSET4 +uF41rNm4bDH88QvQpRFnXxIJK0vO9FnpYLymfj3hMfMY8v/wgQ/1gadA/r5W9nEJ +zzhLZJuKzX6hMhTMDxXJKzlAEQV5J1geW0le0dP6dYswi0d9n0srJKeeJ0hS7Hqe +GWNS5zKRWYQq33jZyCg20LbweqMC1q8Nb8zGySsYD+LP84hCK4TQMyVf3CqlJrPc +Mwo10bFicWO/xrThg7pSwdToTarGMMOqznAJQiqG/fD7BcvdAgMBAAGjRTBDMA4G +A1UdDwEB/wQEAwICBDASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQORWCb +lOmWc/q0xTyadPzbWUslfjANBgkqhkiG9w0BAQsFAAOCAYEAxiZp3frU+6WMaozI +KGK7pGVtLFY/cMyDfAwN2JMLNwDhvDLhikeXNGMnndXxQsAYdXrvbL40TYsELm/L +vSiV3cMnRPN+xTC1Z+5VaVhvzcwZhMJ0y/CAEAbWACCNQfdqJxzfSoBPw6hTnbGC +EvzfbcpOW7thqHCeK3Wvzqa9bTGuJEPBj+LJemKB3dvtPSDyPGXhH9YklN0p+1Ss +i1rkJIc7y1uobU5lKC155uEO95b+F1i4RJs0pc7ou3lwDewtIcqUUQnm6dyR9Zi4 +vm3w/jqxlnP/CqLgnHDXyFZ+GCxgmyhQ7duR3ghjmLwUGuZtKaORFCzsY9Pruu4x +ado1nsjbUFskJ7s4dIgt6otrMggmvLArdCaaQgo7ml2lgQzYt+HzX1EeFdQ4uwsp +6QgA2dxwvWPMk6fbQLqpaWYhYZ4PAFf1JCpydnwC1uQmY7nYNj11EF6vSK59SRIh +pTnln020J7kUVCwtHokydn/sbTuHr0pivrD8PSFgLZ6qnozH -----END CERTIFICATE----- diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 563eab8b1a47c..7d9c5f251ea58 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -12,7 +12,7 @@ anyhow = "1.0.86" chrono.workspace = true clap.workspace = true clap-verbosity-flag = "2.2.1" -clap_complete = "4.5.14" +clap_complete = "4.5.23" confy = "0.6.1" directories = "5.0.1" # remove this when stabilized https://doc.rust-lang.org/stable/std/path/fn.absolute.html diff --git a/website/content/en/releases/0.40.1.md b/website/content/en/releases/0.40.1.md new file mode 100644 index 0000000000000..5e9ff496c086c --- /dev/null +++ b/website/content/en/releases/0.40.1.md @@ -0,0 +1,4 @@ +--- +title: Vector v0.40.1 release notes +weight: 21 +--- diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 8d9f0b31144b3..2f36235549807 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -626,7 +626,7 @@ base: components: sinks: aws_s3: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index bdd87633fd602..147a65125f54f 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -443,7 +443,7 @@ base: components: sinks: azure_blob: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index be76387deb9f7..9e90891cc23b9 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -294,7 +294,7 @@ base: components: sinks: console: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index 09a05f3202575..d22a799caf617 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -314,7 +314,7 @@ base: components: sinks: file: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 810daba581b47..7f3095c6050ec 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -497,7 +497,7 @@ base: components: sinks: gcp_cloud_storage: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/honeycomb.cue b/website/cue/reference/components/sinks/base/honeycomb.cue index 4c9fd5e837030..e7561234c930a 100644 --- a/website/cue/reference/components/sinks/base/honeycomb.cue +++ b/website/cue/reference/components/sinks/base/honeycomb.cue @@ -97,6 +97,14 @@ base: components: sinks: honeycomb: configuration: { } } } + endpoint: { + description: "Honeycomb's endpoint to send logs to" + required: false + type: string: { + default: "https://api.honeycomb.io" + examples: ["https://api.honeycomb.io", "https://api.eu1.honeycomb.io"] + } + } request: { description: """ Middleware settings for outbound requests. diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 3675a85c8420b..6630fd76347c3 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -407,7 +407,7 @@ base: components: sinks: http: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index 268ceeb3246a0..563a81d463310 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -307,7 +307,7 @@ base: components: sinks: socket: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index f169a416de32f..e3cac4f1fd012 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -376,7 +376,7 @@ base: components: sinks: webhdfs: configuration: { type: object: options: delimiter: { description: "The ASCII (7-bit) character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } } length_delimited: { diff --git a/website/cue/reference/components/sources/base/amqp.cue b/website/cue/reference/components/sources/base/amqp.cue index eeba20ddf3b49..6e5ad1d0bae17 100644 --- a/website/cue/reference/components/sources/base/amqp.cue +++ b/website/cue/reference/components/sources/base/amqp.cue @@ -309,7 +309,7 @@ base: components: sources: amqp: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue index 2cba53e9f847e..e9941fc2f23ee 100644 --- a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue @@ -307,7 +307,7 @@ base: components: sources: aws_kinesis_firehose: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 4538062a02679..43fd7ed0dd239 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -407,7 +407,7 @@ base: components: sources: aws_s3: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 0d4377a71eab8..e96b82f0967a7 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -411,7 +411,7 @@ base: components: sources: aws_sqs: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/datadog_agent.cue b/website/cue/reference/components/sources/base/datadog_agent.cue index 352d34b09912a..426e8c29eb940 100644 --- a/website/cue/reference/components/sources/base/datadog_agent.cue +++ b/website/cue/reference/components/sources/base/datadog_agent.cue @@ -304,7 +304,7 @@ base: components: sources: datadog_agent: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/demo_logs.cue b/website/cue/reference/components/sources/base/demo_logs.cue index b24b6bb96197d..99c0f3c62ade3 100644 --- a/website/cue/reference/components/sources/base/demo_logs.cue +++ b/website/cue/reference/components/sources/base/demo_logs.cue @@ -300,7 +300,7 @@ base: components: sources: demo_logs: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index ee5ac7f67b7e2..eb35bb8826222 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -288,7 +288,7 @@ base: components: sources: exec: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/file_descriptor.cue b/website/cue/reference/components/sources/base/file_descriptor.cue index bdb44c7a93239..84941688d4cac 100644 --- a/website/cue/reference/components/sources/base/file_descriptor.cue +++ b/website/cue/reference/components/sources/base/file_descriptor.cue @@ -266,7 +266,7 @@ base: components: sources: file_descriptor: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/gcp_pubsub.cue b/website/cue/reference/components/sources/base/gcp_pubsub.cue index 4cd21c96a17df..adf442059641b 100644 --- a/website/cue/reference/components/sources/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/base/gcp_pubsub.cue @@ -343,7 +343,7 @@ base: components: sources: gcp_pubsub: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 060b03da1ca64..ba5600f6c3e3f 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -301,7 +301,7 @@ base: components: sources: heroku_logs: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index 6ae781d019c90..dc02088bd56c2 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -316,7 +316,7 @@ base: components: sources: http: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/http_client.cue b/website/cue/reference/components/sources/base/http_client.cue index c2ca5a39190d1..dbe7cc3761874 100644 --- a/website/cue/reference/components/sources/base/http_client.cue +++ b/website/cue/reference/components/sources/base/http_client.cue @@ -304,7 +304,7 @@ base: components: sources: http_client: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index ebb5095f87d01..cb28b7e22c8a7 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -316,7 +316,7 @@ base: components: sources: http_server: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/kafka.cue b/website/cue/reference/components/sources/base/kafka.cue index d5a7f8dbd8f77..18f3ecb3fd1ce 100644 --- a/website/cue/reference/components/sources/base/kafka.cue +++ b/website/cue/reference/components/sources/base/kafka.cue @@ -337,7 +337,7 @@ base: components: sources: kafka: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/nats.cue b/website/cue/reference/components/sources/base/nats.cue index e7bb596422861..c1c582d5c2d30 100644 --- a/website/cue/reference/components/sources/base/nats.cue +++ b/website/cue/reference/components/sources/base/nats.cue @@ -356,7 +356,7 @@ base: components: sources: nats: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/pulsar.cue b/website/cue/reference/components/sources/base/pulsar.cue index 7f4d00bba16a9..7eb0cd6795116 100644 --- a/website/cue/reference/components/sources/base/pulsar.cue +++ b/website/cue/reference/components/sources/base/pulsar.cue @@ -367,7 +367,7 @@ base: components: sources: pulsar: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/redis.cue b/website/cue/reference/components/sources/base/redis.cue index 791776717814f..05a73fbc1538e 100644 --- a/website/cue/reference/components/sources/base/redis.cue +++ b/website/cue/reference/components/sources/base/redis.cue @@ -274,7 +274,7 @@ base: components: sources: redis: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index 8cb212d68359c..898d438023dc6 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -276,7 +276,7 @@ base: components: sources: socket: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/components/sources/base/stdin.cue b/website/cue/reference/components/sources/base/stdin.cue index 6a2fb7056289e..3079d25a3c91b 100644 --- a/website/cue/reference/components/sources/base/stdin.cue +++ b/website/cue/reference/components/sources/base/stdin.cue @@ -259,7 +259,7 @@ base: components: sources: stdin: configuration: { delimiter: { description: "The character that delimits byte sequences." required: true - type: uint: {} + type: ascii_char: {} } max_length: { description: """ diff --git a/website/cue/reference/releases/0.40.0.cue b/website/cue/reference/releases/0.40.0.cue index e34d72609e86d..e7c017578cd31 100644 --- a/website/cue/reference/releases/0.40.0.cue +++ b/website/cue/reference/releases/0.40.0.cue @@ -15,6 +15,10 @@ releases: "0.40.0": { This release contains a mix of enhancements and bug fixes. See the changelog below. """ + known_issues: [ + "A regression in the `reduce` transform caused it to not group top level objects correctly (see [#21065](https://github.com/vectordotdev/vector/issues/21065)). This is fixed in v0.40.1.", + ] + changelog: [ { type: "enhancement" diff --git a/website/cue/reference/releases/0.40.1.cue b/website/cue/reference/releases/0.40.1.cue new file mode 100644 index 0000000000000..7219b58425fb2 --- /dev/null +++ b/website/cue/reference/releases/0.40.1.cue @@ -0,0 +1,36 @@ +package metadata + +releases: "0.40.1": { + date: "2024-08-26" + codename: "" + + whats_next: [] + + description: """ + The Vector team is pleased to announce version 0.40.1! + + This is the first release with packages signed with the new GPG key as part of [Datadog's key + rotation](https://docs.datadoghq.com/agent/guide/linux-key-rotation-2024/?tab=debianubuntu). + This should be a transparent change for users as the package repository setup script + [setup.vector.dev](https://setup.vector.dev) has already been importing the new key; + however, if you were manually managing the trusted GPG keys, you will need to update to + the newer ones (either [apt](https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public) or + [rpm](https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public). + + This release also fixes a regression in Vector v0.40.0. + """ + + changelog: [ + { + type: "fix" + description: """ + Fixes a Vector v0.40.0 regression where the `reduce` transform would not group top + level objects correctly. + """ + }, + ] + + commits: [ + {sha: "e9455f6596a138fc8d0ea978897b7a697c77fbb3", date: "2024-08-17 05:29:36 UTC", description: "use the correct merge strategy for top level objects", pr_number: 21067, scopes: ["reduce transform"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 332, deletions_count: 36}, + ] +} diff --git a/website/cue/reference/remap/functions/match_datadog_query.cue b/website/cue/reference/remap/functions/match_datadog_query.cue index afb7b39a7f361..bef981f5792e6 100644 --- a/website/cue/reference/remap/functions/match_datadog_query.cue +++ b/website/cue/reference/remap/functions/match_datadog_query.cue @@ -39,9 +39,9 @@ remap: functions: match_datadog_query: { return: false }, { - title: "Facet wildcard" + title: "Attribute wildcard" source: #""" - match_datadog_query({"custom": {"name": "foo"}}, "@name:foo*") + match_datadog_query({"name": "foobar"}, "@name:foo*") """# return: true }, diff --git a/website/cue/reference/remap/functions/to_int.cue b/website/cue/reference/remap/functions/to_int.cue index 1c8f3dcb48d41..688c59704c0a2 100644 --- a/website/cue/reference/remap/functions/to_int.cue +++ b/website/cue/reference/remap/functions/to_int.cue @@ -13,7 +13,7 @@ remap: functions: to_int: { The value to convert to an integer. """ required: true - type: ["integer", "float", "boolean", "string", "timestamp"] + type: ["integer", "float", "boolean", "string", "timestamp", "null"] }, ] internal_failure_reasons: [ @@ -28,6 +28,7 @@ remap: functions: to_int: { "If `value` is a string, it must be the string representation of an integer or else an error is raised.", "If `value` is a boolean, `0` is returned for `false` and `1` is returned for `true`.", "If `value` is a timestamp, a [Unix timestamp](\(urls.unix_timestamp)) (in seconds) is returned.", + "If `value` is null, `0` is returned.", ] } diff --git a/website/cue/reference/versions.cue b/website/cue/reference/versions.cue index cb98db8d228a3..7d573a2773148 100644 --- a/website/cue/reference/versions.cue +++ b/website/cue/reference/versions.cue @@ -2,6 +2,7 @@ package metadata // This has to be maintained manually because there's currently no way to sort versions programmatically versions: [string, ...string] & [ + "0.40.1", "0.40.0", "0.39.0", "0.38.0", diff --git a/website/yarn.lock b/website/yarn.lock index d8a563135ae43..37a1f668b5a30 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2051,7 +2051,7 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -braces@^3.0.1, braces@~3.0.2: +braces@^3.0.3, braces@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== @@ -3917,12 +3917,12 @@ methods@~1.1.2: integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== micromatch@^4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" - integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.1" - picomatch "^2.2.3" + braces "^3.0.3" + picomatch "^2.3.1" mime-db@1.52.0: version "1.52.0" @@ -4271,7 +4271,7 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: +picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==