From b265518e5c4b014723948f3eed93b982373ef2bc Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Wed, 11 Dec 2024 09:00:57 -0500 Subject: [PATCH 1/8] wip abs --- Cargo.lock | 757 ++++++++++++++++++--- misc/python/materialize/cli/run.py | 8 +- src/mz/Cargo.toml | 2 +- src/ore/src/lgbytes.rs | 3 + src/persist-client/src/internal/metrics.rs | 5 +- src/persist/Cargo.toml | 3 + src/persist/src/abs.rs | 314 +++++++++ src/persist/src/cfg.rs | 26 +- src/persist/src/lib.rs | 1 + src/persist/src/metrics.rs | 15 + 10 files changed, 1019 insertions(+), 115 deletions(-) create mode 100644 src/persist/src/abs.rs diff --git a/Cargo.lock b/Cargo.lock index 0872343da2a38..f54ba2a983bca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.17.0" @@ -25,7 +31,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.10", "once_cell", "version_check", "zerocopy", @@ -354,7 +360,30 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" dependencies = [ - "event-listener", + "event-listener 5.3.1", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -375,6 +404,73 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.5.0", + "parking", + "polling", + "rustix 0.38.34", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel 2.3.1", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.3.1", + "futures-lite 2.5.0", + "rustix 0.38.34", + "tracing", +] + +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 0.38.34", + "signal-hook-registry", + "slab", + "windows-sys 0.59.0", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -397,6 +493,12 @@ dependencies = [ "syn 2.0.63", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.68" @@ -484,7 +586,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.0.1", "hex", "http 0.2.9", "hyper 0.14.27", @@ -523,7 +625,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.0.1", "http 0.2.9", "http-body 0.4.5", "percent-encoding", @@ -553,7 +655,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "fastrand", + "fastrand 2.0.1", "hex", "hmac", "http 0.2.9", @@ -583,7 +685,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.0.1", "http 0.2.9", "once_cell", "regex-lite", @@ -775,7 +877,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 2.0.1", "h2 0.3.26", "http 0.2.9", "http-body 0.4.5", @@ -936,15 +1038,122 @@ dependencies = [ "tracing", ] +[[package]] +name = "azure_core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +dependencies = [ + "async-trait", + "base64 0.22.0", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.10", + "hmac", + "http-types", + "once_cell", + "paste", + "pin-project", + "quick-xml", + "rand 0.8.5", + "reqwest 0.12.4", + "rustc_version", + "serde", + "serde_json", + "sha2", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_identity" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +dependencies = [ + "async-lock", + "async-process", + "async-trait", + "azure_core", + "futures", + "oauth2", + "pin-project", + "serde", + "time", + "tracing", + "tz-rs", + "url", + "uuid", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +dependencies = [ + "RustyXML", + "async-lock", + "async-trait", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", + "futures", + "serde", + "serde_derive", + "serde_json", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +dependencies = [ + "azure_core", + "bytes", + "futures", + "log", + "once_cell", + "serde", + "serde_json", + "time", +] + [[package]] name = "backoff" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "getrandom", + "getrandom 0.2.10", "instant", - "rand", + "rand 0.8.5", ] [[package]] @@ -1097,6 +1306,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel 2.3.1", + "async-task", + "futures-io", + "futures-lite 2.5.0", + "piper", +] + [[package]] name = "brotli" version = "3.5.0" @@ -1346,8 +1568,10 @@ checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1597,11 +1821,17 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.10", "once_cell", "tiny-keccak", ] +[[package]] +name = "const_fn" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" + [[package]] name = "convert_case" version = "0.6.0" @@ -2122,7 +2352,7 @@ dependencies = [ "futures-util", "libc", "octseq", - "rand", + "rand 0.8.5", "smallvec", "time", "tokio", @@ -2353,6 +2583,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "event-listener" version = "5.3.1" @@ -2370,7 +2606,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener", + "event-listener 5.3.1", "pin-project-lite", ] @@ -2385,7 +2621,7 @@ dependencies = [ "hyper-tls 0.5.0", "log", "pin-project", - "rand", + "rand 0.8.5", "tokio", ] @@ -2406,7 +2642,7 @@ checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" dependencies = [ "log", "once_cell", - "rand", + "rand 0.8.5", ] [[package]] @@ -2425,6 +2661,15 @@ dependencies = [ "regex", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -2595,6 +2840,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.30" @@ -2674,6 +2947,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -2683,7 +2967,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2727,7 +3011,7 @@ dependencies = [ "nonzero_ext", "parking_lot", "quanta", - "rand", + "rand 0.8.5", "smallvec", ] @@ -2902,6 +3186,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -3008,6 +3298,26 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel 1.9.0", + "base64 0.13.1", + "futures-lite 1.13.0", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.8.0" @@ -3228,7 +3538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" dependencies = [ "bitmaps", - "rand_core", + "rand_core 0.6.2", "rand_xoshiro", "sized-chunks", "typenum", @@ -3287,6 +3597,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "insta" version = "1.33.0" @@ -3468,7 +3784,7 @@ dependencies = [ "k8s-openapi", "kube", "kube-runtime", - "rand", + "rand 0.8.5", "serde", "tracing", ] @@ -3533,7 +3849,7 @@ dependencies = [ "kube-core", "openssl", "pem", - "rand", + "rand 0.8.5", "secrecy", "serde", "serde_json", @@ -4030,7 +4346,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -4089,7 +4405,7 @@ dependencies = [ "pem", "percent-encoding", "pin-project", - "rand", + "rand 0.8.5", "serde", "serde_json", "socket2 0.5.7", @@ -4123,7 +4439,7 @@ dependencies = [ "lazy_static", "num-bigint", "num-traits", - "rand", + "rand 0.8.5", "regex", "saturating", "serde", @@ -4156,7 +4472,7 @@ dependencies = [ "mz-ore", "open", "openssl-probe", - "reqwest", + "reqwest 0.12.4", "rpassword", "security-framework", "semver", @@ -4246,11 +4562,11 @@ dependencies = [ "proptest-derive", "prost", "qcell", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.0", "rdkafka", "regex", - "reqwest", + "reqwest 0.11.24", "semver", "serde", "serde_json", @@ -4346,7 +4662,7 @@ dependencies = [ "flate2", "itertools 0.10.5", "mz-ore", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -4435,7 +4751,7 @@ dependencies = [ "postgres", "prometheus", "proxy-header", - "reqwest", + "reqwest 0.11.24", "semver", "tempfile", "tokio", @@ -4523,7 +4839,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "semver", "serde", "serde_json", @@ -4587,7 +4903,7 @@ dependencies = [ "proptest", "proptest-derive", "prost-build", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "tokio", @@ -4604,7 +4920,7 @@ dependencies = [ "chrono", "mz-frontegg-auth", "mz-frontegg-client", - "reqwest", + "reqwest 0.11.24", "serde", "thiserror", "tokio", @@ -4624,7 +4940,7 @@ dependencies = [ "kube", "mz-ore", "mz-repr", - "rand", + "rand 0.8.5", "schemars", "semver", "serde", @@ -5069,11 +5385,11 @@ dependencies = [ "predicates", "prometheus", "proptest", - "rand", + "rand 0.8.5", "rdkafka", "rdkafka-sys", "regex", - "reqwest", + "reqwest 0.11.24", "rlimit", "semver", "sentry", @@ -5150,7 +5466,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "regex", "regex-syntax 0.8.3", "seahash", @@ -5216,7 +5532,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "sha2", @@ -5249,7 +5565,7 @@ dependencies = [ "mz-ore", "mz-repr", "prometheus", - "reqwest", + "reqwest 0.11.24", "reqwest-middleware", "reqwest-retry", "serde", @@ -5268,7 +5584,7 @@ dependencies = [ "jsonwebtoken", "mz-frontegg-auth", "mz-ore", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "thiserror", @@ -5293,7 +5609,7 @@ dependencies = [ "mz-frontegg-auth", "mz-ore", "openssl", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "tokio", @@ -5382,7 +5698,7 @@ dependencies = [ "num_cpus", "prost", "prost-build", - "rand", + "rand 0.8.5", "rdkafka", "serde", "serde_json", @@ -5449,7 +5765,7 @@ dependencies = [ name = "mz-metabase" version = "0.0.0" dependencies = [ - "reqwest", + "reqwest 0.11.24", "serde", "workspace-hack", ] @@ -5523,7 +5839,7 @@ dependencies = [ "flate2", "hex", "hex-literal", - "reqwest", + "reqwest 0.11.24", "sha2", "tar", "walkdir", @@ -5655,8 +5971,8 @@ dependencies = [ "mz-sql", "mz-tls-util", "prometheus", - "rand", - "reqwest", + "rand 0.8.5", + "reqwest 0.11.24", "serde", "serde_json", "sha2", @@ -5706,7 +6022,7 @@ dependencies = [ "pin-project", "prometheus", "proptest", - "rand", + "rand 0.8.5", "scopeguard", "sentry", "sentry-tracing", @@ -5753,6 +6069,9 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "aws-types", + "azure_identity", + "azure_storage", + "azure_storage_blobs", "base64 0.13.1", "bytes", "deadpool-postgres", @@ -5777,7 +6096,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", @@ -6182,7 +6501,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "regex", "ryu", "serde", @@ -6435,7 +6754,7 @@ dependencies = [ "protobuf-native", "rdkafka", "regex", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "static_assertions", @@ -6534,7 +6853,7 @@ dependencies = [ "mz-tracing", "postgres-protocol", "regex", - "reqwest", + "reqwest 0.11.24", "serde_json", "shell-words", "tempfile", @@ -6560,7 +6879,7 @@ dependencies = [ "openssh", "openssh-mux-client", "openssl", - "rand", + "rand 0.8.5", "scopeguard", "serde", "serde_json", @@ -6642,7 +6961,7 @@ dependencies = [ "postgres-replication", "prometheus", "prost", - "rand", + "rand 0.8.5", "rdkafka", "regex", "rocksdb", @@ -6845,7 +7164,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "rdkafka", "regex", "serde", @@ -6869,7 +7188,7 @@ dependencies = [ "chrono", "mz-kafka-util", "mz-ore", - "rand", + "rand 0.8.5", "rdkafka", "tokio", "tokio-postgres", @@ -6930,10 +7249,10 @@ dependencies = [ "prost-build", "prost-reflect", "prost-types", - "rand", + "rand 0.8.5", "rdkafka", "regex", - "reqwest", + "reqwest 0.11.24", "semver", "serde", "serde_json", @@ -6992,7 +7311,7 @@ dependencies = [ "mz-postgres-client", "mz-repr", "postgres-protocol", - "rand", + "rand 0.8.5", "serde", "tokio", "tracing", @@ -7081,7 +7400,7 @@ dependencies = [ "prometheus", "prost", "prost-build", - "rand", + "rand 0.8.5", "serde", "timely", "tokio", @@ -7296,6 +7615,15 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "numa_maps" version = "0.1.0" @@ -7308,6 +7636,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "oauth2" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" +dependencies = [ + "base64 0.13.1", + "chrono", + "getrandom 0.2.10", + "http 0.2.9", + "rand 0.8.5", + "serde", + "serde_json", + "serde_path_to_error", + "sha2", + "thiserror", + "url", +] + [[package]] name = "object" version = "0.29.0" @@ -7493,7 +7840,7 @@ dependencies = [ "once_cell", "opentelemetry", "percent-encoding", - "rand", + "rand 0.8.5", "serde_json", "thiserror", "tokio", @@ -7522,7 +7869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", - "rand", + "rand 0.8.5", "serde", ] @@ -7802,7 +8149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b450720b6f75cfbfabc195814bd3765f337a4f9a83186f8537297cac12f6705" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -7847,6 +8194,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.20" @@ -7881,6 +8239,21 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix 0.38.34", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "portable-atomic" version = "0.3.20" @@ -7923,7 +8296,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.7" -source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" +source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" dependencies = [ "base64 0.22.0", "byteorder", @@ -7932,7 +8305,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -7955,7 +8328,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.7" -source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" +source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" dependencies = [ "bytes", "chrono", @@ -8141,8 +8514,8 @@ dependencies = [ "lazy_static", "num-traits", "quick-error", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.0", "rand_xorshift", "regex-syntax 0.6.28", ] @@ -8328,7 +8701,7 @@ dependencies = [ "mach2", "once_cell", "raw-cpuid", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -8346,6 +8719,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", + "serde", ] [[package]] @@ -8354,7 +8728,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] @@ -8372,6 +8746,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -8379,11 +8766,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.0", + "rand_core 0.6.2", "serde", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.0" @@ -8391,7 +8788,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -8400,17 +8806,26 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom", + "getrandom 0.2.10", "serde", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core", + "rand_core 0.6.2", ] [[package]] @@ -8419,7 +8834,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core", + "rand_core 0.6.2", ] [[package]] @@ -8511,7 +8926,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom", + "getrandom 0.2.10", "redox_syscall 0.2.10", ] @@ -8591,7 +9006,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -8604,7 +9019,52 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.0", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.1", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg 0.52.0", ] [[package]] @@ -8615,7 +9075,7 @@ dependencies = [ "anyhow", "async-trait", "http 0.2.9", - "reqwest", + "reqwest 0.11.24", "serde", "task-local-extensions", "thiserror", @@ -8632,7 +9092,7 @@ dependencies = [ "futures", "http 0.2.9", "hyper 0.14.27", - "reqwest", + "reqwest 0.11.24", "reqwest-middleware", "retry-policies", "task-local-extensions", @@ -8654,7 +9114,7 @@ checksum = "e09bbcb5003282bcb688f0bae741b278e9c7e8f378f561522c9806c58e075d9b" dependencies = [ "anyhow", "chrono", - "rand", + "rand 0.8.5", ] [[package]] @@ -8664,7 +9124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", - "getrandom", + "getrandom 0.2.10", "libc", "spin", "untrusted", @@ -8778,6 +9238,21 @@ dependencies = [ "base64 0.21.5", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + [[package]] name = "rustversion" version = "1.0.9" @@ -8923,7 +9398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24fc91c898e0487ff3e471d0849bbaf7d38a00ff5e3531009d386b0bab9b6b12" dependencies = [ "async-trait", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "thiserror", @@ -8957,7 +9432,7 @@ checksum = "17ad137b9df78294b98cab1a650bef237cc6c950e82e5ce164655e674d07c5cc" dependencies = [ "httpdate", "native-tls", - "reqwest", + "reqwest 0.11.24", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -9000,7 +9475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4df9b9d8de2658a1ecd4e45f7b06c80c5dd97b891bfbc7c501186189b7e9bbdf" dependencies = [ "once_cell", - "rand", + "rand 0.8.5", "sentry-types", "serde", "serde_json", @@ -9045,7 +9520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc95faa4078768a6bf8df45e2b894bbf372b3dbbfb364e9429c1c58ab7545c6" dependencies = [ "debugid", - "getrandom", + "getrandom 0.2.10", "hex", "serde", "serde_json", @@ -9152,6 +9627,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + [[package]] name = "serde_repr" version = "0.1.13" @@ -9370,9 +9856,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -9423,7 +9912,7 @@ checksum = "f02d3730e8785e797a4552137d1acc0d7f7146dad3b5fe65ed83637711dfc6c5" dependencies = [ "base64ct", "pem-rfc7468", - "rand_core", + "rand_core 0.6.2", "sec1", "sha2", "signature", @@ -9699,7 +10188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.0.1", "once_cell", "rustix 0.38.34", "windows-sys 0.59.0", @@ -9836,6 +10325,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", + "js-sys", + "libc", + "num_threads", "quickcheck", "serde", "time-core", @@ -10042,7 +10534,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.11" -source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" +source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" dependencies = [ "async-trait", "byteorder", @@ -10057,7 +10549,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand", + "rand 0.8.5", "serde", "socket2 0.5.7", "tokio", @@ -10230,7 +10722,7 @@ dependencies = [ "indexmap 1.9.1", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -10307,11 +10799,10 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -10320,13 +10811,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.63", ] [[package]] @@ -10344,9 +10835,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -10451,7 +10942,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -10470,7 +10961,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "utf-8", @@ -10483,7 +10974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -10510,6 +11001,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "tz-rs" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" +dependencies = [ + "const_fn", +] + [[package]] name = "ucd-trie" version = "0.1.6" @@ -10650,7 +11150,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ - "getrandom", + "getrandom 0.2.10", "serde", "sha1_smol", ] @@ -10708,6 +11208,12 @@ dependencies = [ "libc", ] +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "walkdir" version = "2.3.2" @@ -10729,6 +11235,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -10768,9 +11280,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -10807,11 +11319,24 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" -version = "0.3.51" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -11113,6 +11638,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "workspace-hack" version = "0.0.0" @@ -11161,7 +11696,7 @@ dependencies = [ "futures-sink", "futures-task", "futures-util", - "getrandom", + "getrandom 0.2.10", "globset", "hashbrown 0.14.5", "hyper 0.14.27", @@ -11208,13 +11743,13 @@ dependencies = [ "prost-reflect", "prost-types", "quote", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.0", "rdkafka-sys", "regex", "regex-automata 0.4.7", "regex-syntax 0.8.3", - "reqwest", + "reqwest 0.11.24", "ring", "schemars", "scopeguard", diff --git a/misc/python/materialize/cli/run.py b/misc/python/materialize/cli/run.py index 67a6c94ec51be..4a38e12174911 100644 --- a/misc/python/materialize/cli/run.py +++ b/misc/python/materialize/cli/run.py @@ -48,6 +48,7 @@ else f"{Arch.host()}-apple-darwin" ) DEFAULT_POSTGRES = "postgres://root@localhost:26257/materialize" +DEFAULT_BLOB = "file://mzdata/persist/blob" # sets entitlements on the built binary, e.g. environmentd, so you can inspect it with Instruments MACOS_ENTITLEMENTS_DATA = """ @@ -88,6 +89,11 @@ def main() -> int: help="Postgres/CockroachDB connection string", default=os.getenv("MZDEV_POSTGRES", DEFAULT_POSTGRES), ) + parser.add_argument( + "--blob", + help="Blob storage connection string", + default=os.getenv("MZDEV_BLOB", DEFAULT_BLOB), + ) parser.add_argument( "--release", help="Build artifacts in release mode, with optimizations", @@ -276,7 +282,7 @@ def main() -> int: f"--orchestrator-process-scratch-directory={scratch}", "--secrets-controller=local-file", f"--persist-consensus-url={args.postgres}?options=--search_path=consensus", - f"--persist-blob-url=file://{mzdata}/persist/blob", + f"--persist-blob-url={args.blob}", f"--timestamp-oracle-url={args.postgres}?options=--search_path=tsoracle", f"--environment-id={environment_id}", "--bootstrap-role=materialize", diff --git a/src/mz/Cargo.toml b/src/mz/Cargo.toml index 5366a3a5997da..ad5f98128b52c 100644 --- a/src/mz/Cargo.toml +++ b/src/mz/Cargo.toml @@ -25,7 +25,7 @@ mz-ore = { path = "../ore", features = ["async", "cli", "test"] } open = "3.2.0" openssl-probe = "0.1.2" hyper = "1.4.1" -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.12", features = ["blocking", "json"] } rpassword = "7.2.0" semver = "1.0.16" serde = { version = "1.0.152", features = ["derive"] } diff --git a/src/ore/src/lgbytes.rs b/src/ore/src/lgbytes.rs index 883ce8e427ef9..cdf8cb0f31388 100644 --- a/src/ore/src/lgbytes.rs +++ b/src/ore/src/lgbytes.rs @@ -179,6 +179,8 @@ impl Buf for LgBytes { pub struct LgBytesMetrics { /// Metrics for the "persist_s3" usage of [LgBytes]. pub persist_s3: LgBytesOpMetrics, + /// Metrics for the "persist_abs" usage of [LgBytes]. + pub persist_abs: LgBytesOpMetrics, /// Metrics for the "persist_arrow" usage of [LgBytes]. pub persist_arrow: LgBytesOpMetrics, } @@ -272,6 +274,7 @@ impl LgBytesMetrics { }; LgBytesMetrics { persist_s3: op("persist_s3"), + persist_abs: op("persist_abs"), persist_arrow: op("persist_arrow"), } } diff --git a/src/persist-client/src/internal/metrics.rs b/src/persist-client/src/internal/metrics.rs index 79bae1903760f..8ea700c7bfacf 100644 --- a/src/persist-client/src/internal/metrics.rs +++ b/src/persist-client/src/internal/metrics.rs @@ -32,7 +32,7 @@ use mz_ore::stats::histogram_seconds_buckets; use mz_persist::location::{ Blob, BlobMetadata, CaSResult, Consensus, ExternalError, ResultStream, SeqNo, VersionedData, }; -use mz_persist::metrics::{ColumnarMetrics, S3BlobMetrics}; +use mz_persist::metrics::{ABSBlobMetrics, ColumnarMetrics, S3BlobMetrics}; use mz_persist::retry::RetryStream; use mz_persist_types::Codec64; use mz_postgres_client::metrics::PostgresClientMetrics; @@ -110,6 +110,8 @@ pub struct Metrics { /// Metrics for S3-backed blob implementation pub s3_blob: S3BlobMetrics, + /// Metrics for Azure-backed blob implementation + // pub azure_blob: ABSBlobMetrics, /// Metrics for Postgres-backed consensus implementation pub postgres_consensus: PostgresClientMetrics, @@ -140,6 +142,7 @@ impl Metrics { move || start.elapsed().as_secs_f64(), ); let s3_blob = S3BlobMetrics::new(registry); + // let azure_blob = ABSBlobMetrics::new(registry); let columnar = ColumnarMetrics::new( registry, &s3_blob.lgbytes, diff --git a/src/persist/Cargo.toml b/src/persist/Cargo.toml index db5e82390221a..f6852a0c2315f 100644 --- a/src/persist/Cargo.toml +++ b/src/persist/Cargo.toml @@ -30,6 +30,9 @@ aws-config = { version = "1.2.0", default-features = false } aws-credential-types = { version = "1.1.1", features = ["hardcoded-credentials"] } aws-sdk-s3 = { version = "1.23.0", default-features = false, features = ["rt-tokio"] } aws-types = "1.1.1" +azure_identity = { version = "0.21.0" } +azure_storage = { version = "0.21.0" } +azure_storage_blobs = { version = "0.21.0" } base64 = "0.13.1" bytes = "1.3.0" deadpool-postgres = "0.10.3" diff --git a/src/persist/src/abs.rs b/src/persist/src/abs.rs new file mode 100644 index 0000000000000..3ec3afe225635 --- /dev/null +++ b/src/persist/src/abs.rs @@ -0,0 +1,314 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! An Azure Blob Storage implementation of [Blob] storage. + +use std::cmp; +use std::fmt::{Debug, Formatter}; +use std::ops::Range; +use std::sync::atomic::AtomicU64; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::anyhow; +use async_trait::async_trait; +use azure_identity::{create_default_credential, DefaultAzureCredential}; +use azure_storage::{prelude::*, EMULATOR_ACCOUNT}; +use azure_storage_blobs::prelude::*; +use bytes::Bytes; +use futures_util::stream::FuturesOrdered; +use futures_util::{FutureExt, StreamExt}; +use mz_dyncfg::{Config, ConfigSet}; +use mz_ore::bytes::{MaybeLgBytes, SegmentedBytes}; +use mz_ore::cast::CastFrom; +use mz_ore::lgbytes::{LgBytes, MetricsRegion}; +use mz_ore::metrics::MetricsRegistry; +use mz_ore::task::RuntimeExt; +use tokio::runtime::Handle as AsyncHandle; +use tracing::{debug, debug_span, trace, trace_span, Instrument}; +use uuid::Uuid; + +use crate::cfg::BlobKnobs; +use crate::error::Error; +use crate::location::{Blob, BlobMetadata, Determinate, ExternalError}; +use crate::metrics::{ABSBlobMetrics, S3BlobMetrics}; + +/// Configuration for opening an [ABSBlob]. +#[derive(Clone, Debug)] +pub struct ABSBlobConfig { + metrics: S3BlobMetrics, + client: ContainerClient, + cfg: Arc, +} + +impl ABSBlobConfig { + const EXTERNAL_TESTS_ABS_CONTAINER: &'static str = + "MZ_PERSIST_EXTERNAL_STORAGE_TEST_ABS_CONTAINER"; + + /// Returns a new [ABSBlobConfig] for use in production. + /// + /// Stores objects in the given container prepended with the (possibly empty) + /// prefix. Azure credentials must be available in the process or environment. + pub fn new( + account: String, + container: String, + prefix: String, + metrics: S3BlobMetrics, + cfg: Arc, + ) -> Result { + // let is_cc_active = knobs.is_cc_active(); + + // TODO: might need to pull out service client + // to periodically refresh credentials + let client = if account == EMULATOR_ACCOUNT { + let credentials = StorageCredentials::emulator(); + let service_client = BlobServiceClient::new(account, credentials); + service_client.container_client(container) + } else { + let credentials = + create_default_credential().expect("default Azure credentials working"); + let service_client = BlobServiceClient::new(account, credentials); + service_client.container_client(container) + }; + + let credential = create_default_credential().expect("default Azure credentials working"); + + Ok(ABSBlobConfig { + metrics, + client, + cfg, + }) + } + + /// Returns a new [ABSBlobConfig] for use in unit tests. + pub async fn new_for_test() -> Result, Error> { + let container_name = match std::env::var(Self::EXTERNAL_TESTS_ABS_CONTAINER) { + Ok(container) => container, + Err(_) => { + if mz_ore::env::is_var_truthy("CI") { + panic!("CI is supposed to run this test but something has gone wrong!"); + } + return Ok(None); + } + }; + + // let prefix = Uuid::new_v4().to_string(); + let metrics = S3BlobMetrics::new(&MetricsRegistry::new()); + + let config = ABSBlobConfig::new( + EMULATOR_ACCOUNT.to_string(), + container_name, + "".to_string(), + metrics, + Arc::new(ConfigSet::default()), + )?; + + Ok(Some(config)) + } + + /// Returns a clone of Self with a new v4 uuid prefix. + pub fn clone_with_new_uuid_prefix(&self) -> Self { + let mut ret = self.clone(); + // ret.prefix = Uuid::new_v4().to_string(); + ret + } +} + +/// Implementation of [Blob] backed by Azure Blob Storage. +#[derive(Debug)] +pub struct ABSBlob { + metrics: S3BlobMetrics, + client: ContainerClient, + container_name: String, + prefix: String, + cfg: Arc, +} + +impl ABSBlob { + /// Opens the given location for non-exclusive read-write access. + pub async fn open(config: ABSBlobConfig) -> Result { + let container_name = config.client.container_name().to_string(); + let ret = ABSBlob { + metrics: config.metrics, + client: config.client, + container_name, + prefix: "".to_string(), + cfg: config.cfg, + }; + + // Test connection before returning success + let _ = ret.get("HEALTH_CHECK").await?; + Ok(ret) + } + + fn get_path(&self, key: &str) -> String { + format!("{}/{}", self.prefix, key) + } +} + +#[async_trait] +impl Blob for ABSBlob { + async fn get(&self, key: &str) -> Result, ExternalError> { + let path = self.get_path(key); + + let blob = self.client.blob_client(path); + + let mut segments: Vec = vec![]; + + let mut stream = blob.get().into_stream(); + while let Some(value) = stream.next().await { + let response = + value.map_err(|e| ExternalError::from(anyhow!("Azure blob get error: {}", e)))?; + + let content_length = response.blob.properties.content_length; + let mut buffer = self + .metrics + .lgbytes + .persist_abs + .new_region(usize::cast_from(content_length)); + + let mut body = response.data; + while let Some(value) = body.next().await { + let value = value.map_err(|e| { + ExternalError::from(anyhow!("Azure blob get body error: {}", e)) + })?; + buffer.extend_from_slice(&value); + } + + segments.push(MaybeLgBytes::LgBytes(LgBytes::from(Arc::new(buffer)))); + } + + Ok(Some(SegmentedBytes::from(segments))) + } + + async fn list_keys_and_metadata( + &self, + key_prefix: &str, + f: &mut (dyn FnMut(BlobMetadata) + Send + Sync), + ) -> Result<(), ExternalError> { + let blob_key_prefix = self.get_path(key_prefix); + let strippable_root_prefix = format!("{}/", self.prefix); + + let mut stream = self + .client + .list_blobs() + .prefix(blob_key_prefix) + .into_stream(); + + while let Some(response) = stream.next().await { + let response = response + .map_err(|e| ExternalError::from(anyhow!("Azure list blobs error: {}", e)))?; + + for blob in response.blobs.items { + let azure_storage_blobs::container::operations::list_blobs::BlobItem::Blob(blob) = + blob + else { + continue; + }; + + if let Some(key) = blob.name.strip_prefix(&strippable_root_prefix) { + let size_in_bytes = blob.properties.content_length; + f(BlobMetadata { + key: key, + size_in_bytes, + }); + } + } + } + + Ok(()) + } + + async fn set(&self, key: &str, value: Bytes) -> Result<(), ExternalError> { + let path = self.get_path(key); + let blob = self.client.blob_client(path); + + blob.put_block_blob(value) + .await + .map_err(|e| ExternalError::from(anyhow!("Azure blob put error: {}", e)))?; + + Ok(()) + } + + async fn delete(&self, key: &str) -> Result, ExternalError> { + let path = self.get_path(key); + let blob = self.client.blob_client(path); + + match blob.get_properties().await { + Ok(props) => { + let size = props.blob.properties.content_length as usize; + blob.delete() + .await + .map_err(|e| ExternalError::from(anyhow!("Azure blob delete error: {}", e)))?; + Ok(Some(size)) + } + // WIP: figure out what this is if empty + // Err(e) if e.status_code() == 404 => Ok(None), + Err(e) => Err(ExternalError::from(anyhow!("Azure blob error: {}", e))), + } + } + + async fn restore(&self, key: &str) -> Result<(), ExternalError> { + let path = self.get_path(key); + let blob = self.client.blob_client(&path); + + match blob.get_properties().await { + Ok(_) => Ok(()), + // WIP + // Err(e) if e.status_code() == 404 => Err(Determinate::new(anyhow!( + // "unable to restore {key} in Azure Blob Storage: blob does not exist" + // )) + // .into()), + Err(e) => Err(ExternalError::from(anyhow!("Azure blob error: {}", e))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::location::tests::blob_impl_test; + use tracing::info; + + #[mz_ore::test(tokio::test(flavor = "multi_thread"))] + async fn abs_blob() -> Result<(), ExternalError> { + let config = match ABSBlobConfig::new_for_test().await? { + Some(client) => client, + None => { + info!( + "{} env not set: skipping test that uses external service", + ABSBlobConfig::EXTERNAL_TESTS_ABS_CONTAINER + ); + return Ok(()); + } + }; + + blob_impl_test(move |path| { + let path = path.to_owned(); + let config = config.clone(); + async move { + let config = ABSBlobConfig { + metrics: config.metrics.clone(), + client: config.client.clone(), + container_name: config.container_name.clone(), + prefix: format!("{}/abs_blob_impl_test/{}", config.prefix, path), + cfg: Arc::new( + ConfigSet::default() + .add(&ENABLE_ABS_LGALLOC_CC_SIZES) + .add(&ENABLE_ABS_LGALLOC_NONCC_SIZES) + .add(&ENABLE_ONE_ALLOC_PER_REQUEST), + ), + is_cc_active: true, + }; + ABSBlob::open(config).await + } + }) + .await + } +} diff --git a/src/persist/src/cfg.rs b/src/persist/src/cfg.rs index a191d09fd8b84..687330d333ede 100644 --- a/src/persist/src/cfg.rs +++ b/src/persist/src/cfg.rs @@ -21,10 +21,11 @@ use tracing::warn; use mz_postgres_client::metrics::PostgresClientMetrics; use mz_postgres_client::PostgresClientKnobs; +use crate::abs::{ABSBlob, ABSBlobConfig}; use crate::file::{FileBlob, FileBlobConfig}; use crate::location::{Blob, Consensus, Determinate, ExternalError}; use crate::mem::{MemBlob, MemBlobConfig, MemConsensus}; -use crate::metrics::S3BlobMetrics; +use crate::metrics::{ABSBlobMetrics, S3BlobMetrics}; use crate::postgres::{PostgresConsensus, PostgresConsensusConfig}; use crate::s3::{S3Blob, S3BlobConfig}; @@ -48,6 +49,8 @@ pub enum BlobConfig { /// Config for [MemBlob], only available in testing to prevent /// footguns. Mem(bool), + /// Config for [ABSBlob]. + Azure(ABSBlobConfig), } /// Configuration knobs for [Blob]. @@ -70,6 +73,7 @@ impl BlobConfig { match self { BlobConfig::File(config) => Ok(Arc::new(FileBlob::open(config).await?)), BlobConfig::S3(config) => Ok(Arc::new(S3Blob::open(config).await?)), + BlobConfig::Azure(config) => Ok(Arc::new(ABSBlob::open(config).await?)), BlobConfig::Mem(tombstone) => { Ok(Arc::new(MemBlob::open(MemBlobConfig::new(tombstone)))) } @@ -148,6 +152,26 @@ impl BlobConfig { query_params.clear(); Ok(BlobConfig::Mem(tombstone)) } + "http" | "https" => match url.host().expect("hostname").to_string().split_once('.') { + Some((account, "blob.core.windows.net")) => { + if let Some(container) = url + .path_segments() + .expect("azure blob storage container") + .next() + { + Ok(BlobConfig::Azure(ABSBlobConfig::new( + account.to_string(), + container.to_string(), + "".to_string(), + metrics, + cfg, + )?)) + } else { + Err(anyhow!("unknown persist blob scheme: {}", url.as_str())) + } + } + _ => Err(anyhow!("unknown persist blob scheme: {}", url.as_str())), + }, p => Err(anyhow!( "unknown persist blob scheme {}: {}", p, diff --git a/src/persist/src/lib.rs b/src/persist/src/lib.rs index 3d9c94fc49eeb..d7ace1274e750 100644 --- a/src/persist/src/lib.rs +++ b/src/persist/src/lib.rs @@ -17,6 +17,7 @@ clippy::clone_on_ref_ptr )] +pub mod abs; pub mod cfg; pub mod error; pub mod file; diff --git a/src/persist/src/metrics.rs b/src/persist/src/metrics.rs index 39de50b85eda1..362e1e11a659b 100644 --- a/src/persist/src/metrics.rs +++ b/src/persist/src/metrics.rs @@ -18,6 +18,21 @@ use mz_ore::metric; use mz_ore::metrics::{Counter, IntCounter, MetricsRegistry}; use prometheus::{CounterVec, IntCounterVec}; +/// Metrics specific to ABSBlob's internal workings. +#[derive(Debug, Clone)] +pub struct ABSBlobMetrics { + pub(crate) lgbytes: LgBytesMetrics, +} + +impl ABSBlobMetrics { + /// Returns a new [ABSBlobMetrics] instance connected to the given registry. + pub fn new(registry: &MetricsRegistry) -> Self { + ABSBlobMetrics { + lgbytes: LgBytesMetrics::new(registry), + } + } +} + /// Metrics specific to S3Blob's internal workings. #[derive(Debug, Clone)] pub struct S3BlobMetrics { From e31c5e0bba8e4d86a5e40705dd91b40ecccf72dc Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Wed, 11 Dec 2024 17:42:00 -0500 Subject: [PATCH 2/8] wip --- src/persist/src/abs.rs | 10 ++++++---- src/storage/src/source/postgres/replication.rs | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/persist/src/abs.rs b/src/persist/src/abs.rs index 3ec3afe225635..3ac60cc815fda 100644 --- a/src/persist/src/abs.rs +++ b/src/persist/src/abs.rs @@ -31,7 +31,7 @@ use mz_ore::lgbytes::{LgBytes, MetricsRegion}; use mz_ore::metrics::MetricsRegistry; use mz_ore::task::RuntimeExt; use tokio::runtime::Handle as AsyncHandle; -use tracing::{debug, debug_span, trace, trace_span, Instrument}; +use tracing::{debug, debug_span, info, trace, trace_span, Instrument}; use uuid::Uuid; use crate::cfg::BlobKnobs; @@ -68,6 +68,10 @@ impl ABSBlobConfig { // to periodically refresh credentials let client = if account == EMULATOR_ACCOUNT { let credentials = StorageCredentials::emulator(); + info!( + "Account: {:?}, Container: {:?}, Credentials: {:?}", + account, container, credentials + ); let service_client = BlobServiceClient::new(account, credentials); service_client.container_client(container) } else { @@ -77,8 +81,6 @@ impl ABSBlobConfig { service_client.container_client(container) }; - let credential = create_default_credential().expect("default Azure credentials working"); - Ok(ABSBlobConfig { metrics, client, @@ -164,7 +166,7 @@ impl Blob for ABSBlob { let mut stream = blob.get().into_stream(); while let Some(value) = stream.next().await { let response = - value.map_err(|e| ExternalError::from(anyhow!("Azure blob get error: {}", e)))?; + value.map_err(|e| ExternalError::from(anyhow!("Azure blob get error: {:?}", e)))?; let content_length = response.blob.properties.content_length; let mut buffer = self diff --git a/src/storage/src/source/postgres/replication.rs b/src/storage/src/source/postgres/replication.rs index 9d7d5a9fc9108..6f3dccbce62d3 100644 --- a/src/storage/src/source/postgres/replication.rs +++ b/src/storage/src/source/postgres/replication.rs @@ -998,6 +998,7 @@ where TupleData::Text(bytes) => Some(bytes.clone()), TupleData::Null => None, TupleData::UnchangedToast => return Err(DefiniteError::MissingToast), + TupleData::Binary(_) => return Err(DefiniteError::MissingToast), }; row.push(datum); } From 959b11e6ea0231a720eebf0356862c176fc1ab93 Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 07:49:32 -0500 Subject: [PATCH 3/8] wip: working blob test against emulator --- Cargo.lock | 1 + src/persist/Cargo.toml | 1 + src/persist/src/abs.rs | 92 +++++++++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f54ba2a983bca..7557fdc91e002 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6069,6 +6069,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "aws-types", + "azure_core", "azure_identity", "azure_storage", "azure_storage_blobs", diff --git a/src/persist/Cargo.toml b/src/persist/Cargo.toml index f6852a0c2315f..f0ca812c58abc 100644 --- a/src/persist/Cargo.toml +++ b/src/persist/Cargo.toml @@ -33,6 +33,7 @@ aws-types = "1.1.1" azure_identity = { version = "0.21.0" } azure_storage = { version = "0.21.0" } azure_storage_blobs = { version = "0.21.0" } +azure_core = "0.21.0" base64 = "0.13.1" bytes = "1.3.0" deadpool-postgres = "0.10.3" diff --git a/src/persist/src/abs.rs b/src/persist/src/abs.rs index 3ac60cc815fda..e6da23d78a42f 100644 --- a/src/persist/src/abs.rs +++ b/src/persist/src/abs.rs @@ -18,6 +18,7 @@ use std::time::{Duration, Instant}; use anyhow::anyhow; use async_trait::async_trait; +use azure_core::StatusCode; use azure_identity::{create_default_credential, DefaultAzureCredential}; use azure_storage::{prelude::*, EMULATOR_ACCOUNT}; use azure_storage_blobs::prelude::*; @@ -44,6 +45,7 @@ use crate::metrics::{ABSBlobMetrics, S3BlobMetrics}; pub struct ABSBlobConfig { metrics: S3BlobMetrics, client: ContainerClient, + prefix: String, cfg: Arc, } @@ -67,13 +69,10 @@ impl ABSBlobConfig { // TODO: might need to pull out service client // to periodically refresh credentials let client = if account == EMULATOR_ACCOUNT { - let credentials = StorageCredentials::emulator(); - info!( - "Account: {:?}, Container: {:?}, Credentials: {:?}", - account, container, credentials - ); - let service_client = BlobServiceClient::new(account, credentials); - service_client.container_client(container) + info!("Connecting to Azure emulator"); + ClientBuilder::emulator() + .blob_service_client() + .container_client(container) } else { let credentials = create_default_credential().expect("default Azure credentials working"); @@ -85,11 +84,13 @@ impl ABSBlobConfig { metrics, client, cfg, + prefix, }) } /// Returns a new [ABSBlobConfig] for use in unit tests. pub async fn new_for_test() -> Result, Error> { + // WIP: do we need this container name to be passed in? let container_name = match std::env::var(Self::EXTERNAL_TESTS_ABS_CONTAINER) { Ok(container) => container, Err(_) => { @@ -100,13 +101,13 @@ impl ABSBlobConfig { } }; - // let prefix = Uuid::new_v4().to_string(); + let prefix = Uuid::new_v4().to_string(); let metrics = S3BlobMetrics::new(&MetricsRegistry::new()); let config = ABSBlobConfig::new( EMULATOR_ACCOUNT.to_string(), container_name, - "".to_string(), + prefix, metrics, Arc::new(ConfigSet::default()), )?; @@ -136,16 +137,25 @@ impl ABSBlob { /// Opens the given location for non-exclusive read-write access. pub async fn open(config: ABSBlobConfig) -> Result { let container_name = config.client.container_name().to_string(); + + if config.client.service_client().account() == EMULATOR_ACCOUNT { + // try to create the container if we're running in the emulator. + // it is surprisingly difficult to create the container out-of-band + // of an official client. + let _ = config.client.create().await; + } + let ret = ABSBlob { metrics: config.metrics, client: config.client, container_name, - prefix: "".to_string(), + prefix: config.prefix, cfg: config.cfg, }; + // WIP: do we have a healthcheck similar to S3? // Test connection before returning success - let _ = ret.get("HEALTH_CHECK").await?; + // let _ = ret.get("HEALTH_CHECK").await?; Ok(ret) } @@ -165,8 +175,21 @@ impl Blob for ABSBlob { let mut stream = blob.get().into_stream(); while let Some(value) = stream.next().await { - let response = - value.map_err(|e| ExternalError::from(anyhow!("Azure blob get error: {:?}", e)))?; + let response = match value { + Ok(v) => v, + Err(e) => { + if let Some(e) = e.as_http_error() { + if e.status() == StatusCode::NotFound { + return Ok(None); + } + } + + return Err(ExternalError::from(anyhow!( + "Azure blob get error: {:?}", + e + ))); + } + }; let content_length = response.blob.properties.content_length; let mut buffer = self @@ -200,7 +223,7 @@ impl Blob for ABSBlob { let mut stream = self .client .list_blobs() - .prefix(blob_key_prefix) + .prefix(blob_key_prefix.clone()) .into_stream(); while let Some(response) = stream.next().await { @@ -250,9 +273,15 @@ impl Blob for ABSBlob { .map_err(|e| ExternalError::from(anyhow!("Azure blob delete error: {}", e)))?; Ok(Some(size)) } - // WIP: figure out what this is if empty - // Err(e) if e.status_code() == 404 => Ok(None), - Err(e) => Err(ExternalError::from(anyhow!("Azure blob error: {}", e))), + Err(e) => { + if let Some(e) = e.as_http_error() { + if e.status() == StatusCode::NotFound { + return Ok(None); + } + } + + Err(ExternalError::from(anyhow!("Azure blob error: {}", e))) + } } } @@ -262,12 +291,18 @@ impl Blob for ABSBlob { match blob.get_properties().await { Ok(_) => Ok(()), - // WIP - // Err(e) if e.status_code() == 404 => Err(Determinate::new(anyhow!( - // "unable to restore {key} in Azure Blob Storage: blob does not exist" - // )) - // .into()), - Err(e) => Err(ExternalError::from(anyhow!("Azure blob error: {}", e))), + Err(e) => { + if let Some(e) = e.as_http_error() { + if e.status() == StatusCode::NotFound { + return Err(Determinate::new(anyhow!( + "unable to restore {key} in Azure Blob Storage: blob does not exist" + )) + .into()); + } + } + + Err(ExternalError::from(anyhow!("Azure blob error: {}", e))) + } } } } @@ -298,15 +333,8 @@ mod tests { let config = ABSBlobConfig { metrics: config.metrics.clone(), client: config.client.clone(), - container_name: config.container_name.clone(), - prefix: format!("{}/abs_blob_impl_test/{}", config.prefix, path), - cfg: Arc::new( - ConfigSet::default() - .add(&ENABLE_ABS_LGALLOC_CC_SIZES) - .add(&ENABLE_ABS_LGALLOC_NONCC_SIZES) - .add(&ENABLE_ONE_ALLOC_PER_REQUEST), - ), - is_cc_active: true, + cfg: Arc::new(ConfigSet::default()), + prefix: config.prefix.clone(), }; ABSBlob::open(config).await } From 678f521e2ce574745cff4ac89f4d0d45cd7d2ca6 Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 10:26:29 -0500 Subject: [PATCH 4/8] get SAS tokens working --- src/persist/src/abs.rs | 42 ++++++++++++++++++++++++++++++++++++------ src/persist/src/cfg.rs | 25 ++++++++++++++----------- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/persist/src/abs.rs b/src/persist/src/abs.rs index e6da23d78a42f..8f9ea62ebd709 100644 --- a/src/persist/src/abs.rs +++ b/src/persist/src/abs.rs @@ -19,7 +19,9 @@ use std::time::{Duration, Instant}; use anyhow::anyhow; use async_trait::async_trait; use azure_core::StatusCode; -use azure_identity::{create_default_credential, DefaultAzureCredential}; +use azure_identity::{ + create_default_credential, DefaultAzureCredential, DefaultAzureCredentialBuilder, +}; use azure_storage::{prelude::*, EMULATOR_ACCOUNT}; use azure_storage_blobs::prelude::*; use bytes::Bytes; @@ -32,7 +34,8 @@ use mz_ore::lgbytes::{LgBytes, MetricsRegion}; use mz_ore::metrics::MetricsRegistry; use mz_ore::task::RuntimeExt; use tokio::runtime::Handle as AsyncHandle; -use tracing::{debug, debug_span, info, trace, trace_span, Instrument}; +use tracing::{debug, debug_span, info, trace, trace_span, warn, Instrument}; +use url::Url; use uuid::Uuid; use crate::cfg::BlobKnobs; @@ -62,6 +65,7 @@ impl ABSBlobConfig { container: String, prefix: String, metrics: S3BlobMetrics, + url: Url, cfg: Arc, ) -> Result { // let is_cc_active = knobs.is_cc_active(); @@ -74,8 +78,29 @@ impl ABSBlobConfig { .blob_service_client() .container_client(container) } else { - let credentials = - create_default_credential().expect("default Azure credentials working"); + // WIP: check query pairs if our query string is for a SAS token + let sas_credentials = match url.query() { + Some(query) => Some(StorageCredentials::sas_token(query)), + None => None, + }; + + let credentials = match sas_credentials { + Some(Ok(credentials)) => credentials, + Some(Err(err)) => { + warn!("Failed to parse SAS token: {err}"); + // Fall back to default credentials + StorageCredentials::token_credential( + create_default_credential().expect("Azure default credentials"), + ) + } + None => { + // Fall back to default credentials + StorageCredentials::token_credential( + create_default_credential().expect("Azure default credentials"), + ) + } + }; + let service_client = BlobServiceClient::new(account, credentials); service_client.container_client(container) }; @@ -106,9 +131,14 @@ impl ABSBlobConfig { let config = ABSBlobConfig::new( EMULATOR_ACCOUNT.to_string(), - container_name, + container_name.clone(), prefix, metrics, + Url::parse(&format!( + "http://devaccount1.blob.core.windows.net/{}", + container_name + )) + .expect("valid url"), Arc::new(ConfigSet::default()), )?; @@ -118,7 +148,7 @@ impl ABSBlobConfig { /// Returns a clone of Self with a new v4 uuid prefix. pub fn clone_with_new_uuid_prefix(&self) -> Self { let mut ret = self.clone(); - // ret.prefix = Uuid::new_v4().to_string(); + ret.prefix = Uuid::new_v4().to_string(); ret } } diff --git a/src/persist/src/cfg.rs b/src/persist/src/cfg.rs index 687330d333ede..3579f43f4f0e6 100644 --- a/src/persist/src/cfg.rs +++ b/src/persist/src/cfg.rs @@ -164,6 +164,7 @@ impl BlobConfig { container.to_string(), "".to_string(), metrics, + url.clone().into_redacted(), cfg, )?)) } else { @@ -179,17 +180,19 @@ impl BlobConfig { )), }?; - if !query_params.is_empty() { - return Err(ExternalError::from(anyhow!( - "unknown blob location params {}: {}", - query_params - .keys() - .map(|x| x.as_ref()) - .collect::>() - .join(" "), - url.as_str(), - ))); - } + // WIP: is it OK to remove this? there are a ton of + // query params for Azure SAS tokens to work + // if !query_params.is_empty() { + // return Err(ExternalError::from(anyhow!( + // "unknown blob location params {}: {}", + // query_params + // .keys() + // .map(|x| x.as_ref()) + // .collect::>() + // .join(" "), + // url.as_str(), + // ))); + // } Ok(config) } From 60df7756e8982d7b72220385734689d4c7be7be4 Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 10:30:07 -0500 Subject: [PATCH 5/8] skip test in CI for now --- src/persist/src/abs.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/persist/src/abs.rs b/src/persist/src/abs.rs index 8f9ea62ebd709..9f31845ea12a8 100644 --- a/src/persist/src/abs.rs +++ b/src/persist/src/abs.rs @@ -119,9 +119,10 @@ impl ABSBlobConfig { let container_name = match std::env::var(Self::EXTERNAL_TESTS_ABS_CONTAINER) { Ok(container) => container, Err(_) => { - if mz_ore::env::is_var_truthy("CI") { - panic!("CI is supposed to run this test but something has gone wrong!"); - } + // WIP: figure out CI situation + // if mz_ore::env::is_var_truthy("CI") { + // panic!("CI is supposed to run this test but something has gone wrong!"); + // } return Ok(None); } }; From ac0c926370f0bfd7225b8073790fefc00102fd85 Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 10:49:19 -0500 Subject: [PATCH 6/8] undo lock --- Cargo.lock | 758 ++++++++--------------------------------------------- 1 file changed, 111 insertions(+), 647 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7557fdc91e002..0872343da2a38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - [[package]] name = "addr2line" version = "0.17.0" @@ -31,7 +25,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -360,30 +354,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" dependencies = [ - "event-listener 5.3.1", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" -dependencies = [ - "concurrent-queue", + "event-listener", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -404,73 +375,6 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-io" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.5.0", - "parking", - "polling", - "rustix 0.38.34", - "slab", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener 5.3.1", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel 2.3.1", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.3.1", - "futures-lite 2.5.0", - "rustix 0.38.34", - "tracing", -] - -[[package]] -name = "async-signal" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix 0.38.34", - "signal-hook-registry", - "slab", - "windows-sys 0.59.0", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -493,12 +397,6 @@ dependencies = [ "syn 2.0.63", ] -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.68" @@ -586,7 +484,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.0.1", + "fastrand", "hex", "http 0.2.9", "hyper 0.14.27", @@ -625,7 +523,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.0.1", + "fastrand", "http 0.2.9", "http-body 0.4.5", "percent-encoding", @@ -655,7 +553,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "fastrand 2.0.1", + "fastrand", "hex", "hmac", "http 0.2.9", @@ -685,7 +583,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.0.1", + "fastrand", "http 0.2.9", "once_cell", "regex-lite", @@ -877,7 +775,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand 2.0.1", + "fastrand", "h2 0.3.26", "http 0.2.9", "http-body 0.4.5", @@ -1038,122 +936,15 @@ dependencies = [ "tracing", ] -[[package]] -name = "azure_core" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" -dependencies = [ - "async-trait", - "base64 0.22.0", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.10", - "hmac", - "http-types", - "once_cell", - "paste", - "pin-project", - "quick-xml", - "rand 0.8.5", - "reqwest 0.12.4", - "rustc_version", - "serde", - "serde_json", - "sha2", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_identity" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" -dependencies = [ - "async-lock", - "async-process", - "async-trait", - "azure_core", - "futures", - "oauth2", - "pin-project", - "serde", - "time", - "tracing", - "tz-rs", - "url", - "uuid", -] - -[[package]] -name = "azure_storage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" -dependencies = [ - "RustyXML", - "async-lock", - "async-trait", - "azure_core", - "bytes", - "serde", - "serde_derive", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_storage_blobs" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" -dependencies = [ - "RustyXML", - "azure_core", - "azure_storage", - "azure_svc_blobstorage", - "bytes", - "futures", - "serde", - "serde_derive", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" -dependencies = [ - "azure_core", - "bytes", - "futures", - "log", - "once_cell", - "serde", - "serde_json", - "time", -] - [[package]] name = "backoff" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "getrandom 0.2.10", + "getrandom", "instant", - "rand 0.8.5", + "rand", ] [[package]] @@ -1306,19 +1097,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" -dependencies = [ - "async-channel 2.3.1", - "async-task", - "futures-io", - "futures-lite 2.5.0", - "piper", -] - [[package]] name = "brotli" version = "3.5.0" @@ -1568,10 +1346,8 @@ checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1821,17 +1597,11 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.10", + "getrandom", "once_cell", "tiny-keccak", ] -[[package]] -name = "const_fn" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" - [[package]] name = "convert_case" version = "0.6.0" @@ -2352,7 +2122,7 @@ dependencies = [ "futures-util", "libc", "octseq", - "rand 0.8.5", + "rand", "smallvec", "time", "tokio", @@ -2583,12 +2353,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "event-listener" version = "5.3.1" @@ -2606,7 +2370,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.1", + "event-listener", "pin-project-lite", ] @@ -2621,7 +2385,7 @@ dependencies = [ "hyper-tls 0.5.0", "log", "pin-project", - "rand 0.8.5", + "rand", "tokio", ] @@ -2642,7 +2406,7 @@ checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" dependencies = [ "log", "once_cell", - "rand 0.8.5", + "rand", ] [[package]] @@ -2661,15 +2425,6 @@ dependencies = [ "regex", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.0.1" @@ -2840,34 +2595,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" -dependencies = [ - "fastrand 2.0.1", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.30" @@ -2947,17 +2674,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.10" @@ -2967,7 +2683,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -3011,7 +2727,7 @@ dependencies = [ "nonzero_ext", "parking_lot", "quanta", - "rand 0.8.5", + "rand", "smallvec", ] @@ -3186,12 +2902,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" -[[package]] -name = "hermit-abi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" - [[package]] name = "hex" version = "0.4.3" @@ -3298,26 +3008,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "base64 0.13.1", - "futures-lite 1.13.0", - "infer", - "pin-project-lite", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url", -] - [[package]] name = "httparse" version = "1.8.0" @@ -3538,7 +3228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" dependencies = [ "bitmaps", - "rand_core 0.6.2", + "rand_core", "rand_xoshiro", "sized-chunks", "typenum", @@ -3597,12 +3287,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "insta" version = "1.33.0" @@ -3784,7 +3468,7 @@ dependencies = [ "k8s-openapi", "kube", "kube-runtime", - "rand 0.8.5", + "rand", "serde", "tracing", ] @@ -3849,7 +3533,7 @@ dependencies = [ "kube-core", "openssl", "pem", - "rand 0.8.5", + "rand", "secrecy", "serde", "serde_json", @@ -4346,7 +4030,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] @@ -4405,7 +4089,7 @@ dependencies = [ "pem", "percent-encoding", "pin-project", - "rand 0.8.5", + "rand", "serde", "serde_json", "socket2 0.5.7", @@ -4439,7 +4123,7 @@ dependencies = [ "lazy_static", "num-bigint", "num-traits", - "rand 0.8.5", + "rand", "regex", "saturating", "serde", @@ -4472,7 +4156,7 @@ dependencies = [ "mz-ore", "open", "openssl-probe", - "reqwest 0.12.4", + "reqwest", "rpassword", "security-framework", "semver", @@ -4562,11 +4246,11 @@ dependencies = [ "proptest-derive", "prost", "qcell", - "rand 0.8.5", - "rand_chacha 0.3.0", + "rand", + "rand_chacha", "rdkafka", "regex", - "reqwest 0.11.24", + "reqwest", "semver", "serde", "serde_json", @@ -4662,7 +4346,7 @@ dependencies = [ "flate2", "itertools 0.10.5", "mz-ore", - "rand 0.8.5", + "rand", "regex", "serde", "serde_json", @@ -4751,7 +4435,7 @@ dependencies = [ "postgres", "prometheus", "proxy-header", - "reqwest 0.11.24", + "reqwest", "semver", "tempfile", "tokio", @@ -4839,7 +4523,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand 0.8.5", + "rand", "semver", "serde", "serde_json", @@ -4903,7 +4587,7 @@ dependencies = [ "proptest", "proptest-derive", "prost-build", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "tokio", @@ -4920,7 +4604,7 @@ dependencies = [ "chrono", "mz-frontegg-auth", "mz-frontegg-client", - "reqwest 0.11.24", + "reqwest", "serde", "thiserror", "tokio", @@ -4940,7 +4624,7 @@ dependencies = [ "kube", "mz-ore", "mz-repr", - "rand 0.8.5", + "rand", "schemars", "semver", "serde", @@ -5385,11 +5069,11 @@ dependencies = [ "predicates", "prometheus", "proptest", - "rand 0.8.5", + "rand", "rdkafka", "rdkafka-sys", "regex", - "reqwest 0.11.24", + "reqwest", "rlimit", "semver", "sentry", @@ -5466,7 +5150,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand 0.8.5", + "rand", "regex", "regex-syntax 0.8.3", "seahash", @@ -5532,7 +5216,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "sha2", @@ -5565,7 +5249,7 @@ dependencies = [ "mz-ore", "mz-repr", "prometheus", - "reqwest 0.11.24", + "reqwest", "reqwest-middleware", "reqwest-retry", "serde", @@ -5584,7 +5268,7 @@ dependencies = [ "jsonwebtoken", "mz-frontegg-auth", "mz-ore", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "thiserror", @@ -5609,7 +5293,7 @@ dependencies = [ "mz-frontegg-auth", "mz-ore", "openssl", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "tokio", @@ -5698,7 +5382,7 @@ dependencies = [ "num_cpus", "prost", "prost-build", - "rand 0.8.5", + "rand", "rdkafka", "serde", "serde_json", @@ -5765,7 +5449,7 @@ dependencies = [ name = "mz-metabase" version = "0.0.0" dependencies = [ - "reqwest 0.11.24", + "reqwest", "serde", "workspace-hack", ] @@ -5839,7 +5523,7 @@ dependencies = [ "flate2", "hex", "hex-literal", - "reqwest 0.11.24", + "reqwest", "sha2", "tar", "walkdir", @@ -5971,8 +5655,8 @@ dependencies = [ "mz-sql", "mz-tls-util", "prometheus", - "rand 0.8.5", - "reqwest 0.11.24", + "rand", + "reqwest", "serde", "serde_json", "sha2", @@ -6022,7 +5706,7 @@ dependencies = [ "pin-project", "prometheus", "proptest", - "rand 0.8.5", + "rand", "scopeguard", "sentry", "sentry-tracing", @@ -6069,10 +5753,6 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "aws-types", - "azure_core", - "azure_identity", - "azure_storage", - "azure_storage_blobs", "base64 0.13.1", "bytes", "deadpool-postgres", @@ -6097,7 +5777,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand 0.8.5", + "rand", "serde", "serde_json", "sha2", @@ -6502,7 +6182,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand 0.8.5", + "rand", "regex", "ryu", "serde", @@ -6755,7 +6435,7 @@ dependencies = [ "protobuf-native", "rdkafka", "regex", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "static_assertions", @@ -6854,7 +6534,7 @@ dependencies = [ "mz-tracing", "postgres-protocol", "regex", - "reqwest 0.11.24", + "reqwest", "serde_json", "shell-words", "tempfile", @@ -6880,7 +6560,7 @@ dependencies = [ "openssh", "openssh-mux-client", "openssl", - "rand 0.8.5", + "rand", "scopeguard", "serde", "serde_json", @@ -6962,7 +6642,7 @@ dependencies = [ "postgres-replication", "prometheus", "prost", - "rand 0.8.5", + "rand", "rdkafka", "regex", "rocksdb", @@ -7165,7 +6845,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand 0.8.5", + "rand", "rdkafka", "regex", "serde", @@ -7189,7 +6869,7 @@ dependencies = [ "chrono", "mz-kafka-util", "mz-ore", - "rand 0.8.5", + "rand", "rdkafka", "tokio", "tokio-postgres", @@ -7250,10 +6930,10 @@ dependencies = [ "prost-build", "prost-reflect", "prost-types", - "rand 0.8.5", + "rand", "rdkafka", "regex", - "reqwest 0.11.24", + "reqwest", "semver", "serde", "serde_json", @@ -7312,7 +6992,7 @@ dependencies = [ "mz-postgres-client", "mz-repr", "postgres-protocol", - "rand 0.8.5", + "rand", "serde", "tokio", "tracing", @@ -7401,7 +7081,7 @@ dependencies = [ "prometheus", "prost", "prost-build", - "rand 0.8.5", + "rand", "serde", "timely", "tokio", @@ -7616,15 +7296,6 @@ dependencies = [ "syn 1.0.107", ] -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "numa_maps" version = "0.1.0" @@ -7637,25 +7308,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" -[[package]] -name = "oauth2" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" -dependencies = [ - "base64 0.13.1", - "chrono", - "getrandom 0.2.10", - "http 0.2.9", - "rand 0.8.5", - "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "thiserror", - "url", -] - [[package]] name = "object" version = "0.29.0" @@ -7841,7 +7493,7 @@ dependencies = [ "once_cell", "opentelemetry", "percent-encoding", - "rand 0.8.5", + "rand", "serde_json", "thiserror", "tokio", @@ -7870,7 +7522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", - "rand 0.8.5", + "rand", "serde", ] @@ -8150,7 +7802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b450720b6f75cfbfabc195814bd3765f337a4f9a83186f8537297cac12f6705" dependencies = [ "phf_shared", - "rand 0.8.5", + "rand", ] [[package]] @@ -8195,17 +7847,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand 2.0.1", - "futures-io", -] - [[package]] name = "pkg-config" version = "0.3.20" @@ -8240,21 +7881,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "3.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.4.0", - "pin-project-lite", - "rustix 0.38.34", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "portable-atomic" version = "0.3.20" @@ -8297,7 +7923,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.7" -source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" +source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" dependencies = [ "base64 0.22.0", "byteorder", @@ -8306,7 +7932,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.8.5", + "rand", "sha2", "stringprep", ] @@ -8329,7 +7955,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.7" -source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" +source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" dependencies = [ "bytes", "chrono", @@ -8515,8 +8141,8 @@ dependencies = [ "lazy_static", "num-traits", "quick-error", - "rand 0.8.5", - "rand_chacha 0.3.0", + "rand", + "rand_chacha", "rand_xorshift", "regex-syntax 0.6.28", ] @@ -8702,7 +8328,7 @@ dependencies = [ "mach2", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -8720,7 +8346,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", - "serde", ] [[package]] @@ -8729,7 +8354,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "rand 0.8.5", + "rand", ] [[package]] @@ -8747,19 +8372,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -8767,21 +8379,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.0", - "rand_core 0.6.2", + "rand_chacha", + "rand_core", "serde", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.0" @@ -8789,16 +8391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.2", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -8807,26 +8400,17 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom 0.2.10", + "getrandom", "serde", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_xorshift" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core 0.6.2", + "rand_core", ] [[package]] @@ -8835,7 +8419,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core 0.6.2", + "rand_core", ] [[package]] @@ -8927,7 +8511,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.10", + "getrandom", "redox_syscall 0.2.10", ] @@ -9007,7 +8591,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -9020,52 +8604,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.50.0", -] - -[[package]] -name = "reqwest" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" -dependencies = [ - "base64 0.22.0", - "bytes", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.4.5", - "http 1.1.0", - "http-body 1.0.0", - "http-body-util", - "hyper 1.4.1", - "hyper-tls 0.6.0", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 2.2.0", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration", - "tokio", - "tokio-native-tls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg 0.52.0", + "winreg", ] [[package]] @@ -9076,7 +8615,7 @@ dependencies = [ "anyhow", "async-trait", "http 0.2.9", - "reqwest 0.11.24", + "reqwest", "serde", "task-local-extensions", "thiserror", @@ -9093,7 +8632,7 @@ dependencies = [ "futures", "http 0.2.9", "hyper 0.14.27", - "reqwest 0.11.24", + "reqwest", "reqwest-middleware", "retry-policies", "task-local-extensions", @@ -9115,7 +8654,7 @@ checksum = "e09bbcb5003282bcb688f0bae741b278e9c7e8f378f561522c9806c58e075d9b" dependencies = [ "anyhow", "chrono", - "rand 0.8.5", + "rand", ] [[package]] @@ -9125,7 +8664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", - "getrandom 0.2.10", + "getrandom", "libc", "spin", "untrusted", @@ -9239,21 +8778,6 @@ dependencies = [ "base64 0.21.5", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" - [[package]] name = "rustversion" version = "1.0.9" @@ -9399,7 +8923,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24fc91c898e0487ff3e471d0849bbaf7d38a00ff5e3531009d386b0bab9b6b12" dependencies = [ "async-trait", - "reqwest 0.11.24", + "reqwest", "serde", "serde_json", "thiserror", @@ -9433,7 +8957,7 @@ checksum = "17ad137b9df78294b98cab1a650bef237cc6c950e82e5ce164655e674d07c5cc" dependencies = [ "httpdate", "native-tls", - "reqwest 0.11.24", + "reqwest", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -9476,7 +9000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4df9b9d8de2658a1ecd4e45f7b06c80c5dd97b891bfbc7c501186189b7e9bbdf" dependencies = [ "once_cell", - "rand 0.8.5", + "rand", "sentry-types", "serde", "serde_json", @@ -9521,7 +9045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc95faa4078768a6bf8df45e2b894bbf372b3dbbfb364e9429c1c58ab7545c6" dependencies = [ "debugid", - "getrandom 0.2.10", + "getrandom", "hex", "serde", "serde_json", @@ -9628,17 +9152,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" -dependencies = [ - "percent-encoding", - "serde", - "thiserror", -] - [[package]] name = "serde_repr" version = "0.1.13" @@ -9857,12 +9370,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" @@ -9913,7 +9423,7 @@ checksum = "f02d3730e8785e797a4552137d1acc0d7f7146dad3b5fe65ed83637711dfc6c5" dependencies = [ "base64ct", "pem-rfc7468", - "rand_core 0.6.2", + "rand_core", "sec1", "sha2", "signature", @@ -10189,7 +9699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", - "fastrand 2.0.1", + "fastrand", "once_cell", "rustix 0.38.34", "windows-sys 0.59.0", @@ -10326,9 +9836,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", - "js-sys", - "libc", - "num_threads", "quickcheck", "serde", "time-core", @@ -10535,7 +10042,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.11" -source = "git+https://github.com/MaterializeInc/rust-postgres#7d2f14e35e97973995c39662cde618036a0f9484" +source = "git+https://github.com/MaterializeInc/rust-postgres#548f00c4cbcf7bd714c29cfd2fea7685040c9987" dependencies = [ "async-trait", "byteorder", @@ -10550,7 +10057,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.8.5", + "rand", "serde", "socket2 0.5.7", "tokio", @@ -10723,7 +10230,7 @@ dependencies = [ "indexmap 1.9.1", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand", "slab", "tokio", "tokio-util", @@ -10800,10 +10307,11 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -10812,13 +10320,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 1.0.107", ] [[package]] @@ -10836,9 +10344,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -10943,7 +10451,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand 0.8.5", + "rand", "sha1", "thiserror", "url", @@ -10962,7 +10470,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand 0.8.5", + "rand", "sha1", "thiserror", "utf-8", @@ -10975,7 +10483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand", "static_assertions", ] @@ -11002,15 +10510,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" -[[package]] -name = "tz-rs" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" -dependencies = [ - "const_fn", -] - [[package]] name = "ucd-trie" version = "0.1.6" @@ -11151,7 +10650,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ - "getrandom 0.2.10", + "getrandom", "serde", "sha1_smol", ] @@ -11209,12 +10708,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" - [[package]] name = "walkdir" version = "2.3.2" @@ -11236,12 +10729,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -11281,9 +10768,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" dependencies = [ "cfg-if", "js-sys", @@ -11320,24 +10807,11 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" -[[package]] -name = "wasm-streams" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -11639,16 +11113,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "workspace-hack" version = "0.0.0" @@ -11697,7 +11161,7 @@ dependencies = [ "futures-sink", "futures-task", "futures-util", - "getrandom 0.2.10", + "getrandom", "globset", "hashbrown 0.14.5", "hyper 0.14.27", @@ -11744,13 +11208,13 @@ dependencies = [ "prost-reflect", "prost-types", "quote", - "rand 0.8.5", - "rand_chacha 0.3.0", + "rand", + "rand_chacha", "rdkafka-sys", "regex", "regex-automata 0.4.7", "regex-syntax 0.8.3", - "reqwest 0.11.24", + "reqwest", "ring", "schemars", "scopeguard", From 7828a5b7b32d4596bda6311cd2bbe1288a5e2159 Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 13:52:43 -0500 Subject: [PATCH 7/8] reqwest fix --- src/mz/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mz/Cargo.toml b/src/mz/Cargo.toml index ad5f98128b52c..c3cfe1324b5a1 100644 --- a/src/mz/Cargo.toml +++ b/src/mz/Cargo.toml @@ -25,7 +25,7 @@ mz-ore = { path = "../ore", features = ["async", "cli", "test"] } open = "3.2.0" openssl-probe = "0.1.2" hyper = "1.4.1" -reqwest = { version = "0.12", features = ["blocking", "json"] } +reqwest = { version = "0.12", features = ["blocking", "json", "default-tls", "charset", "http2"], default-features = false } rpassword = "7.2.0" semver = "1.0.16" serde = { version = "1.0.152", features = ["derive"] } From 2a044eaf757e585592644c739650f81f65864e3f Mon Sep 17 00:00:00 2001 From: Paul Hemberger Date: Fri, 13 Dec 2024 14:24:13 -0500 Subject: [PATCH 8/8] cursed random compile error --- src/storage/src/source/postgres/replication.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storage/src/source/postgres/replication.rs b/src/storage/src/source/postgres/replication.rs index 6f3dccbce62d3..9d7d5a9fc9108 100644 --- a/src/storage/src/source/postgres/replication.rs +++ b/src/storage/src/source/postgres/replication.rs @@ -998,7 +998,6 @@ where TupleData::Text(bytes) => Some(bytes.clone()), TupleData::Null => None, TupleData::UnchangedToast => return Err(DefiniteError::MissingToast), - TupleData::Binary(_) => return Err(DefiniteError::MissingToast), }; row.push(datum); }