diff --git a/Cargo.lock b/Cargo.lock index 662052bd2ee7..f2a7ddfde6f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -497,6 +497,383 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5b3469636cdf8543cceab175efca534471f36eee12fb8374aba00eb5e7e7f8a" +[[package]] +name = "aws-config" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.0.1", + "hex", + "http", + "hyper", + "ring 0.17.5", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-http" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-runtime" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "fastrand 2.0.1", + "http", + "percent-encoding", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a2ccbd49e784c36d0f0596da681275496f679e249059cae3fa92dd3c2c72e1" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.4", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "p256", + "percent-encoding", + "regex", + "ring 0.17.5", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d293ef76a982c573f7384e30c28f9a2472f8dd5f4ce5abcceb0e909a15098e8e" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d2c1844aee465dc023924dbe19d730b116eaf3587d7c2a9b4a41f9c4e980ee" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.0.1", + "http", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.9", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", + "rustc_version 0.4.0", + "tracing", +] + [[package]] name = "axum" version = "0.6.20" @@ -573,6 +950,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -585,6 +968,16 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -880,6 +1273,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -1313,6 +1716,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" +[[package]] +name = "crc32c" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +dependencies = [ + "rustc_version 0.4.0", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -1433,6 +1845,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f85c3514d2a6e64160359b45a3918c3b4178bcbf4ae5d03ab2d02e521c479a" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1704,6 +2138,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.8" @@ -1890,6 +2334,18 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "edge_test_file_write_on_full_disk" version = "0.0.0" @@ -1906,6 +2362,26 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -2072,6 +2548,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "finl_unicode" version = "1.2.0" @@ -2474,6 +2960,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -2758,7 +3255,9 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "log", + "rustls 0.21.9", + "rustls-native-certs", "tokio", "tokio-rustls", ] @@ -3684,7 +4183,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-pemfile", "serde", "serde_bytes", @@ -4261,6 +4760,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "opendal-benchmark-vs-s3" +version = "0.0.0" +dependencies = [ + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "criterion", + "dotenvy", + "opendal", + "rand 0.8.5", + "tokio", + "uuid", +] + [[package]] name = "opendal-c" version = "0.42.0" @@ -4570,12 +5084,29 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "panic-message" version = "0.3.0" @@ -4792,9 +5323,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -4803,8 +5344,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.8", + "spki 0.7.2", ] [[package]] @@ -5566,7 +6107,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-native-certs", "ryu", "sha1_smol", @@ -5728,7 +6269,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-native-certs", "rustls-pemfile", "serde", @@ -5765,6 +6306,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.16.20" @@ -5846,10 +6398,10 @@ dependencies = [ "num-iter", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.1.0", + "spki 0.7.2", "subtle", "zeroize", ] @@ -6011,12 +6563,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.5", "rustls-webpki", "sct 0.7.0", ] @@ -6044,12 +6596,12 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -6141,6 +6693,20 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -6368,6 +6934,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.1.0" @@ -6511,6 +7087,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -6518,7 +7104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.8", ] [[package]] @@ -6978,7 +7564,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.9", "tokio", ] @@ -7126,7 +7712,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.12.1", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-pemfile", "tokio", "tokio-rustls", @@ -7506,7 +8092,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.21.7", + "rustls 0.21.9", "rustls-webpki", "url", "webpki-roots 0.25.2", @@ -7581,6 +8167,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -8047,6 +8639,12 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xmltree" version = "0.10.3" diff --git a/Cargo.toml b/Cargo.toml index 6ec1b15ddb39..9db84f39842b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ "core", "core/fuzz", "core/edge/*", - "core/benches/vs_fs", + "core/benches/vs_*", "bindings/c", "bindings/nodejs", diff --git a/core/benches/vs_s3/Cargo.toml b/core/benches/vs_s3/Cargo.toml new file mode 100644 index 000000000000..db6a71fe20af --- /dev/null +++ b/core/benches/vs_s3/Cargo.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-benchmark-vs-s3" +description = "OpenDAL Benchmark vs s3" +version = "0.0.0" +publish = false + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +opendal = { path = "../..", features = ["tests"] } +tokio = { version = "1", features = ["full"] } +uuid = { version = "1", features = ["v4"] } +criterion = { version = "0.4", features = ["async", "async_tokio"] } +rand = "0.8" +aws-sdk-s3 = "0.38" +aws-config = { version = "0.101.0", features = ["behavior-version-latest"] } +dotenvy = "0.15" +aws-credential-types = { version = "0.101.0", features = ["hardcoded-credentials"] } diff --git a/core/benches/vs_s3/README.md b/core/benches/vs_s3/README.md new file mode 100644 index 000000000000..2a44431a5fdf --- /dev/null +++ b/core/benches/vs_s3/README.md @@ -0,0 +1,46 @@ +# OpenDAL Benchmark VS AWS SDK S3 + +This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`. + +## Goal + +We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `10%` range of `aws_sdk_s3`. + +## Usage + +For test: `cargo run` + +```shell +Testing read/aws_s3_sdk_collect +Success +Testing read/aws_s3_sdk_into_async_read +Success +Testing read/aws_s3_sdk_into_async_read_with_size_known +Success +Testing read/opendal_s3 +Success +Testing read/opendal_s3_with_range +Success +``` + +For bench: `cargo run --release -- --bench` + +```shell +read/aws_s3_sdk_collect time: [47.264 ms 47.378 ms 47.504 ms] + thrpt: [336.82 MiB/s 337.71 MiB/s 338.53 MiB/s] + +read/aws_s3_sdk_into_async_read + time: [9.8422 ms 11.607 ms 13.703 ms] + thrpt: [1.1403 GiB/s 1.3462 GiB/s 1.5876 GiB/s] + +read/aws_s3_sdk_into_async_read_with_size_known + time: [7.9572 ms 8.1055 ms 8.2552 ms] + thrpt: [1.8927 GiB/s 1.9277 GiB/s 1.9636 GiB/s] + +read/opendal_s3 time: [8.9068 ms 9.2614 ms 9.6912 ms] + thrpt: [1.6123 GiB/s 1.6871 GiB/s 1.7543 GiB/s] + +read/opendal_s3_with_range + time: [8.5459 ms 8.7592 ms 8.9739 ms] + thrpt: [1.7412 GiB/s 1.7838 GiB/s 1.8284 GiB/s] +``` diff --git a/core/benches/vs_s3/src/main.rs b/core/benches/vs_s3/src/main.rs new file mode 100644 index 000000000000..29e7bf1389cd --- /dev/null +++ b/core/benches/vs_s3/src/main.rs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use aws_config::{BehaviorVersion, Region}; +use aws_credential_types::Credentials; +use criterion::Criterion; +use opendal::raw::tests::TEST_RUNTIME; +use opendal::services; +use opendal::Operator; +use rand::prelude::*; +use std::env; +use tokio::io::AsyncReadExt; + +fn main() { + let _ = dotenvy::dotenv(); + + let endpoint = env::var("OPENDAL_S3_ENDPOINT").unwrap(); + let access_key = env::var("OPENDAL_S3_ACCESS_KEY_ID").unwrap(); + let secret_key = env::var("OPENDAL_S3_SECRET_ACCESS_KEY").unwrap(); + let bucket = env::var("OPENDAL_S3_BUCKET").unwrap(); + let region = env::var("OPENDAL_S3_REGION").unwrap(); + + // Init OpenDAL Operator. + let mut cfg = services::S3::default(); + cfg.endpoint(&endpoint); + cfg.access_key_id(&access_key); + cfg.secret_access_key(&secret_key); + cfg.bucket(&bucket); + cfg.region(®ion); + let op = Operator::new(cfg).unwrap().finish(); + + // Init AWS S3 SDK. + let mut config_loader = aws_config::defaults(BehaviorVersion::latest()); + config_loader = config_loader.endpoint_url(&endpoint); + config_loader = config_loader.region(Region::new(region.to_string())); + config_loader = + config_loader.credentials_provider(Credentials::from_keys(&access_key, &secret_key, None)); + let config = TEST_RUNTIME.block_on(config_loader.load()); + let s3_client = aws_sdk_s3::Client::new(&config); + + let mut c = Criterion::default().configure_from_args(); + bench_read(&mut c, op, s3_client, &bucket); + + c.final_summary(); +} + +fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bucket: &str) { + let mut group = c.benchmark_group("read"); + group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024)); + + let path = TEST_RUNTIME.block_on(prepare(op.clone())); + + group.bench_function("aws_s3_sdk_collect", |b| { + b.to_async(&*TEST_RUNTIME).iter(|| async { + let _: Vec = s3_client + .get_object() + .bucket(bucket) + .key(&path) + .send() + .await + .unwrap() + .body + .collect() + .await + .unwrap() + .to_vec(); + }); + }); + group.bench_function("aws_s3_sdk_into_async_read", |b| { + b.to_async(&*TEST_RUNTIME).iter(|| async { + let mut r = s3_client + .get_object() + .bucket(bucket) + .key(&path) + .send() + .await + .unwrap() + .body + .into_async_read(); + let mut bs = Vec::new(); + let _ = r.read_to_end(&mut bs).await.unwrap(); + }); + }); + group.bench_function("aws_s3_sdk_into_async_read_with_size_known", |b| { + b.to_async(&*TEST_RUNTIME).iter(|| async { + let mut r = s3_client + .get_object() + .bucket(bucket) + .key(&path) + .send() + .await + .unwrap() + .body + .into_async_read(); + let mut bs = Vec::with_capacity(16 * 1024 * 1024); + let _ = r.read_to_end(&mut bs).await.unwrap(); + }); + }); + group.bench_function("opendal_s3", |b| { + b.to_async(&*TEST_RUNTIME).iter(|| async { + let _: Vec = op.read(&path).await.unwrap(); + }); + }); + group.bench_function("opendal_s3_with_range", |b| { + b.to_async(&*TEST_RUNTIME).iter(|| async { + let _: Vec = op + .read_with(&path) + .range(0..16 * 1024 * 1024) + .await + .unwrap(); + }); + }); + + group.finish() +} + +async fn prepare(op: Operator) -> String { + let mut rng = thread_rng(); + let mut content = vec![0; 16 * 1024 * 1024]; + rng.fill_bytes(&mut content); + + let name = uuid::Uuid::new_v4().to_string(); + op.write(&name, content).await.unwrap(); + + name +} diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index 4732be231be0..e96143dd32fb 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -835,6 +835,7 @@ mod tests { let op = new_test_operator(Capability { read: true, + stat: true, ..Default::default() }); let res = op.read("path").await; diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs index 8b7d5da5af09..fec20f9eec73 100644 --- a/core/src/raw/http_util/body.rs +++ b/core/src/raw/http_util/body.rs @@ -159,13 +159,19 @@ impl oio::Read for IncomingAsyncBody { return Poll::Ready(Ok(0)); } - // We must get a valid bytes from underlying stream - let mut bs = loop { - match ready!(self.poll_next(cx)) { - Some(Ok(bs)) if bs.is_empty() => continue, - Some(Ok(bs)) => break bs, - Some(Err(err)) => return Poll::Ready(Err(err)), - None => return Poll::Ready(Ok(0)), + // Avoid extra poll of next if we already have chunks. + let mut bs = if let Some(chunk) = self.chunk.take() { + chunk + } else { + loop { + match ready!(self.poll_next(cx)) { + // It's possible for underlying stream to return empty bytes, we should continue + // to fetch next one. + Some(Ok(bs)) if bs.is_empty() => continue, + Some(Ok(bs)) => break bs, + Some(Err(err)) => return Poll::Ready(Err(err)), + None => return Poll::Ready(Ok(0)), + } } }; diff --git a/core/src/raw/oio/read/api.rs b/core/src/raw/oio/read/api.rs index 03a362c6d9d7..cbb8e20d1ab1 100644 --- a/core/src/raw/oio/read/api.rs +++ b/core/src/raw/oio/read/api.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::cmp; use std::fmt::Display; use std::fmt::Formatter; use std::io; @@ -204,14 +203,7 @@ pub trait ReadExt: Read { /// Build a future for `read_to_end`. fn read_to_end<'a>(&'a mut self, buf: &'a mut Vec) -> ReadToEndFuture<'a, Self> { - let start = buf.len(); - ReadToEndFuture { - reader: self, - buf, - start, - length: start, - next: MIN_READ_TO_END_GROW_SIZE, - } + ReadToEndFuture { reader: self, buf } } } @@ -271,19 +263,11 @@ where } } -/// The MIN read to end grow size. -const MIN_READ_TO_END_GROW_SIZE: usize = 8 * 1024; -/// The MAX read to end grow size. -const MAX_READ_TO_END_GROW_SIZE: usize = 4 * 1024 * 1024; - /// Make this future `!Unpin` for compatibility with async trait methods. #[pin_project(!Unpin)] pub struct ReadToEndFuture<'a, R: Read + Unpin + ?Sized> { reader: &'a mut R, buf: &'a mut Vec, - start: usize, - length: usize, - next: usize, } impl Future for ReadToEndFuture<'_, R> @@ -294,43 +278,50 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.project(); + let start_len = this.buf.len(); + let start_cap = this.buf.capacity(); loop { - if this.buf.capacity() == *this.length { - this.buf.reserve(*this.next); - // # Safety - // - // We make sure that the length of buf is maintained correctly. - #[allow(clippy::uninit_vec)] - unsafe { - this.buf.set_len(this.buf.capacity()); - } + if this.buf.len() == this.buf.capacity() { + this.buf.reserve(32); // buf is full, need more space } - let buf = &mut this.buf[*this.length..]; - match ready!(this.reader.poll_read(cx, buf)) { + let spare = this.buf.spare_capacity_mut(); + let mut read_buf: ReadBuf = ReadBuf::uninit(spare); + + // SAFETY: These bytes were initialized but not filled in the previous loop + unsafe { + read_buf.assume_init(read_buf.capacity()); + } + + match ready!(this.reader.poll_read(cx, read_buf.initialize_unfilled())) { Ok(0) => { - unsafe { - this.buf.set_len(*this.length); - } - return Poll::Ready(Ok(*this.length - *this.start)); + return Poll::Ready(Ok(this.buf.len() - start_len)); } Ok(n) => { - *this.next = if n >= *this.next { - cmp::min((*this.next).saturating_mul(2), MAX_READ_TO_END_GROW_SIZE) - } else if n >= *this.next / 2 { - *this.next - } else { - cmp::max((*this.next).saturating_div(2), MIN_READ_TO_END_GROW_SIZE) - }; - // We can't allow bogus values from read. If it is too large, the returned vec could have its length - // set past its capacity, or if it overflows the vec could be shortened which could create an invalid - // string if this is called via read_to_string. - assert!(n <= buf.len()); - *this.length += n; + // SAFETY: Read API makes sure that returning `n` is correct. + unsafe { + this.buf.set_len(this.buf.len() + n); + } } Err(e) => return Poll::Ready(Err(e)), } + + // The buffer might be an exact fit. Let's read into a probe buffer + // and see if it returns `Ok(0)`. If so, we've avoided an + // unnecessary doubling of the capacity. But if not, append the + // probe buffer to the primary buffer and let its capacity grow. + if this.buf.len() == this.buf.capacity() && this.buf.capacity() == start_cap { + let mut probe = [0u8; 32]; + + match ready!(this.reader.poll_read(cx, &mut probe)) { + Ok(0) => return Poll::Ready(Ok(this.buf.len() - start_len)), + Ok(n) => { + this.buf.extend_from_slice(&probe[..n]); + } + Err(e) => return Poll::Ready(Err(e)), + } + } } } } diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 7e1b8b3346bb..9b1da6f23455 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -375,6 +375,11 @@ mod tests { ) .mount(&mock_server) .await; + Mock::given(method("HEAD")) + .and(path("/hello")) + .respond_with(ResponseTemplate::new(200).insert_header("content-length", "13")) + .mount(&mock_server) + .await; let mut builder = HttpBuilder::default(); builder.endpoint(&mock_server.uri()); @@ -404,6 +409,12 @@ mod tests { ) .mount(&mock_server) .await; + Mock::given(method("HEAD")) + .and(path("/hello")) + .and(basic_auth(username, password)) + .respond_with(ResponseTemplate::new(200).insert_header("content-length", "13")) + .mount(&mock_server) + .await; let mut builder = HttpBuilder::default(); builder.endpoint(&mock_server.uri()); @@ -434,6 +445,12 @@ mod tests { ) .mount(&mock_server) .await; + Mock::given(method("HEAD")) + .and(path("/hello")) + .and(bearer_token(token)) + .respond_with(ResponseTemplate::new(200).insert_header("content-length", "13")) + .mount(&mock_server) + .await; let mut builder = HttpBuilder::default(); builder.endpoint(&mock_server.uri()); @@ -484,6 +501,11 @@ mod tests { ) .mount(&mock_server) .await; + Mock::given(method("HEAD")) + .and(path("/hello")) + .respond_with(ResponseTemplate::new(200).insert_header("content-length", "13")) + .mount(&mock_server) + .await; let mut builder = HttpBuilder::default(); builder.endpoint(&mock_server.uri()); diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index 0d24728f5734..e1de7cc74cf3 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -367,8 +367,22 @@ impl Operator { .with_context("path", &path)); } + let range = args.range(); + let size_hint = match range.size() { + Some(v) => v, + None => { + let mut size = inner + .stat(&path, OpStat::default()) + .await? + .into_metadata() + .content_length(); + size -= range.offset().unwrap_or(0); + size + } + }; + let (_, mut s) = inner.read(&path, args).await?; - let mut buf = Vec::new(); + let mut buf = Vec::with_capacity(size_hint as usize); s.read_to_end(&mut buf).await?; Ok(buf)