From f0c6607634646e856ec4b564c9b1b7fcf730ddd2 Mon Sep 17 00:00:00 2001 From: xxchan Date: Tue, 5 Mar 2024 12:00:28 +0800 Subject: [PATCH 1/4] fix: fix main CI - add previous default features to bindings (cpp,dotnet,lua,ruby) - fix ghac ci --- .github/workflows/service_test_ghac.yml | 2 +- bindings/cpp/Cargo.toml | 19 ++++++++++++++++++- bindings/dotnet/Cargo.toml | 19 ++++++++++++++++++- bindings/lua/Cargo.toml | 19 ++++++++++++++++++- bindings/ruby/Cargo.toml | 19 ++++++++++++++++++- 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/service_test_ghac.yml b/.github/workflows/service_test_ghac.yml index ad4ae589c862..826f67d6f433 100644 --- a/.github/workflows/service_test_ghac.yml +++ b/.github/workflows/service_test_ghac.yml @@ -57,7 +57,7 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run behavior --features tests + run: cargo nextest run behavior --features tests services-ghac env: OPENDAL_TEST: ghac GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/bindings/cpp/Cargo.toml b/bindings/cpp/Cargo.toml index 57f1b7cfb731..ab8c2caf97c5 100644 --- a/bindings/cpp/Cargo.toml +++ b/bindings/cpp/Cargo.toml @@ -34,7 +34,24 @@ crate-type = ["staticlib"] anyhow = "1.0" chrono = "0.4" cxx = "1.0" -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } [build-dependencies] cxx-build = "1.0" diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml index a0320fdb772a..d1848d8d8bf6 100644 --- a/bindings/dotnet/Cargo.toml +++ b/bindings/dotnet/Cargo.toml @@ -32,4 +32,21 @@ crate-type = ["cdylib"] doc = false [dependencies] -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml index c0b6bbd73f93..0dceaa2f7d32 100644 --- a/bindings/lua/Cargo.toml +++ b/bindings/lua/Cargo.toml @@ -39,4 +39,21 @@ mlua = { version = "0.9", features = [ "module", "macros", ], default-features = false, optional = true } -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml index 49d46686e52f..435d20136de6 100644 --- a/bindings/ruby/Cargo.toml +++ b/bindings/ruby/Cargo.toml @@ -34,7 +34,24 @@ name = "opendal_ruby" [dependencies] magnus = { version = "0.5", features = ["bytes-crate"] } -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } rb-sys = { version = "0.9.77", default-features = false } [build-dependencies] From bf8cfd07593a7d15d930479979cf37772998fcf9 Mon Sep 17 00:00:00 2001 From: xxchan Date: Tue, 5 Mar 2024 12:07:06 +0800 Subject: [PATCH 2/4] update all bindings --- bindings/haskell/Cargo.toml | 19 ++++++++++++++++++- bindings/ocaml/Cargo.toml | 19 ++++++++++++++++++- bindings/php/Cargo.toml | 20 +++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml index 32d9da1cec82..26fdf5ae68d5 100644 --- a/bindings/haskell/Cargo.toml +++ b/bindings/haskell/Cargo.toml @@ -34,4 +34,21 @@ doc = false [dependencies] chrono = "0.4" log = { version = "0.4", features = ["std"] } -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml index 09347ade690d..01448cc92c92 100644 --- a/bindings/ocaml/Cargo.toml +++ b/bindings/ocaml/Cargo.toml @@ -33,7 +33,24 @@ doc = false [dependencies] ocaml = { version = "^1.0.0-beta" } -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } [build-dependencies] ocaml-build = { version = "^1.0.0-beta" } diff --git a/bindings/php/Cargo.toml b/bindings/php/Cargo.toml index 4e2f62898d4f..e4b5e268d5c2 100644 --- a/bindings/php/Cargo.toml +++ b/bindings/php/Cargo.toml @@ -32,4 +32,22 @@ crate-type = ["cdylib"] [dependencies] ext-php-rs = "0.11.2" -opendal = { version = "0.45.1", path = "../../core" } +opendal = { version = "0.45.1", path = "../../core", features = [ + # These are default features before v0.46. TODO: change to optional features + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", + "services-azfile", +] } + From e5fb8908fca5aa47504769096942245e5b21357b Mon Sep 17 00:00:00 2001 From: xxchan Date: Tue, 5 Mar 2024 15:41:24 +0800 Subject: [PATCH 3/4] fix compile for oss (once_cell not found) --- core/Cargo.lock | 103 ++++++++++++++++++++++++++++++++++++++++++++++-- core/Cargo.toml | 2 +- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 3cb8756de2d1..2dfab9bbafe1 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.7.7" @@ -1055,6 +1066,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.5.1" @@ -1253,6 +1273,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.0.83" @@ -1325,6 +1354,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.7.0" @@ -3050,6 +3089,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -3721,7 +3770,7 @@ dependencies = [ "hmac", "lazy_static", "md-5", - "pbkdf2", + "pbkdf2 0.11.0", "percent-encoding", "rand 0.8.5", "rustc_version_runtime", @@ -4393,6 +4442,16 @@ dependencies = [ "digest", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -4532,6 +4591,21 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der 0.7.8", + "pbkdf2 0.12.2", + "scrypt", + "sha2", + "spki 0.7.3", +] + [[package]] name = "pkcs8" version = "0.9.0" @@ -4549,6 +4623,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der 0.7.8", + "pkcs5", + "rand_core 0.6.4", "spki 0.7.3", ] @@ -5258,9 +5334,9 @@ dependencies = [ [[package]] name = "reqsign" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed08ac3aa0676637644b1b892202f1ae789c28c15ebfa906128d111ae8086062" +checksum = "43e319d9de9ff4d941abf4ac718897118b0fe04577ea3f8e0f5788971784eef5" dependencies = [ "anyhow", "async-trait", @@ -5453,6 +5529,7 @@ dependencies = [ "pkcs1", "pkcs8 0.10.2", "rand_core 0.6.4", + "sha2", "signature 2.2.0", "spki 0.7.3", "subtle", @@ -5654,6 +5731,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -5699,6 +5785,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2 0.12.2", + "salsa20", + "sha2", +] + [[package]] name = "sct" version = "0.7.1" diff --git a/core/Cargo.toml b/core/Cargo.toml index 83155584137e..138102fe9128 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -274,7 +274,7 @@ sha1 = { version = "0.10.6", optional = true } sha2 = { version = "0.10", optional = true } # For http based services. -reqsign = { version = "0.14.7", default-features = false, optional = true } +reqsign = { version = "0.14.9", default-features = false, optional = true } # for services-atomic-server atomic_lib = { version = "0.34.5", optional = true } From 88c062c5af45cc7c175bd04f578acd22af1b864c Mon Sep 17 00:00:00 2001 From: xxchan Date: Tue, 5 Mar 2024 15:42:58 +0800 Subject: [PATCH 4/4] include memory in default --- core/Cargo.toml | 2 +- core/src/docs/upgrade.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 138102fe9128..451280d71c4c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -39,7 +39,7 @@ default-members = ["."] members = [".", "fuzz", "edge/*", "benches/vs_*"] [features] -default = ["rustls"] +default = ["rustls", "services-memory"] # Build test utils or not. # diff --git a/core/src/docs/upgrade.md b/core/src/docs/upgrade.md index dfd7198acced..3ffa84544419 100644 --- a/core/src/docs/upgrade.md +++ b/core/src/docs/upgrade.md @@ -2,7 +2,7 @@ ## Services Feature Flag -Starting from v0.46, OpenDAL does not include any services in default features to avoid compiling unneeded services' code. Please enable each service's feature flag to use it. +Starting from v0.46, OpenDAL does not include any services except memory service in default features to avoid compiling unneeded services' code. Please enable each service's feature flag to use it. # Upgrade to v0.45