feat(core): Make gcs available on wasm32 arch (#3816)

* Fix build on gcs Signed-off-by: Xuanwo <[email protected]> * Add CI Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
apache · Dec 25, 2023 · dd59418 · dd59418
1 parent c8bd5c4
commit dd59418
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -293,7 +293,7 @@ jobs:
           )
           cargo build --features "${FEATURES[*]}"
 
-  # We only support some services(see `available_services` below) for now, but we will extend wasm support for other services.
+  # We only support some services(see `available_services` below) for now.
   build_under_wasm:
     runs-on: ubuntu-latest
     steps:
@@ -304,14 +304,13 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build
         run: |
-          available_services=(
+          FEATURES=(
             services-azblob
             services-gdrive
             services-s3
           )
-          IFS=","
           rustup target add wasm32-unknown-unknown
-          cargo build --target wasm32-unknown-unknown --no-default-features --features="${available_services[*]}"
+          cargo build --target wasm32-unknown-unknown --no-default-features --features="${FEATURES[*]}"
 
   unit:
     runs-on: ubuntu-latest

diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -329,4 +329,4 @@ tracing-subscriber = { version = "0.3", features = [
   "env-filter",
   "tracing-log",
 ] }
-wiremock = "0.5"
+wiremock = "0.5"
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
@@ -20,7 +20,6 @@ use std::task::Context;
 use std::task::Poll;
 
 use async_trait::async_trait;
-use futures::future::BoxFuture;
 use futures::FutureExt;
 
 use crate::raw::oio::WriteBuf;
@@ -48,7 +47,8 @@ use crate::*;
 /// - Services impl `RangeWrite`
 /// - `RangeWriter` impl `Write`
 /// - Expose `RangeWriter` as `Accessor::Writer`
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 pub trait RangeWrite: Send + Sync + Unpin + 'static {
     /// write_once is used to write the data to underlying storage at once.
     ///
@@ -93,12 +93,17 @@ pub struct RangeWriter<W: RangeWrite> {
 
 enum State<W> {
     Idle(Option<W>),
-    Init(BoxFuture<'static, (W, Result<String>)>),
-    Write(BoxFuture<'static, (W, Result<u64>)>),
-    Complete(BoxFuture<'static, (W, Result<()>)>),
-    Abort(BoxFuture<'static, (W, Result<()>)>),
+    Init(BoxedFuture<(W, Result<String>)>),
+    Write(BoxedFuture<(W, Result<u64>)>),
+    Complete(BoxedFuture<(W, Result<()>)>),
+    Abort(BoxedFuture<(W, Result<()>)>),
 }
 
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this state.
+unsafe impl<S: RangeWrite> Send for State<S> {}
+
 /// # Safety
 ///
 /// We will only take `&mut Self` reference for State.

diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
@@ -34,8 +34,8 @@ use super::core::*;
 use super::error::parse_error;
 use super::lister::GcsLister;
 use super::writer::GcsWriter;
+use super::writer::GcsWriters;
 use crate::raw::*;
-use crate::services::gcs::writer::GcsWriters;
 use crate::*;
 
 const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";
@@ -269,6 +269,11 @@ impl Builder for GcsBuilder {
         if let Some(cred) = &self.config.credential_path {
             cred_loader = cred_loader.with_path(cred);
         }
+        #[cfg(target_arch = "wasm32")]
+        {
+            cred_loader = cred_loader.with_disable_env();
+            cred_loader = cred_loader.with_disable_well_known_location();
+        }
 
         let scope = if let Some(scope) = &self.config.scope {
             scope
@@ -313,7 +318,8 @@ pub struct GcsBackend {
     core: Arc<GcsCore>,
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl Accessor for GcsBackend {
     type Reader = IncomingAsyncBody;
     type BlockingReader = ();

diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs
@@ -60,7 +60,8 @@ impl GcsLister {
     }
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl oio::PageList for GcsLister {
     async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
         let resp = self

diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
@@ -43,7 +43,8 @@ impl GcsWriter {
     }
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl oio::RangeWrite for GcsWriter {
     async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
         let mut req = self.core.gcs_insert_object_request(