feat: Add github action cache service support (#1111)

* feat: Add github action cache service support Signed-off-by: Xuanwo <[email protected]> * Fix typo Signed-off-by: Xuanwo <[email protected]> * Fix build Signed-off-by: Xuanwo <[email protected]> * Set env Signed-off-by: Xuanwo <[email protected]> * Set version for cache api Signed-off-by: Xuanwo <[email protected]> * Fix cache_id not found Signed-off-by: Xuanwo <[email protected]> * Ignore conflict errors Signed-off-by: Xuanwo <[email protected]> * make sure path encoded Signed-off-by: Xuanwo <[email protected]> * Add preview Signed-off-by: Xuanwo <[email protected]> * Fix no content Signed-off-by: Xuanwo <[email protected]> * Make test easier Signed-off-by: Xuanwo <[email protected]> * Fix write file Signed-off-by: Xuanwo <[email protected]> * Add github token Signed-off-by: Xuanwo <[email protected]> * Add opendal user agent Signed-off-by: Xuanwo <[email protected]> * Add version in user agent Signed-off-by: Xuanwo <[email protected]> * Rename Signed-off-by: Xuanwo <[email protected]> * Only test create file Signed-off-by: Xuanwo <[email protected]> * Fix reserve size Signed-off-by: Xuanwo <[email protected]> * Add operation Signed-off-by: Xuanwo <[email protected]> * Fix operation not with Signed-off-by: Xuanwo <[email protected]> * Fix typo Signed-off-by: Xuanwo <[email protected]> * For test Signed-off-by: Xuanwo <[email protected]> * Gogo Signed-off-by: Xuanwo <[email protected]> * Try again Signed-off-by: Xuanwo <[email protected]> * Try again Signed-off-by: Xuanwo <[email protected]> * ENable create simulation Signed-off-by: Xuanwo <[email protected]> * Run full test Signed-off-by: Xuanwo <[email protected]> * Fix typo Signed-off-by: Xuanwo <[email protected]> * Return unsupported for read with suffix Signed-off-by: Xuanwo <[email protected]> * Remove debug log Signed-off-by: Xuanwo <[email protected]> * Revert "Make test easier" This reverts commit 94fcff2. * Remove token Signed-off-by: Xuanwo <[email protected]> Signed-off-by: Xuanwo <[email protected]>
apache · Jan 3, 2023 · 707ee5b · 707ee5b · github-actions · Jan 3, 2023
1 parent 547a2c5
commit 707ee5b
Show file tree

Hide file tree

Showing 11 changed files with 718 additions and 1 deletion.
diff --git a/.env.example b/.env.example
@@ -63,3 +63,5 @@ OPENDAL_ROCKSDB_DATADIR=/path/to/database
 OPENDAL_ROCKSDB_ROOT=/path/to/root
 # moka
 OPENDAL_MOKA_TEST=false
+# ghac
+OPENDAL_GHAC_TEST=false
diff --git a/.github/workflows/service_test_ghac.yml b/.github/workflows/service_test_ghac.yml
@@ -0,0 +1,39 @@
+name: Service Test Ghac
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  ghac:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Configure Cache Env
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Test
+        shell: bash
+        run: cargo test ghac --features compress -- --nocapture
+        env:
+          RUST_BACKTRACE: full
+          RUST_LOG: debug
+          OPENDAL_GHAC_TEST: ${{ secrets.OPENDAL_GHAC_TEST }}
+          OPENDAL_GHAC_ENABLE_CREATE_SIMULATION: true
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/src/operator.rs b/src/operator.rs
@@ -151,6 +151,7 @@ impl Operator {
             #[cfg(feature = "services-ftp")]
             Scheme::Ftp => services::ftp::Builder::from_iter(it).build()?.into(),
             Scheme::Gcs => services::gcs::Builder::from_iter(it).build()?.into(),
+            Scheme::Ghac => services::ghac::Builder::from_iter(it).build()?.into(),
             #[cfg(feature = "services-hdfs")]
             Scheme::Hdfs => services::hdfs::Builder::from_iter(it).build()?.into(),
             Scheme::Http => services::http::Builder::from_iter(it).build()?.into(),

diff --git a/src/raw/http_util/bytes_content_range.rs b/src/raw/http_util/bytes_content_range.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Display;
+use std::fmt::Formatter;
 use std::ops::Range;
 use std::ops::RangeInclusive;
 use std::str::FromStr;
@@ -102,6 +104,11 @@ impl BytesContentRange {
         }
     }
 
+    /// Convert bytes content range into Content-Range header.
+    pub fn to_header(&self) -> String {
+        format!("bytes {self}")
+    }
+
     /// Calculate bytes content range from size and specfied range.
     pub fn from_bytes_range(total_size: u64, range: BytesRange) -> Self {
         let (start, end) = match (range.offset(), range.size()) {
@@ -124,6 +131,17 @@ impl BytesContentRange {
     }
 }
 
+impl Display for BytesContentRange {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match (self.0, self.1, self.2) {
+            (Some(start), Some(end), Some(size)) => write!(f, "{start}-{end}/{size}"),
+            (Some(start), Some(end), None) => write!(f, "{start}-{end}/*"),
+            (None, None, Some(size)) => write!(f, "*/{size}"),
+            _ => unreachable!("invalid bytes range: {:?}", self),
+        }
+    }
+}
+
 impl FromStr for BytesContentRange {
     type Err = Error;
 
@@ -255,4 +273,32 @@ mod tests {
             assert_eq!(expected, actual, "{name}")
         }
     }
+
+    #[test]
+    fn test_bytes_content_range_to_string() {
+        let h = BytesContentRange::default().with_size(1024);
+        assert_eq!(h.to_string(), "*/1024");
+
+        let h = BytesContentRange::default().with_range(0, 1023);
+        assert_eq!(h.to_string(), "0-1023/*");
+
+        let h = BytesContentRange::default()
+            .with_range(0, 1023)
+            .with_size(1024);
+        assert_eq!(h.to_string(), "0-1023/1024");
+    }
+
+    #[test]
+    fn test_bytes_content_range_to_header() {
+        let h = BytesContentRange::default().with_size(1024);
+        assert_eq!(h.to_header(), "bytes */1024");
+
+        let h = BytesContentRange::default().with_range(0, 1023);
+        assert_eq!(h.to_header(), "bytes 0-1023/*");
+
+        let h = BytesContentRange::default()
+            .with_range(0, 1023)
+            .with_size(1024);
+        assert_eq!(h.to_header(), "bytes 0-1023/1024");
+    }
 }
diff --git a/src/raw/path.rs b/src/raw/path.rs
@@ -308,6 +308,7 @@ mod tests {
             ("input dir", "/abc/", "def/", "abc/def/"),
             ("input file", "/abc/", "def", "abc/def"),
             ("input abs file with root /", "/", "/", ""),
+            ("input empty with root /", "/", "", ""),
             ("input dir with root /", "/", "def/", "def/"),
             ("input file with root /", "/", "def", "def"),
         ];

diff --git a/src/scheme.rs b/src/scheme.rs
@@ -36,6 +36,8 @@ pub enum Scheme {
     Fs,
     /// [gcs][crate::services::gcs]: Google Cloud Storage backend.
     Gcs,
+    /// [ghac][crate::services::ghac]: Github Action Cache services.
+    Ghac,
     /// [hdfs][crate::services::hdfs]: Hadoop Distributed File System.
     #[cfg(feature = "services-hdfs")]
     Hdfs,
@@ -94,9 +96,10 @@ impl Display for Scheme {
             Scheme::Azblob => write!(f, "azblob"),
             Scheme::Azdfs => write!(f, "azdfs"),
             Scheme::Fs => write!(f, "fs"),
+            Scheme::Gcs => write!(f, "gcs"),
+            Scheme::Ghac => write!(f, "ghac"),
             #[cfg(feature = "services-hdfs")]
             Scheme::Hdfs => write!(f, "hdfs"),
-            Scheme::Gcs => write!(f, "gcs"),
             Scheme::Http => write!(f, "http"),
             #[cfg(feature = "services-ftp")]
             Scheme::Ftp => write!(f, "ftp"),
@@ -128,6 +131,7 @@ impl FromStr for Scheme {
             "azdfs" => Ok(Scheme::Azdfs),
             "fs" => Ok(Scheme::Fs),
             "gcs" => Ok(Scheme::Gcs),
+            "ghac" => Ok(Scheme::Ghac),
             #[cfg(feature = "services-hdfs")]
             "hdfs" => Ok(Scheme::Hdfs),
             "http" | "https" => Ok(Scheme::Http),
@@ -158,6 +162,7 @@ impl From<Scheme> for &'static str {
             Scheme::Azdfs => "azdfs",
             Scheme::Fs => "fs",
             Scheme::Gcs => "gcs",
+            Scheme::Ghac => "ghac",
             #[cfg(feature = "services-hdfs")]
             Scheme::Hdfs => "hdfs",
             Scheme::Http => "http",