From afda01e143c70dcc8035cfdf87fd1e5cd1fe0553 Mon Sep 17 00:00:00 2001 From: Mike Chong Date: Thu, 13 Jun 2024 03:00:23 +0000 Subject: [PATCH 1/7] fix gdrive --- core/src/services/gdrive/core.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index ddc3968b884..46cd5a15d01 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -396,6 +396,9 @@ impl PathQuery for GdrivePathQuery { async fn create_dir(&self, parent_id: &str, name: &str) -> Result { let url = "https://www.googleapis.com/drive/v3/files"; + // trim "/" at the end of name + let name = name.trim_end_matches('/'); + let content = serde_json::to_vec(&json!({ "name": name, "mimeType": "application/vnd.google-apps.folder", From 0cdbbe22e7070c1c4b3cffb4e18a493de29cb924 Mon Sep 17 00:00:00 2001 From: Mike Chong Date: Thu, 13 Jun 2024 03:11:03 +0000 Subject: [PATCH 2/7] f --- core/src/services/gdrive/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 46cd5a15d01..91d61c3a887 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -397,7 +397,7 @@ impl PathQuery for GdrivePathQuery { let url = "https://www.googleapis.com/drive/v3/files"; // trim "/" at the end of name - let name = name.trim_end_matches('/'); + let name = name.trim_end_matches("/"); let content = serde_json::to_vec(&json!({ "name": name, From 1260920fed6c31f72929f790dce689f760ea8c4f Mon Sep 17 00:00:00 2001 From: Mike Chong Date: Fri, 14 Jun 2024 04:04:19 +0000 Subject: [PATCH 3/7] normalize path for gdrive? --- core/src/raw/path_cache.rs | 34 ++++++++++++++++++++++++++++++-- core/src/services/gdrive/core.rs | 4 ++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs index 249fcb297b4..679f8f60471 100644 --- a/core/src/raw/path_cache.rs +++ b/core/src/raw/path_cache.rs @@ -93,6 +93,8 @@ impl PathCacher { pub async fn insert(&self, path: &str, id: &str) { let _guard = self.lock().await; + let path = normalize_dir_path(path); + // This should never happen, but let's ignore the insert if happened. if self.cache.contains_key(path) { debug_assert!( @@ -109,6 +111,8 @@ impl PathCacher { pub async fn remove(&self, path: &str) { let _guard = self.lock().await; + let path = normalize_dir_path(path); + self.cache.invalidate(path) } @@ -116,6 +120,8 @@ impl PathCacher { pub async fn get(&self, path: &str) -> Result> { let _guard = self.lock().await; + let path = normalize_dir_path(path); + if let Some(id) = self.cache.get(path) { return Ok(Some(id)); } @@ -158,6 +164,7 @@ impl PathCacher { /// Ensure input dir exists. pub async fn ensure_dir(&self, path: &str) -> Result { let _guard = self.lock().await; + let path = normalize_dir_path(path); let mut tmp = "".to_string(); // All parents that need to check. @@ -177,7 +184,8 @@ impl PathCacher { None => self.query.root().await?, }; for parent in parents { - parent_id = match self.cache.get(&parent) { + let parent = normalize_dir_path(&parent); + parent_id = match self.cache.get(parent) { Some(value) => value, None => { let value = match self.query.query(&parent_id, get_basename(&parent)).await? { @@ -188,7 +196,7 @@ impl PathCacher { .await? } }; - self.cache.insert(parent, value.clone()); + self.cache.insert(parent.to_owned(), value.clone()); value } } @@ -198,9 +206,22 @@ impl PathCacher { } } +/// Normalize the path for dirs +pub fn normalize_dir_path(path: &str) -> &str { + if path == "/" { + path + } else if path.ends_with('/') { + path.split_at(path.len() - 1).0 + } else { + path + } +} + #[cfg(test)] mod tests { + use raw::normalize_dir_path; + use crate::raw::PathCacher; use crate::raw::PathQuery; use crate::*; @@ -241,4 +262,13 @@ mod tests { assert_eq!(actual.as_deref(), expect, "{}", name) } } + + #[test] + fn test_normalize_dir_path() { + assert_eq!(normalize_dir_path("/"), "/"); + assert_eq!(normalize_dir_path("/a"), "/a"); + assert_eq!(normalize_dir_path("/a/"), "/a"); + assert_eq!(normalize_dir_path("/a/b"), "/a/b"); + assert_eq!(normalize_dir_path("/a/b/"), "/a/b"); + } } diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 91d61c3a887..4fbf1aea066 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -396,8 +396,8 @@ impl PathQuery for GdrivePathQuery { async fn create_dir(&self, parent_id: &str, name: &str) -> Result { let url = "https://www.googleapis.com/drive/v3/files"; - // trim "/" at the end of name - let name = name.trim_end_matches("/"); + // trim "/" at the end of name because Google Drive API includes "/" in the name of folder + let name = normalize_dir_path(name); let content = serde_json::to_vec(&json!({ "name": name, From 24e93cac80bc16683b4e301bb603d786bc53576d Mon Sep 17 00:00:00 2001 From: Mike Chong Date: Tue, 18 Jun 2024 12:51:59 +0000 Subject: [PATCH 4/7] Revert "normalize path for gdrive?" This reverts commit 1260920fed6c31f72929f790dce689f760ea8c4f. --- core/src/raw/path_cache.rs | 34 ++------------------------------ core/src/services/gdrive/core.rs | 4 ++-- 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs index 679f8f60471..249fcb297b4 100644 --- a/core/src/raw/path_cache.rs +++ b/core/src/raw/path_cache.rs @@ -93,8 +93,6 @@ impl PathCacher { pub async fn insert(&self, path: &str, id: &str) { let _guard = self.lock().await; - let path = normalize_dir_path(path); - // This should never happen, but let's ignore the insert if happened. if self.cache.contains_key(path) { debug_assert!( @@ -111,8 +109,6 @@ impl PathCacher { pub async fn remove(&self, path: &str) { let _guard = self.lock().await; - let path = normalize_dir_path(path); - self.cache.invalidate(path) } @@ -120,8 +116,6 @@ impl PathCacher { pub async fn get(&self, path: &str) -> Result> { let _guard = self.lock().await; - let path = normalize_dir_path(path); - if let Some(id) = self.cache.get(path) { return Ok(Some(id)); } @@ -164,7 +158,6 @@ impl PathCacher { /// Ensure input dir exists. pub async fn ensure_dir(&self, path: &str) -> Result { let _guard = self.lock().await; - let path = normalize_dir_path(path); let mut tmp = "".to_string(); // All parents that need to check. @@ -184,8 +177,7 @@ impl PathCacher { None => self.query.root().await?, }; for parent in parents { - let parent = normalize_dir_path(&parent); - parent_id = match self.cache.get(parent) { + parent_id = match self.cache.get(&parent) { Some(value) => value, None => { let value = match self.query.query(&parent_id, get_basename(&parent)).await? { @@ -196,7 +188,7 @@ impl PathCacher { .await? } }; - self.cache.insert(parent.to_owned(), value.clone()); + self.cache.insert(parent, value.clone()); value } } @@ -206,22 +198,9 @@ impl PathCacher { } } -/// Normalize the path for dirs -pub fn normalize_dir_path(path: &str) -> &str { - if path == "/" { - path - } else if path.ends_with('/') { - path.split_at(path.len() - 1).0 - } else { - path - } -} - #[cfg(test)] mod tests { - use raw::normalize_dir_path; - use crate::raw::PathCacher; use crate::raw::PathQuery; use crate::*; @@ -262,13 +241,4 @@ mod tests { assert_eq!(actual.as_deref(), expect, "{}", name) } } - - #[test] - fn test_normalize_dir_path() { - assert_eq!(normalize_dir_path("/"), "/"); - assert_eq!(normalize_dir_path("/a"), "/a"); - assert_eq!(normalize_dir_path("/a/"), "/a"); - assert_eq!(normalize_dir_path("/a/b"), "/a/b"); - assert_eq!(normalize_dir_path("/a/b/"), "/a/b"); - } } diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 17537a1eddf..2d0d60f8e82 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -396,8 +396,8 @@ impl PathQuery for GdrivePathQuery { async fn create_dir(&self, parent_id: &str, name: &str) -> Result { let url = "https://www.googleapis.com/drive/v3/files"; - // trim "/" at the end of name because Google Drive API includes "/" in the name of folder - let name = normalize_dir_path(name); + // trim "/" at the end of name + let name = name.trim_end_matches("/"); let content = serde_json::to_vec(&json!({ "name": name, From fee42a6b11cab97f8dd66357bf147033f0058ee7 Mon Sep 17 00:00:00 2001 From: Mike Chong Date: Tue, 18 Jun 2024 14:28:55 +0000 Subject: [PATCH 5/7] fix --- core/src/services/gdrive/core.rs | 31 ++++++++++++++++++++++++++---- core/src/services/gdrive/lister.rs | 5 ++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 2d0d60f8e82..3accf002b39 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -57,7 +57,7 @@ impl Debug for GdriveCore { impl GdriveCore { pub async fn gdrive_stat(&self, path: &str) -> Result> { - let path = build_abs_path(&self.root, path); + let path = gdrive_normalize_dir_path(build_abs_path(&self.root, path)); let file_id = self.path_cache.get(&path).await?.ok_or(Error::new( ErrorKind::NotFound, format!("path not found: {}", path), @@ -77,7 +77,7 @@ impl GdriveCore { } pub async fn gdrive_get(&self, path: &str, range: BytesRange) -> Result> { - let path = build_abs_path(&self.root, path); + let path = gdrive_normalize_dir_path(build_abs_path(&self.root, path)); let path_id = self.path_cache.get(&path).await?.ok_or(Error::new( ErrorKind::NotFound, format!("path not found: {}", path), @@ -184,7 +184,10 @@ impl GdriveCore { size: u64, body: Buffer, ) -> Result> { - let parent = self.path_cache.ensure_dir(get_parent(path)).await?; + let parent = self + .path_cache + .ensure_dir(&gdrive_normalize_dir_path(get_parent(path).to_owned())) + .await?; let url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart"; @@ -397,7 +400,7 @@ impl PathQuery for GdrivePathQuery { let url = "https://www.googleapis.com/drive/v3/files"; // trim "/" at the end of name - let name = name.trim_end_matches("/"); + let name = gdrive_normalize_dir_path(name.to_owned()); let content = serde_json::to_vec(&json!({ "name": name, @@ -457,3 +460,23 @@ pub(crate) struct GdriveFileList { pub(crate) files: Vec, pub(crate) next_page_token: Option, } + +/// On Google Drive, dir path name cannot end with slash (/) because if we add a slash, it would be different from the dir name +pub(super) fn gdrive_normalize_dir_path(path: String) -> String { + if path == "/" { + path + } else if path.ends_with('/') { + path.split_at(path.len() - 1).0.to_owned() + } else { + path + } +} + +#[test] +fn test_normalize_dir_path() { + assert_eq!(gdrive_normalize_dir_path("/".to_owned()), "/"); + assert_eq!(gdrive_normalize_dir_path("/a".to_owned()), "/a"); + assert_eq!(gdrive_normalize_dir_path("/a/".to_owned()), "/a"); + assert_eq!(gdrive_normalize_dir_path("/a/b".to_owned()), "/a/b"); + assert_eq!(gdrive_normalize_dir_path("/a/b/".to_owned()), "/a/b"); +} diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs index e77a5022da7..f99a519b829 100644 --- a/core/src/services/gdrive/lister.rs +++ b/core/src/services/gdrive/lister.rs @@ -32,7 +32,10 @@ pub struct GdriveLister { impl GdriveLister { pub fn new(path: String, core: Arc) -> Self { - Self { path, core } + Self { + path: super::core::gdrive_normalize_dir_path(path), + core, + } } } From 947ce4a3ffdfa875ac0ba845d933f5a96b5e16df Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 20 Jun 2024 10:49:31 -0600 Subject: [PATCH 6/7] remove all use for normlaize --- core/src/services/gdrive/core.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 3accf002b39..7afb076a95d 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -57,7 +57,7 @@ impl Debug for GdriveCore { impl GdriveCore { pub async fn gdrive_stat(&self, path: &str) -> Result> { - let path = gdrive_normalize_dir_path(build_abs_path(&self.root, path)); + let path = build_abs_path(&self.root, path); let file_id = self.path_cache.get(&path).await?.ok_or(Error::new( ErrorKind::NotFound, format!("path not found: {}", path), @@ -77,7 +77,7 @@ impl GdriveCore { } pub async fn gdrive_get(&self, path: &str, range: BytesRange) -> Result> { - let path = gdrive_normalize_dir_path(build_abs_path(&self.root, path)); + let path = build_abs_path(&self.root, path); let path_id = self.path_cache.get(&path).await?.ok_or(Error::new( ErrorKind::NotFound, format!("path not found: {}", path), @@ -184,10 +184,7 @@ impl GdriveCore { size: u64, body: Buffer, ) -> Result> { - let parent = self - .path_cache - .ensure_dir(&gdrive_normalize_dir_path(get_parent(path).to_owned())) - .await?; + let parent = self.path_cache.ensure_dir(get_parent(path)).await?; let url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart"; @@ -399,9 +396,6 @@ impl PathQuery for GdrivePathQuery { async fn create_dir(&self, parent_id: &str, name: &str) -> Result { let url = "https://www.googleapis.com/drive/v3/files"; - // trim "/" at the end of name - let name = gdrive_normalize_dir_path(name.to_owned()); - let content = serde_json::to_vec(&json!({ "name": name, "mimeType": "application/vnd.google-apps.folder", From 70f059c12a010d6d9a38c2538b8d35a7dbe829d9 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 20 Jun 2024 11:21:35 -0600 Subject: [PATCH 7/7] simplfiy impl --- core/src/services/gdrive/core.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 7afb076a95d..d8e8bd51d51 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -354,7 +354,10 @@ impl PathQuery for GdrivePathQuery { // Make sure name has been replaced with escaped name. // // ref: - format!("name = '{}'", name.replace('\'', "\\'")), + format!( + "name = '{}'", + gdrive_normalize_dir_path(name.to_owned()).replace('\'', "\\'") + ), format!("'{}' in parents", parent_id), "trashed = false".to_string(), ]; @@ -397,7 +400,7 @@ impl PathQuery for GdrivePathQuery { let url = "https://www.googleapis.com/drive/v3/files"; let content = serde_json::to_vec(&json!({ - "name": name, + "name": gdrive_normalize_dir_path(name.to_owned()), "mimeType": "application/vnd.google-apps.folder", // If the parent is not provided, the folder will be created in the root folder. "parents": [parent_id],