From b0dd931197c23d8cd005029aa2fdd5dddb777d63 Mon Sep 17 00:00:00 2001 From: max Date: Thu, 22 Feb 2024 13:48:24 +0200 Subject: [PATCH 01/24] add storage endpoint `artifacts_path` --- ...d_storage_endpoint_artifacts_path.down.sql | 1 + ...add_storage_endpoint_artifacts_path.up.sql | 2 ++ .../src/api/admin/create_storage_endpoint.rs | 23 +++++++----- y-server/src/api/admin/storage_endpoints.rs | 2 +- y-server/src/storage_endpoint.rs | 3 +- y-web/i18n/en.json | 6 ++-- .../storage/endpoints/[endpointId]/index.tsx | 7 ++++ .../pages/storage/endpoints/new/index.tsx | 35 +++++++++++++++++++ .../storage-endpoint/storage-endpoint.api.ts | 2 ++ .../storage-endpoint.codecs.ts | 1 + 10 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.down.sql create mode 100644 y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.up.sql diff --git a/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.down.sql b/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.down.sql new file mode 100644 index 0000000..59d716a --- /dev/null +++ b/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.down.sql @@ -0,0 +1 @@ +ALTER TABLE IF EXISTS public.storage_endpoints DROP COLUMN IF EXISTS artifacts_path; \ No newline at end of file diff --git a/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.up.sql b/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.up.sql new file mode 100644 index 0000000..8590356 --- /dev/null +++ b/y-server/migrations/20240222112849_add_storage_endpoint_artifacts_path.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE IF EXISTS public.storage_endpoints + ADD COLUMN artifacts_path character varying(512); \ No newline at end of file diff --git a/y-server/src/api/admin/create_storage_endpoint.rs b/y-server/src/api/admin/create_storage_endpoint.rs index ec0de58..f10c20e 100644 --- a/y-server/src/api/admin/create_storage_endpoint.rs +++ b/y-server/src/api/admin/create_storage_endpoint.rs @@ -15,6 +15,7 @@ struct CreateStorageEndpointInput { endpoint_type: String, preserve_file_structure: bool, base_path: String, + artifacts_path: String, description: String, } @@ -38,25 +39,29 @@ async fn create_storage_endpoint( let form = form.into_inner(); let base_path = std::path::Path::new(&form.base_path); + let artifacts_path = std::path::Path::new(&form.artifacts_path); - if !base_path.exists() { - return error("create_storage_endpoint.base_path_does_not_exist"); - } + for path in &[base_path, artifacts_path] { + if !path.exists() { + return error("create_storage_endpoint.path_does_not_exist"); + } - if !base_path.is_dir() { - return error("create_storage_endpoint.base_path_not_a_directory"); - } + if !path.is_dir() { + return error("create_storage_endpoint.path_not_a_directory"); + } - if !base_path.is_absolute() { - return error("create_storage_endpoint.base_path_not_absolute"); + if !path.is_absolute() { + return error("create_storage_endpoint.path_not_absolute"); + } } - let result = sqlx::query_scalar("INSERT INTO storage_endpoints (name, endpoint_type, status, preserve_file_structure, base_path, description) VALUES ($1, $2::storage_endpoint_type, $3::storage_endpoint_status, $4, $5, $6) RETURNING id") + let result = sqlx::query_scalar("INSERT INTO storage_endpoints (name, endpoint_type, status, preserve_file_structure, base_path, artifacts_path, description) VALUES ($1, $2::storage_endpoint_type, $3::storage_endpoint_status, $4, $5, $6, $7) RETURNING id") .bind(form.name) .bind(form.endpoint_type) .bind("active") .bind(form.preserve_file_structure) .bind(form.base_path) + .bind(form.artifacts_path) .bind(form.description) .fetch_one(&**pool) .await; diff --git a/y-server/src/api/admin/storage_endpoints.rs b/y-server/src/api/admin/storage_endpoints.rs index 2d5d6b4..4677abe 100644 --- a/y-server/src/api/admin/storage_endpoints.rs +++ b/y-server/src/api/admin/storage_endpoints.rs @@ -26,7 +26,7 @@ async fn storage_enpoints( return error("storage_endpoints.unauthorized"); } - let endpoints = sqlx::query_as::<_, StorageEndpointRow>("SELECT id, name, endpoint_type::TEXT, status::TEXT, preserve_file_structure, base_path, description FROM storage_endpoints") + let endpoints = sqlx::query_as::<_, StorageEndpointRow>("SELECT id, name, endpoint_type::TEXT, status::TEXT, preserve_file_structure, base_path, artifacts_path, description FROM storage_endpoints") .fetch_all(&**pool) .await; diff --git a/y-server/src/storage_endpoint.rs b/y-server/src/storage_endpoint.rs index fc3f6d9..969ede5 100644 --- a/y-server/src/storage_endpoint.rs +++ b/y-server/src/storage_endpoint.rs @@ -11,6 +11,7 @@ pub struct StorageEndpointRow { pub status: String, pub preserve_file_structure: bool, pub base_path: String, + pub artifacts_path: Option, pub description: Option, } @@ -18,7 +19,7 @@ pub async fn get_storage_endpoint( endpoint_id: i32, pool: &RequestPool, ) -> Result { - sqlx::query_as::<_, StorageEndpointRow>("SELECT id, name, endpoint_type::TEXT, status::TEXT, preserve_file_structure, base_path, description FROM storage_endpoints WHERE id = $1") + sqlx::query_as::<_, StorageEndpointRow>("SELECT id, name, endpoint_type::TEXT, status::TEXT, preserve_file_structure, base_path, artifacts_path, description FROM storage_endpoints WHERE id = $1") .bind(endpoint_id) .fetch_one(pool) .await diff --git a/y-web/i18n/en.json b/y-web/i18n/en.json index 8736578..e31569f 100644 --- a/y-web/i18n/en.json +++ b/y-web/i18n/en.json @@ -163,9 +163,9 @@ "other": "Other" }, "create_storage_endpoint": { - "base_path_does_not_exist": "Base path points to a non-existing directory", - "base_path_not_a_directory": "Base path does not point to a directory", - "base_path_not_absolute": "Base path must be an absolute path", + "path_does_not_exist": "Path points to a non-existing directory", + "path_not_a_directory": "Path does not point to a directory", + "path_not_absolute": "Path must be an absolute path", "unauthorized": "Permission denied", "other": "Other" }, diff --git a/y-web/src/modules/admin/pages/storage/endpoints/[endpointId]/index.tsx b/y-web/src/modules/admin/pages/storage/endpoints/[endpointId]/index.tsx index 0d5c2cf..03231cc 100644 --- a/y-web/src/modules/admin/pages/storage/endpoints/[endpointId]/index.tsx +++ b/y-web/src/modules/admin/pages/storage/endpoints/[endpointId]/index.tsx @@ -183,6 +183,13 @@ const StorageEndpointPage = () => { value={$storageEndpoint.data?.base_path ?? ""} onChange={() => void 0} /> + void 0} + /> diff --git a/y-web/src/modules/admin/pages/storage/endpoints/new/index.tsx b/y-web/src/modules/admin/pages/storage/endpoints/new/index.tsx index d766126..f229b2d 100644 --- a/y-web/src/modules/admin/pages/storage/endpoints/new/index.tsx +++ b/y-web/src/modules/admin/pages/storage/endpoints/new/index.tsx @@ -32,6 +32,7 @@ type StorageEndpointFieldValues = { type: IStorageEndpointType preserveFileStructure: boolean basePath: string + artifactsPath: string description: string } @@ -51,6 +52,7 @@ const NewStorageEndpointPage: Component = () => { type: "local_fs", preserveFileStructure: false, basePath: "/var/y_storage", + artifactsPath: "/var/y_artifacts", description: "", }, watch: ["type", "preserveFileStructure"], @@ -223,6 +225,39 @@ const NewStorageEndpointPage: Component = () => { /> + + + + + +
TCreateStorageEndpoint.parse(data)) diff --git a/y-web/src/modules/admin/storage/storage-endpoint/storage-endpoint.codecs.ts b/y-web/src/modules/admin/storage/storage-endpoint/storage-endpoint.codecs.ts index d4992d0..188962c 100644 --- a/y-web/src/modules/admin/storage/storage-endpoint/storage-endpoint.codecs.ts +++ b/y-web/src/modules/admin/storage/storage-endpoint/storage-endpoint.codecs.ts @@ -26,6 +26,7 @@ export const TStorageEndpoint = z.object({ status: TStorageEndpointStatus, preserve_file_structure: z.boolean(), base_path: z.string(), + artifacts_path: z.nullable(z.string()), description: z.nullable(z.string()), }) export type IStorageEndpoint = z.infer From a5c05a7a836bbc9ff5ccd9060e2d108727f3ed16 Mon Sep 17 00:00:00 2001 From: max Date: Tue, 27 Feb 2024 12:58:08 +0200 Subject: [PATCH 02/24] remove rights check for `admin/features` endpoint --- y-server/src/api/admin/features.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/y-server/src/api/admin/features.rs b/y-server/src/api/admin/features.rs index 192fb9a..f96d8bd 100644 --- a/y-server/src/api/admin/features.rs +++ b/y-server/src/api/admin/features.rs @@ -3,7 +3,6 @@ use serde::Serialize; use crate::request::error; -use crate::user::get_client_rights; use crate::util::RequestPool; #[derive(Serialize)] @@ -24,18 +23,7 @@ struct FeatureRow { } #[get("/features")] -async fn features(pool: web::Data, req: actix_web::HttpRequest) -> impl Responder { - let client_rights = get_client_rights(&pool, &req).await; - - let action_allowed = client_rights - .iter() - .find(|right| right.right_name.eq("update_features")) - .is_some(); - - if !action_allowed { - return error("features.unauthorized"); - } - +async fn features(pool: web::Data) -> impl Responder { let feature_rows = sqlx::query_as::<_, FeatureRow>("SELECT feature, enabled FROM features") .fetch_all(&**pool) .await; From 628e5ec20dc1b8f136ce2f128b6ce10f4c0f8116 Mon Sep 17 00:00:00 2001 From: max Date: Thu, 29 Feb 2024 16:58:39 +0200 Subject: [PATCH 03/24] add thumbnail generation logic --- y-server/.env.example | 2 + y-server/Cargo.lock | 29 +++++++- y-server/Cargo.toml | 3 + y-server/src/api/storage/storage_upload.rs | 70 +++++++++++++++--- y-server/src/storage_entry.rs | 83 +++++++++++++++++++--- 5 files changed, 168 insertions(+), 19 deletions(-) diff --git a/y-server/.env.example b/y-server/.env.example index 5fdb899..21a23d9 100644 --- a/y-server/.env.example +++ b/y-server/.env.example @@ -1,4 +1,6 @@ SERVER_ADDRESS=127.0.0.1 SERVER_PORT=8080 +IMAGEMAGICK_BIN=/usr/bin/convert + DATABASE_URL=postgres://y_user:passw0rd!@127.0.0.1:5432/y diff --git a/y-server/Cargo.lock b/y-server/Cargo.lock index fa6860c..cba6834 100644 --- a/y-server/Cargo.lock +++ b/y-server/Cargo.lock @@ -394,7 +394,7 @@ dependencies = [ "log", "parking", "polling 2.8.0", - "rustix 0.37.27", + "rustix 0.37.25", "slab", "socket2 0.4.10", "waker-fn", @@ -658,6 +658,17 @@ dependencies = [ "libc", ] +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -1388,6 +1399,15 @@ dependencies = [ "serde", ] +[[package]] +name = "infer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" +dependencies = [ + "cfb", +] + [[package]] name = "inout" version = "0.1.3" @@ -2010,9 +2030,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.27" +version = "0.37.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" dependencies = [ "bitflags 1.3.2", "errno", @@ -3033,16 +3053,19 @@ dependencies = [ "actix-rt", "actix-web", "async-recursion", + "base64", "chrono", "cron", "dotenvy", "futures", "futures-util", + "infer", "log", "pbkdf2 0.12.2", "rand", "rand_core", "regex", + "rustix 0.37.25", "serde", "serde_json", "serde_with", diff --git a/y-server/Cargo.toml b/y-server/Cargo.toml index b06126d..1951a76 100644 --- a/y-server/Cargo.toml +++ b/y-server/Cargo.toml @@ -30,3 +30,6 @@ actix-rt = "2.9.0" cron = "0.12.0" simplelog = "0.12.1" log = "0.4.20" +rustix = "=0.37.25" +base64 = "0.21.7" +infer = "0.15.0" diff --git a/y-server/src/api/storage/storage_upload.rs b/y-server/src/api/storage/storage_upload.rs index a539cf1..0828d17 100644 --- a/y-server/src/api/storage/storage_upload.rs +++ b/y-server/src/api/storage/storage_upload.rs @@ -1,5 +1,6 @@ use log::*; -use std::{collections::HashMap, fs::OpenOptions, ops::Deref, path::Path}; +use std::{collections::HashMap, fs::OpenOptions, path::Path}; +use std::{env, fs}; use actix_multipart::Multipart; use actix_web::{ @@ -8,16 +9,18 @@ use actix_web::{ HttpResponse, Responder, }; use futures::StreamExt; -use log::*; use serde::{Deserialize, Serialize}; use std::io::Write; use std::time::Instant; +use crate::storage_entry::generate_entry_thumbnail; use crate::{ request::error, storage_endpoint::get_storage_endpoint, user::get_client_rights, util::RequestPool, }; +const MAX_FILE_SIZE_FOR_THUMNAIL_GENERATION: u64 = 50_000_000; + #[derive(Serialize)] struct StorageUploadOutput { skipped_files: Vec, @@ -41,6 +44,7 @@ async fn storage_upload( req: actix_web::HttpRequest, ) -> impl Responder { let now = Instant::now(); + let mut uploaded_files: Vec = Vec::new(); // Check the rights let client_rights = get_client_rights(&pool, &req).await; @@ -134,7 +138,7 @@ async fn storage_upload( // Id of the folder, to which the file's path (`filename`) is relative to. // This is `null` if the target destination of the whole upload is root, or an id, // if a user is uploading all the files into some (already existing) folder. - let mut parent_folder_id: Option = target_folder_id.clone(); + let mut parent_folder_id = target_folder_id.clone(); if file_relative_path.len() != 0 { // This file is not being uploaded to the relative root, it has a relative path! @@ -153,13 +157,13 @@ async fn storage_upload( // findings. No need to traverse the relative path again, just use the // folder_id we've found before. - Some(cached_path_id.unwrap().deref().clone()) + Some(cached_path_id.unwrap().clone()) } else { let file_relative_path_segments = file_relative_path.split("/"); - let mut folder_id: Option = target_folder_id.clone(); + let mut folder_id = target_folder_id.clone(); - let mut path_so_far = Vec::<&str>::new(); + let mut path_so_far: Vec<&str> = Vec::new(); // As long as it's true, we'll try to query the database for each path // segment in order to find that segment's actual folder id. @@ -269,8 +273,8 @@ async fn storage_upload( } else { // We created a row in the database, now let's actually write the // file onto the filesystem - let path = target_endpoint_base_path.join(file_id); - let file = OpenOptions::new().write(true).create_new(true).open(path); + let path = target_endpoint_base_path.join(&file_id); + let file = OpenOptions::new().write(true).create_new(true).open(&path); if let Ok(mut file) = file { while let Some(chunk) = field.next().await { @@ -297,6 +301,8 @@ async fn storage_upload( if commit_result.is_err() { return error("storage.upload.internal"); } + + uploaded_files.push(file_id); } else { // For some reason, we could not write the file onto the filesystem. // This is not critical, so we will just rollback the transaction @@ -324,7 +330,55 @@ async fn storage_upload( if cfg!(debug_assertions) { info!("storage/upload: {}ms", now.elapsed().as_millis()); + dbg!(&uploaded_files); } + // Generate thumbnails + std::thread::spawn(move || { + let convert_bin_path = env::var("IMAGEMAGICK_BIN"); + + if convert_bin_path.is_ok() { + if let Some(target_endpoint_artifacts_path) = &target_endpoint.artifacts_path { + for filesystem_id in uploaded_files { + let path = Path::new(&target_endpoint.base_path).join(&filesystem_id); + + let file_kind = infer::get_from_path(&path); + + if !file_kind.is_err() { + if let Some(file_kind) = file_kind.unwrap() { + let mime_type = file_kind.mime_type(); + + match mime_type { + "image/jpeg" | "image/png" | "image/gif" | "image/webp" + | "image/bmp" => { + let file_metadata = + fs::File::open(&path).unwrap().metadata().unwrap(); + + if file_metadata.len() <= MAX_FILE_SIZE_FOR_THUMNAIL_GENERATION + { + let generate_thumbnail_result = generate_entry_thumbnail( + &filesystem_id, + &target_endpoint.base_path.as_str(), + &target_endpoint_artifacts_path.as_str(), + ); + + if generate_thumbnail_result.is_err() { + error!( + "Failed to create a thumbnail for an uploaded image file. {}", + generate_thumbnail_result.unwrap_err() + ); + } + } + } + + _ => {} + } + } + } + } + } + } + }); + HttpResponse::Ok().json(web::Json(StorageUploadOutput { skipped_files })) } diff --git a/y-server/src/storage_entry.rs b/y-server/src/storage_entry.rs index c0ea380..b270dec 100644 --- a/y-server/src/storage_entry.rs +++ b/y-server/src/storage_entry.rs @@ -1,10 +1,10 @@ -use std::{collections::HashMap, fs::remove_file, path::Path}; +use std::{collections::HashMap, env, fs::remove_file, path::Path, process::Command}; use async_recursion::async_recursion; use serde::Serialize; use sqlx::FromRow; -use crate::util::RequestPool; +use crate::{storage_endpoint::get_storage_endpoint, util::RequestPool}; use log::*; #[derive(FromRow, Serialize)] @@ -250,17 +250,17 @@ pub async fn delete_entries( pool: &RequestPool, ) -> Result<(usize, usize), String> { // Find the endpoint so we know where the files we find will be stored - let target_endpoint = - sqlx::query_scalar::<_, String>("SELECT base_path FROM storage_endpoints WHERE id = $1") - .bind(endpoint_id) - .fetch_one(pool) - .await; + let target_endpoint = get_storage_endpoint(endpoint_id, pool).await; if target_endpoint.is_err() { return Err("Could not get target endpoint".to_string()); } - let endpoint_base_path = target_endpoint.unwrap(); + let target_endpoint = target_endpoint.unwrap(); + + let endpoint_base_path = target_endpoint.base_path; + let endpoint_artifacts_path = target_endpoint.artifacts_path; + let endpoint_files_path = Path::new(&endpoint_base_path); // Recursively find all the folders that reside inside target folders @@ -374,6 +374,25 @@ pub async fn delete_entries( fs_remove_result.unwrap_err() ); } + + if let Some(endpoint_artifacts_path) = &endpoint_artifacts_path { + let thumbnail_path = Path::new(&endpoint_artifacts_path) + .join(file_filesystem_id) + .with_extension("webp"); + + if thumbnail_path.exists() { + let fs_remove_thumbnail_result = remove_file(thumbnail_path); + + if fs_remove_thumbnail_result.is_err() { + error!( + "(storage entry -> delete entries) Could not remove a thumbnail from the filesystem. endpoint_id = {}. filesystem_id = {}. {}", + endpoint_id, + file_filesystem_id, + fs_remove_thumbnail_result.unwrap_err() + ); + } + } + } } return Ok((file_filesystem_ids.len(), all_folders.len())); @@ -383,3 +402,51 @@ pub async fn delete_entries( } } } + +pub fn generate_entry_thumbnail( + filesystem_id: &str, + endpoint_path: &str, + endpoint_artifacts_path: &str, +) -> Result<(), String> { + let convert_bin_path = env::var("IMAGEMAGICK_BIN"); + + if let Ok(convert_bin_path) = &convert_bin_path { + let convert_bin_path = Path::new(&convert_bin_path); + + if convert_bin_path.exists() { + let file_path = Path::new(endpoint_path).join(&filesystem_id); + let endpoint_artifacts_path = Path::new(endpoint_artifacts_path); + + let convert_result = Command::new(convert_bin_path) + .arg("-quality") + .arg("50") + .arg("-resize") + .arg("240x240") + .arg("-define") + .arg("webp:lossless=false") + .arg(file_path.to_str().unwrap()) + .arg( + endpoint_artifacts_path + .join(&filesystem_id) + .with_extension("webp") + .to_str() + .unwrap(), + ) + .output(); + + if let Ok(convert_result) = convert_result { + if convert_result.status.success() { + Ok(()) + } else { + Err("Could not generate a thumbnail for the storage entry".to_string()) + } + } else { + Err("Could not execute the convert command".to_string()) + } + } else { + Err("Could not find the convert binary".to_string()) + } + } else { + Ok(()) + } +} From 946c0575961e1c2d7b905f6b3a970a4eea491c8d Mon Sep 17 00:00:00 2001 From: max Date: Fri, 1 Mar 2024 19:09:01 +0200 Subject: [PATCH 04/24] add `entry-thumbnails` endpoint --- y-server/src/api/storage/mod.rs | 1 + .../api/storage/storage_entry_thumbnails.rs | 132 ++++++++++++++++++ y-server/src/main.rs | 1 + 3 files changed, 134 insertions(+) create mode 100644 y-server/src/api/storage/storage_entry_thumbnails.rs diff --git a/y-server/src/api/storage/mod.rs b/y-server/src/api/storage/mod.rs index 76af2b8..2c1c249 100644 --- a/y-server/src/api/storage/mod.rs +++ b/y-server/src/api/storage/mod.rs @@ -4,5 +4,6 @@ pub mod storage_download; pub mod storage_download_zip; pub mod storage_endpoints; pub mod storage_entries; +pub mod storage_entry_thumbnails; pub mod storage_get_folder_path; pub mod storage_upload; diff --git a/y-server/src/api/storage/storage_entry_thumbnails.rs b/y-server/src/api/storage/storage_entry_thumbnails.rs new file mode 100644 index 0000000..d22b0d5 --- /dev/null +++ b/y-server/src/api/storage/storage_entry_thumbnails.rs @@ -0,0 +1,132 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use actix_web::web::Query; +use actix_web::{get, web, HttpResponse, Responder}; +use base64::prelude::*; +use log::*; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use sqlx::prelude::FromRow; + +use crate::request::error; + +use crate::storage_endpoint::get_storage_endpoint; +use crate::user::get_client_rights; +use crate::util::RequestPool; + +const MAX_ENTRIES_PER_REEQUEST: u32 = 200; + +#[derive(FromRow)] +struct EntryRow { + id: i64, + filesystem_id: String, +} + +#[derive(Deserialize)] +struct QueryParams { + endpoint_id: i32, + entry_ids: String, +} + +#[derive(Serialize)] +struct StorageEntryThumbnaisOutput { + thumbnails: HashMap, +} + +#[get("/entry-thumbnails")] +async fn storage_entry_thumbnails( + pool: web::Data, + query: Query, + req: actix_web::HttpRequest, +) -> impl Responder { + // TODO? should we cache the thumbnails somehow? + + let client_rights = get_client_rights(&pool, &req).await; + + let action_allowed = client_rights + .iter() + .find(|right| right.right_name.eq("storage_list")) + .is_some(); + + if !action_allowed { + return error("storage.entry_thumbnails.unauthorized"); + } + + let endpoint_id = query.endpoint_id; + + // TODO? cache endpoints? + let endpoint = get_storage_endpoint(endpoint_id, &**pool).await; + + if endpoint.is_err() { + return error("storage.entry_thumbnails.endpoint_not_found"); + } + + let endpoint = endpoint.unwrap(); + + if endpoint.artifacts_path.is_none() { + return error("storage.entry_thumbnails.artifacts_disabled"); + } + + let endpoint_artifacts_path = endpoint.artifacts_path.unwrap(); + + let entry_ids_regex = Regex::new(r"^[0-9\,]+$").unwrap(); + if !entry_ids_regex.is_match(&query.entry_ids) { + return error("storage.entry_thumbnails.invalid_entry_ids_param"); + } + + let entry_ids_split = query.entry_ids.split(","); + let mut entries_count = 0; + + for entry_id in entry_ids_split { + if entry_id.parse::().is_err() { + return error("storage.entry_thumbnails.invalid_entry_ids_param"); + } + + if entries_count > MAX_ENTRIES_PER_REEQUEST { + return error("storage.entry_thumbnails.too_many_entries_requested"); + } + + entries_count += 1; + } + + let mut thumbnails: HashMap = HashMap::new(); + + let entries = sqlx::query_as::<_, EntryRow>( + format!( + "SELECT id, filesystem_id FROM storage_files WHERE endpoint_id = $1 AND id IN ({})", + query.entry_ids + ) + .as_str(), + ) + .bind(endpoint_id) + .fetch_all(&**pool) + .await; + + match entries { + Ok(entries) => { + for entry in entries { + let thumbnail_path = Path::new(endpoint_artifacts_path.as_str()) + .join(entry.filesystem_id) + .with_extension("webp"); + + if thumbnail_path.exists() { + let thumbnail = fs::read(thumbnail_path); + + if thumbnail.is_ok() { + let thumbnail_base64 = BASE64_STANDARD.encode(thumbnail.unwrap()); + + thumbnails.insert(entry.id.to_string(), thumbnail_base64); + } + } + } + + return HttpResponse::Ok().json(StorageEntryThumbnaisOutput { thumbnails }); + } + Err(e) => { + error!("{:?}", e); + return error("storage.entry_thumbnails.internal"); + } + } +} diff --git a/y-server/src/main.rs b/y-server/src/main.rs index 27c3e12..53a26bf 100644 --- a/y-server/src/main.rs +++ b/y-server/src/main.rs @@ -149,6 +149,7 @@ async fn main() -> std::io::Result<()> { .service(crate::api::storage::storage_create_folder::storage_create_folder) .service(crate::api::storage::storage_get_folder_path::storage_get_folder_path) .service(crate::api::storage::storage_delete_entries::storage_delete_entries) + .service(crate::api::storage::storage_entry_thumbnails::storage_entry_thumbnails) ) .service( web::scope("/api/admin") From 5d87069fb3161326ee14bf4c44576241a75d40bb Mon Sep 17 00:00:00 2001 From: max Date: Fri, 1 Mar 2024 19:33:04 +0200 Subject: [PATCH 05/24] add thumbnail loading logic --- y-web/i18n/en.json | 8 ++ .../use-file-explorer-thumbnails.ts | 134 ++++++++++++++++++ .../components/storage-entry.tsx | 89 ++++++++++++ .../pages/file-explorer/file-explorer.less | 24 ++++ .../storage/pages/file-explorer/index.tsx | 95 +++++-------- .../storage-entry/storage-entry.api.ts | 20 +++ .../storage-entry/storage-entry.codecs.ts | 4 + 7 files changed, 314 insertions(+), 60 deletions(-) create mode 100644 y-web/src/modules/storage/file-explorer/use-file-explorer-thumbnails.ts create mode 100644 y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx diff --git a/y-web/i18n/en.json b/y-web/i18n/en.json index e31569f..4694b1b 100644 --- a/y-web/i18n/en.json +++ b/y-web/i18n/en.json @@ -219,6 +219,14 @@ "unauthorized": "Permission denied", "file_id_required": "file_id parameter is required", "internal": "Other" + }, + "entry_thumbnails": { + "unauthorized": "Permission denied", + "internal": "Other", + "endpoint_not_found": "Target endpoint not found", + "artifacts_disabled": "Artifacts are disabled for this endpoint", + "invalid_entry_ids_param": "Invalid entry_ids parameter", + "too_many_entries_requested": "Too many entries requested" } } } diff --git a/y-web/src/modules/storage/file-explorer/use-file-explorer-thumbnails.ts b/y-web/src/modules/storage/file-explorer/use-file-explorer-thumbnails.ts new file mode 100644 index 0000000..39327ff --- /dev/null +++ b/y-web/src/modules/storage/file-explorer/use-file-explorer-thumbnails.ts @@ -0,0 +1,134 @@ +import { + createEffect, + createMemo, + createSignal, + on, + onCleanup, + onMount, +} from "solid-js" + +import { genericErrorToast } from "@/app/core/util/toast-utils" +import { storageEntryThumbnails } from "@/modules/storage/storage-entry/storage-entry.api" +import { IStorageEntry } from "@/modules/storage/storage-entry/storage-entry.codecs" + +const THUMBNAILS_PER_LOAD_REQUEST = 200 +const SCROLL_THUMBNAILS_LOAD_DEBOUNCE_MS = 200 + +export type UseFileExplorerThumbnailsProps = { + endpointId: () => number + + browserContentsRef: () => HTMLDivElement + entryRefs: () => HTMLDivElement[] + + entries: () => IStorageEntry[] +} + +export const useFileExplorerThumbnails = ( + props: UseFileExplorerThumbnailsProps +) => { + const [thumbnails, setThumbnails] = createSignal>({}) + const [lastThumbnailIndex, setLastThumbnailIndex] = createSignal(0) + + const fileIds = createMemo(() => + props + .entries() + .filter((entry) => entry.entry_type === "file") + .map((entry) => entry.id) + ) + + const getNextThumbnails = async () => { + const fromIndex = lastThumbnailIndex() + const toIndex = fromIndex + THUMBNAILS_PER_LOAD_REQUEST + + if (fileIds().length === 0) { + setLastThumbnailIndex(toIndex) + return + } + + // Don't load thumbnails that we have already loaded + const entryIds = fileIds() + .slice(fromIndex, toIndex) + .filter((id) => !thumbnails()[id]) + + if (entryIds.length === 0) { + setLastThumbnailIndex(toIndex) + return + } + + const newThumbnailsResponse = await storageEntryThumbnails({ + endpointId: props.endpointId(), + entryIds, + }).catch((error) => { + genericErrorToast(error) + }) + + setThumbnails((previous) => ({ + ...previous, + ...newThumbnailsResponse?.thumbnails, + })) + + setLastThumbnailIndex(toIndex) + } + + /** + * @returns {Promise} Whether or not new thumbnails were loaded. If `false`, we have already loaded all of them for this view (scroll position). + */ + const checkAndLoad = async () => { + // eslint-disable-next-line sonarjs/no-empty-collection + const elementWithLastThumbnail = props.entryRefs()[lastThumbnailIndex()] + + if (elementWithLastThumbnail) { + const elementRect = elementWithLastThumbnail.getBoundingClientRect() + + if (elementRect.top < window.innerHeight) { + await getNextThumbnails() + + return true + } + } + + return false + } + + onMount(() => { + let debounce = 0 + + const handler = () => { + clearTimeout(debounce) + + debounce = setTimeout(async () => { + let shouldLoadMore = true + + while (shouldLoadMore) { + // eslint-disable-next-line no-await-in-loop + shouldLoadMore = await checkAndLoad() + } + }, SCROLL_THUMBNAILS_LOAD_DEBOUNCE_MS) + } + + props.browserContentsRef().addEventListener("scroll", handler) + + onCleanup(() => { + props.browserContentsRef().removeEventListener("scroll", handler) + }) + }) + + createEffect( + on(fileIds, async () => { + setLastThumbnailIndex(0) + + if (fileIds().length > 0) { + let shouldLoadMore = true + + while (shouldLoadMore) { + // eslint-disable-next-line no-await-in-loop + shouldLoadMore = await checkAndLoad() + } + } + }) + ) + + return { + thumbnails, + } +} diff --git a/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx new file mode 100644 index 0000000..0d538ca --- /dev/null +++ b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx @@ -0,0 +1,89 @@ +import { Component, Show, createMemo } from "solid-js" + +import { Checkbox } from "@/app/components/common/checkbox/checkbox" +import { Icon } from "@/app/components/common/icon/icon" +import { IStorageEntry } from "@/modules/storage/storage-entry/storage-entry.codecs" + +export type StorageEntryProps = { + ref?: (ref: HTMLDivElement) => void + + entry: IStorageEntry + + selected?: boolean + thumbnails?: Record + + onNavigateToFolder: (folderId: number) => void + onOpenContextMenu?: (event: MouseEvent) => void + onSelect?: (event: MouseEvent | undefined) => void +} + +export const StorageEntry: Component = (props) => { + const thumbnail = createMemo(() => props.thumbnails?.[props.entry.id]) + + return ( + // TODO: Should be a clickable + + ) +} diff --git a/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx index e97013e..86ddeea 100644 --- a/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx +++ b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx @@ -71,6 +71,7 @@ export const StorageEntry: Component = (props) => { } > diff --git a/y-web/src/modules/storage/pages/file-explorer/file-explorer.less b/y-web/src/modules/storage/pages/file-explorer/file-explorer.less index 0d9c319..cfc925b 100644 --- a/y-web/src/modules/storage/pages/file-explorer/file-explorer.less +++ b/y-web/src/modules/storage/pages/file-explorer/file-explorer.less @@ -22,9 +22,11 @@ align-items: center; gap: 1em; + min-height: 30px; + flex-shrink: 0; - padding: 1em; + padding: 0.5em 1em; } > .browser-contents { diff --git a/y-web/src/modules/storage/pages/file-explorer/index.tsx b/y-web/src/modules/storage/pages/file-explorer/index.tsx index 229a4f6..98ee65b 100644 --- a/y-web/src/modules/storage/pages/file-explorer/index.tsx +++ b/y-web/src/modules/storage/pages/file-explorer/index.tsx @@ -35,11 +35,13 @@ import { deleteStorageEntries, downloadStorageFile, downloadStorageFilesZip, + moveStorageEntries, } from "@/modules/storage/storage-entry/storage-entry.api" import { TUploadEntries } from "@/modules/storage/storage-entry/storage-entry.codecs" import { FileWithPath } from "@/modules/storage/upload" import { FileExplorerPath } from "./components/file-explorer-path" +import { FileExplorerSelectionInfo } from "./components/file-explorer-selection-info" import { FileExplorerUploadStatusToast } from "./components/file-explorer-upload-status-toast" import { NewFolderEntry } from "./components/new-folder-entry" import { StorageEntry } from "./components/storage-entry" @@ -100,6 +102,12 @@ const FileExplorerPage: Component = () => { contextMenuProps: generalContextMenuProps, } = useContextMenu() + const { + open: openSelectionContextMenu, + close: closeSelectionContextMenu, + contextMenuProps: selectionContextMenuProps, + } = useContextMenu() + const [uploadStatus, setUploadStatus] = createStore({ numberOfFiles: 0, percentageUploaded: 0, @@ -109,6 +117,7 @@ const FileExplorerPage: Component = () => { const [folderCreationInitiated, setFolderCreationInitiated] = createSignal(false) + const $moveEntries = createMutation(moveStorageEntries) const $deleteEntries = createMutation(deleteStorageEntries) const $createFolder = createMutation(createStorageFolder) @@ -157,6 +166,25 @@ const FileExplorerPage: Component = () => { ) } + const moveEntries = (entryIds: number[]) => { + if (entryIds.length === 0) return + + $moveEntries.mutate( + { + endpointId: Number.parseInt(params.endpointId as string, 10), + entryIds, + targetFolderId: folderId(), + }, + { + onSuccess: () => { + resetSelection() + void invalidateEntries() + }, + onError: (error) => genericErrorToast(error), + } + ) + } + createEffect(() => { if (uploadStatus.numberOfFiles === 0) { window.removeEventListener("beforeunload", closeTabConfirmation) @@ -312,6 +340,12 @@ const FileExplorerPage: Component = () => { setSearchParams({ folderId: newFolderId }) } /> + 0}> + +
{ New Folder + 0}> +
+ + { + moveEntries([...selectedEntries()]) + closeGeneralContextMenu() + }} + > + Move {selectedEntries().size} entries here + + +
{ + + + { + setSelectedEntries(new Set()) + closeSelectionContextMenu() + }} + > + Remove selection + + +
+ + { + moveEntries([...selectedEntries()]) + closeSelectionContextMenu() + }} + > + Move here + + + + { closeEntryContextMenu() }} > - Download {selectedEntries().size} entries + Download selected entries { closeEntryContextMenu() }} > - Delete {selectedEntries().size} entries + Delete selected entries } diff --git a/y-web/src/modules/storage/storage-entry/storage-entry.api.ts b/y-web/src/modules/storage/storage-entry/storage-entry.api.ts index 6534d87..591bfef 100644 --- a/y-web/src/modules/storage/storage-entry/storage-entry.api.ts +++ b/y-web/src/modules/storage/storage-entry/storage-entry.api.ts @@ -10,6 +10,7 @@ import { downloadResponseBlob } from "./storage-entry.util" export const apiStorageFolderPath = "/storage/folder-path" as const export const apiStorageEntries = "/storage/entries" as const +export const apiStorageMoveEntries = "/storage/move-entries" as const export const apiStorageEntryThumbnails = "/storage/entry-thumbnails" as const // TODO move to POST /folder @@ -91,6 +92,22 @@ export const createStorageFolder = async (input: CreateStorageFolderInput) => { }).then((data) => TCreateStorageFolder.parse(data)) } +export type MoveStorageEntriesInput = { + endpointId: number + entryIds: number[] + targetFolderId: number | undefined +} + +export const moveStorageEntries = async (input: MoveStorageEntriesInput) => { + return post(apiStorageMoveEntries, { + body: { + endpoint_id: input.endpointId, + entry_ids: input.entryIds, + target_folder_id: input.targetFolderId, + }, + }) +} + export type DeleteStorageEntriesInput = { endpointId: number folderIds: number[] From a463baac7874f37952a0e8ef9414fcaabcd7f63f Mon Sep 17 00:00:00 2001 From: max Date: Sun, 3 Mar 2024 21:15:52 +0200 Subject: [PATCH 11/24] add storage entry rename logic --- y-server/src/api/storage/mod.rs | 1 + .../src/api/storage/storage_rename_entry.rs | 44 ++++++++++ y-server/src/main.rs | 1 + y-server/src/right.rs | 6 ++ y-server/src/storage_entry.rs | 37 +++++++++ y-web/i18n/en.json | 8 ++ .../components/storage-entry.tsx | 63 +++++++++++++-- .../storage/pages/file-explorer/index.tsx | 80 +++++++++++++++++-- .../storage-entry/storage-entry.api.ts | 19 ++++- 9 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 y-server/src/api/storage/storage_rename_entry.rs diff --git a/y-server/src/api/storage/mod.rs b/y-server/src/api/storage/mod.rs index 4216913..7380c47 100644 --- a/y-server/src/api/storage/mod.rs +++ b/y-server/src/api/storage/mod.rs @@ -7,4 +7,5 @@ pub mod storage_entries; pub mod storage_entry_thumbnails; pub mod storage_get_folder_path; pub mod storage_move_entries; +pub mod storage_rename_entry; pub mod storage_upload; diff --git a/y-server/src/api/storage/storage_rename_entry.rs b/y-server/src/api/storage/storage_rename_entry.rs new file mode 100644 index 0000000..baf56ba --- /dev/null +++ b/y-server/src/api/storage/storage_rename_entry.rs @@ -0,0 +1,44 @@ +use actix_web::{patch, web, HttpResponse, Responder}; +use serde::Deserialize; + +use crate::request::error; +use crate::storage_entry::rename_entry; +use crate::user::get_client_rights; +use crate::util::RequestPool; + +#[derive(Deserialize)] +struct StorageRenameEntryInput { + endpoint_id: i32, + entry_id: i64, + name: String, +} + +#[patch("/rename-entry")] +async fn storage_rename_entry( + pool: web::Data, + form: web::Json, + req: actix_web::HttpRequest, +) -> impl Responder { + let client_rights = get_client_rights(&pool, &req).await; + + let action_allowed = client_rights + .iter() + .find(|right| right.right_name.eq("storage_rename")) + .is_some(); + + if !action_allowed { + return error("storage.rename.unauthorized"); + } + + let form = form.into_inner(); + let endpoint_id = form.endpoint_id; + let entry_id = form.entry_id; + let name = form.name; + + let result = rename_entry(endpoint_id, entry_id, name.as_str(), &pool).await; + + match result { + Ok(_) => HttpResponse::Ok().body("{}"), + Err(_) => error("storage.rename.other"), + } +} diff --git a/y-server/src/main.rs b/y-server/src/main.rs index b152d03..dc3ac71 100644 --- a/y-server/src/main.rs +++ b/y-server/src/main.rs @@ -151,6 +151,7 @@ async fn main() -> std::io::Result<()> { .service(crate::api::storage::storage_delete_entries::storage_delete_entries) .service(crate::api::storage::storage_entry_thumbnails::storage_entry_thumbnails) .service(crate::api::storage::storage_move_entries::storage_move_entries) + .service(crate::api::storage::storage_rename_entry::storage_rename_entry) ) .service( web::scope("/api/admin") diff --git a/y-server/src/right.rs b/y-server/src/right.rs index cfd32f9..3c38648 100644 --- a/y-server/src/right.rs +++ b/y-server/src/right.rs @@ -153,6 +153,12 @@ pub fn get_right_categories() -> Vec { tags: vec![], feature: Some("storage"), }, + Right { + name: "storage_rename", + options: vec![], + tags: vec![], + feature: Some("storage"), + }, Right { name: "storage_delete", options: vec![], diff --git a/y-server/src/storage_entry.rs b/y-server/src/storage_entry.rs index 87279a5..62011e9 100644 --- a/y-server/src/storage_entry.rs +++ b/y-server/src/storage_entry.rs @@ -455,6 +455,43 @@ pub async fn move_entries( Ok(()) } +pub async fn rename_entry( + endpoint_id: i32, + entry_id: i64, + new_name: &str, + pool: &RequestPool, +) -> Result<(), String> { + let mut transaction = pool.begin().await.unwrap(); + + let file_result = + sqlx::query("UPDATE storage_files SET name = $1 WHERE id = $2 AND endpoint_id = $3") + .bind(new_name) + .bind(entry_id) + .bind(endpoint_id) + .execute(&mut *transaction) + .await; + + let folder_result = + sqlx::query("UPDATE storage_folders SET name = $1 WHERE id = $2 AND endpoint_id = $3") + .bind(new_name) + .bind(entry_id) + .bind(endpoint_id) + .execute(&mut *transaction) + .await; + + if file_result.is_err() || folder_result.is_err() { + return Err("Could not rename storage entry".to_string()); + } + + let transaction_result = transaction.commit().await; + + if transaction_result.is_err() { + return Err("Could not commit the rename transaction".to_string()); + } + + Ok(()) +} + pub fn generate_image_entry_thumbnail( filesystem_id: &str, endpoint_path: &str, diff --git a/y-web/i18n/en.json b/y-web/i18n/en.json index cd68181..1279701 100644 --- a/y-web/i18n/en.json +++ b/y-web/i18n/en.json @@ -59,6 +59,10 @@ "name": "List files", "description": "A general right to see files on a storage endpoint. This is a \"first check\" right, meaning that it is required to list files on any endpoint. Users that do not have this right are guranteed to not be able to see files on any endpoint, while users that do may still be restricted by other permissions checks, such as per endpoint, per folder, or per file." }, + "storage_rename": { + "name": "Rename files", + "description": "Allows renaming files and folders." + }, "storage_move": { "name": "Move files", "description": "Allows moving (cut and paste) files within an endpoint." @@ -223,6 +227,10 @@ "unauthorized": "Permission denied", "internal": "Other" }, + "rename": { + "unauthorized": "Permission denied", + "internal": "Other" + }, "download": { "unauthorized": "Permission denied", "file_id_required": "file_id parameter is required", diff --git a/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx index 86ddeea..ccc8b26 100644 --- a/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx +++ b/y-web/src/modules/storage/pages/file-explorer/components/storage-entry.tsx @@ -1,4 +1,11 @@ -import { Component, Show, createMemo } from "solid-js" +import { + Component, + Show, + createEffect, + createMemo, + onCleanup, + onMount, +} from "solid-js" import { Checkbox } from "@/app/components/common/checkbox/checkbox" import { Icon } from "@/app/components/common/icon/icon" @@ -12,15 +19,45 @@ export type StorageEntryProps = { selected?: boolean temporarySelected?: boolean thumbnails?: Record + isRenaming?: boolean onNavigateToFolder: (folderId: number) => void onOpenContextMenu?: (event: MouseEvent) => void onSelect?: (event: MouseEvent | undefined) => void + onRename?: (newName: string) => void + onCancelRename?: () => void } export const StorageEntry: Component = (props) => { + let nameFieldRef: HTMLInputElement + const thumbnail = createMemo(() => props.thumbnails?.[props.entry.id]) + createEffect(() => { + if (props.isRenaming) { + nameFieldRef.value = props.entry.name + nameFieldRef.select() + } + }) + + onMount(() => { + const handler = (event: KeyboardEvent) => { + if (event.key === "Enter") { + props.onRename?.(nameFieldRef!.value) + } + + if (event.key === "Escape") { + props.onCancelRename?.() + } + } + + nameFieldRef.addEventListener("keyup", handler) + + onCleanup(() => { + nameFieldRef.removeEventListener("keyup", handler) + }) + }) + return ( // TODO: Should be a clickable