From 4164a7d674d5706b4d4ac892d81a7bb9bc618490 Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Tue, 8 Oct 2024 16:55:36 +0200 Subject: [PATCH 1/7] WIP move from proc-macro to build.rs --- Cargo.toml | 2 +- README.md | 9 ++ examples/test/src/main.rs | 4 +- memory-serve-macros/Cargo.toml | 22 --- memory-serve-macros/README.md | 1 - memory-serve-macros/src/asset.rs | 31 ---- memory-serve-macros/src/lib.rs | 62 -------- memory-serve/Cargo.toml | 13 +- memory-serve/build.rs | 28 ++++ memory-serve/src/asset.rs | 3 +- memory-serve/src/lib.rs | 148 +++++++++--------- .../src => memory-serve}/utils.rs | 48 ++++-- 12 files changed, 163 insertions(+), 208 deletions(-) delete mode 100644 memory-serve-macros/Cargo.toml delete mode 120000 memory-serve-macros/README.md delete mode 100644 memory-serve-macros/src/asset.rs delete mode 100644 memory-serve-macros/src/lib.rs create mode 100644 memory-serve/build.rs rename {memory-serve-macros/src => memory-serve}/utils.rs (81%) diff --git a/Cargo.toml b/Cargo.toml index 73f5ae7..d1052fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["memory-serve", "memory-serve-macros", "examples/test"] +members = ["memory-serve", "examples/test"] resolver = "2" [workspace.package] diff --git a/README.md b/README.md index 8e8d18b..71574b7 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,15 @@ the following configuration methods: See [`Cache control`](#cache-control) for the cache control options. +## Asset path + +The `load_assets!` macro accepts relative (from `CARGO_MANIFEST_DIR`) or absolute paths. When the environment variable `MEMORY_SERVE_ROOT` +is set, the path is constructed relative from the path provided by `MEMORY_SERVE_ROOT`. + +## Build cache + + + ## Logging During compilation, problems that occur with the inclusion or compression diff --git a/examples/test/src/main.rs b/examples/test/src/main.rs index 19d7b2a..ac00377 100644 --- a/examples/test/src/main.rs +++ b/examples/test/src/main.rs @@ -1,5 +1,5 @@ use axum::{response::Html, routing::get, Router}; -use memory_serve::{load_assets, MemoryServe}; +use memory_serve::MemoryServe; use std::net::SocketAddr; use tracing::info; @@ -7,7 +7,7 @@ use tracing::info; async fn main() { tracing_subscriber::fmt().init(); - let memory_router = MemoryServe::new(load_assets!("../../static")) + let memory_router = MemoryServe::new() .index_file(Some("/index.html")) .into_router(); diff --git a/memory-serve-macros/Cargo.toml b/memory-serve-macros/Cargo.toml deleted file mode 100644 index 9d5a3dc..0000000 --- a/memory-serve-macros/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "memory-serve-macros" -description = "Macro for memory-serve" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -publish.workspace = true - -[lib] -proc-macro = true - -[dependencies] -brotli = "6.0" -mime_guess = "2.0" -proc-macro2 = "1.0" -sha256 = "1.4" -walkdir = "2" -tracing = "0.1" -quote = { version = "1.0", default-features = false } -syn = { version = "2.0", default-features = false } -tracing-subscriber = { version = "0.3", features = ["fmt", "ansi"], default-features = false } diff --git a/memory-serve-macros/README.md b/memory-serve-macros/README.md deleted file mode 120000 index 32d46ee..0000000 --- a/memory-serve-macros/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/memory-serve-macros/src/asset.rs b/memory-serve-macros/src/asset.rs deleted file mode 100644 index 88d853c..0000000 --- a/memory-serve-macros/src/asset.rs +++ /dev/null @@ -1,31 +0,0 @@ -use syn::LitByteStr; - -/// Internal data structure -pub(crate) struct Asset { - pub(crate) route: String, - pub(crate) path: String, - pub(crate) etag: String, - pub(crate) content_type: String, - pub(crate) bytes: LitByteStr, - pub(crate) brotli_bytes: LitByteStr, -} - -impl PartialEq for Asset { - fn eq(&self, other: &Self) -> bool { - self.route == other.route - } -} - -impl Eq for Asset {} - -impl PartialOrd for Asset { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Asset { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.route.cmp(&other.route) - } -} diff --git a/memory-serve-macros/src/lib.rs b/memory-serve-macros/src/lib.rs deleted file mode 100644 index b8ce56a..0000000 --- a/memory-serve-macros/src/lib.rs +++ /dev/null @@ -1,62 +0,0 @@ -use proc_macro::TokenStream; -use std::{env, path::Path}; -use utils::list_assets; - -mod asset; -mod utils; - -use crate::asset::Asset; - -#[proc_macro] -pub fn load_assets(input: TokenStream) -> TokenStream { - let input = input.to_string(); - let input = input.trim_matches('"'); - let mut asset_path = Path::new(&input).to_path_buf(); - - // skip if a subscriber is already registered (for instance by rust_analyzer) - let _ = tracing_subscriber::fmt() - .without_time() - .with_target(false) - .try_init(); - - if asset_path.is_relative() { - if let Ok(root_dir) = env::var("MEMORY_SERVE_ROOT") { - asset_path = Path::new(&root_dir).join(asset_path); - } else if let Ok(crate_dir) = env::var("CARGO_MANIFEST_DIR") { - asset_path = Path::new(&crate_dir).join(asset_path); - } else { - panic!("Relative path provided but CARGO_MANIFEST_DIR environment variable not set"); - } - } - - asset_path = asset_path - .canonicalize() - .expect("Could not canonicalize the provided path"); - - if !asset_path.exists() { - panic!("The path {:?} does not exists!", asset_path); - } - - let files: Vec = list_assets(&asset_path); - - let route = files.iter().map(|a| &a.route); - let path = files.iter().map(|a| &a.path); - let content_type = files.iter().map(|a| &a.content_type); - let etag = files.iter().map(|a| &a.etag); - let bytes = files.iter().map(|a| &a.bytes); - let brotli_bytes = files.iter().map(|a| &a.brotli_bytes); - - quote::quote! { - &[ - #(memory_serve::Asset { - route: #route, - path: #path, - content_type: #content_type, - etag: #etag, - bytes: #bytes, - brotli_bytes: #brotli_bytes, - }),* - ] - } - .into() -} diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index cea4cb6..c408278 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -11,9 +11,20 @@ description.workspace = true brotli = "6.0" flate2 = "1.0" axum = { version = "0.7", default-features = false } -memory-serve-macros = { version = "0.6", path = "../memory-serve-macros" } tracing = "0.1" sha256 = "1.4" +postcard = { version = "1.0", features = ["use-std"] } +serde = "1.0" + +[build-dependencies] +sha256 = "1.4" +brotli = "6.0" +tracing = "0.1" +mime_guess = "2.0" +walkdir = "2" +tracing-subscriber = { version = "0.3", features = ["fmt", "ansi"], default-features = false } +postcard = { version = "1.0", features = ["use-std"] } +serde = "1.0" [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/memory-serve/build.rs b/memory-serve/build.rs new file mode 100644 index 0000000..c5cc855 --- /dev/null +++ b/memory-serve/build.rs @@ -0,0 +1,28 @@ +use std::path::Path; + +mod utils; + +const ASSET_FILE: &str = "memory_serve_assets.bin"; + +fn main() { + let out_dir: String = std::env::var("OUT_DIR").unwrap(); + + let Ok(memory_serve_dir) = std::env::var("ASSET_DIR") else { + panic!("Please specify the ASSET_DIR environment variable."); + }; + + let path = Path::new(&memory_serve_dir); + let path = path.canonicalize().expect("Unable to canonicalize the path specified by ASSET_DIR."); + + if !path.exists() { + panic!("The path {memory_serve_dir} specified by ASSET_DIR does not exists!"); + } + + let target = Path::new(&out_dir).join(ASSET_FILE); + + let assets = utils::list_assets(&path); + let data = postcard::to_allocvec(&assets).expect("Unable to serialize memory-serve assets."); + std::fs::write(target, data).expect("Unable to write memory-serve asset file."); + + println!("cargo::rerun-if-changed={memory_serve_dir}"); +} \ No newline at end of file diff --git a/memory-serve/src/asset.rs b/memory-serve/src/asset.rs index 74221cf..0f3e4b3 100644 --- a/memory-serve/src/asset.rs +++ b/memory-serve/src/asset.rs @@ -5,6 +5,7 @@ use axum::{ }, response::{IntoResponse, Response}, }; +use serde::{Deserialize, Serialize}; use tracing::{debug, error}; use crate::{ @@ -33,7 +34,7 @@ const GZIP_ENCODING: &str = "gzip"; const GZIP_HEADER: (HeaderName, HeaderValue) = (CONTENT_ENCODING, HeaderValue::from_static(GZIP_ENCODING)); -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct Asset { pub route: &'static str, pub path: &'static str, diff --git a/memory-serve/src/lib.rs b/memory-serve/src/lib.rs index 1bc6e27..a63884a 100644 --- a/memory-serve/src/lib.rs +++ b/memory-serve/src/lib.rs @@ -14,11 +14,6 @@ use crate::util::{compress_gzip, decompress_brotli}; pub use crate::{asset::Asset, cache_control::CacheControl}; -/// Macro to load a directory of static files into the resulting binary -/// (possibly compressed) and create a data structure of (meta)data -/// as an input for [`MemoryServe::new`] -pub use memory_serve_macros::load_assets; - #[derive(Debug, Clone, Copy)] struct ServeOptions { index_file: Option<&'static str>, @@ -60,12 +55,15 @@ pub struct MemoryServe { } impl MemoryServe { - /// Initiate a `MemoryServe` instance, takes the output of `load_assets!` - /// as an argument. `load_assets!` takes a directory name relative from - /// the project root. - pub fn new(assets: &'static [Asset]) -> Self { + /// Initiate a `MemoryServe` instance, takes the contents of `memory_serve_assets.bin` + /// created at build time. + /// Specify which asset directory to include using the environment variable `ASSET_DIR`. + pub fn new() -> Self { + let asset_bytes = include_bytes!("../memory_serve_assets.bin"); + let assets: Vec = postcard::from_bytes(asset_bytes).expect("Could not deserialize assets"); + Self { - assets, + assets: assets.leak(), ..Default::default() } } @@ -246,7 +244,7 @@ impl MemoryServe { #[cfg(test)] mod tests { - use crate::{self as memory_serve, load_assets, Asset, CacheControl, MemoryServe}; + use crate::{CacheControl, MemoryServe}; use axum::{ body::Body, http::{ @@ -283,58 +281,58 @@ mod tests { headers.get(name).unwrap().to_str().unwrap() } - #[test] - fn test_load_assets() { - let assets: &'static [Asset] = load_assets!("../static"); - let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); - let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); - let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); - - assert_eq!( - routes, - [ - "/about.html", - "/assets/icon.jpg", - "/assets/index.css", - "/assets/index.js", - "/assets/stars.svg", - "/blog/index.html", - "/index.html" - ] - ); - assert_eq!( - content_types, - [ - "text/html", - "image/jpeg", - "text/css", - "text/javascript", - "image/svg+xml", - "text/html", - "text/html" - ] - ); - if cfg!(debug_assertions) { - assert_eq!(etags, ["", "", "", "", "", "", ""]); - } else { - assert_eq!( - etags, - [ - "56a0dcb83ec56b6c967966a1c06c7b1392e261069d0844aa4e910ca5c1e8cf58", - "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", - "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", - "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", - "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", - "89e9873a8e49f962fe83ad2bfe6ac9b21ef7c1b4040b99c34eb783dccbadebc5", - "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" - ] - ); - } - } + // #[test] + // fn test_load_assets() { + // let assets: &'static [Asset] = load_assets!("../static"); + // let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); + // let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); + // let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); + + // assert_eq!( + // routes, + // [ + // "/about.html", + // "/assets/icon.jpg", + // "/assets/index.css", + // "/assets/index.js", + // "/assets/stars.svg", + // "/blog/index.html", + // "/index.html" + // ] + // ); + // assert_eq!( + // content_types, + // [ + // "text/html", + // "image/jpeg", + // "text/css", + // "text/javascript", + // "image/svg+xml", + // "text/html", + // "text/html" + // ] + // ); + // if cfg!(debug_assertions) { + // assert_eq!(etags, ["", "", "", "", "", "", ""]); + // } else { + // assert_eq!( + // etags, + // [ + // "56a0dcb83ec56b6c967966a1c06c7b1392e261069d0844aa4e910ca5c1e8cf58", + // "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", + // "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", + // "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", + // "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", + // "89e9873a8e49f962fe83ad2bfe6ac9b21ef7c1b4040b99c34eb783dccbadebc5", + // "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" + // ] + // ); + // } + // } #[tokio::test] async fn if_none_match_handling() { - let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); + let memory_router = MemoryServe::new().into_router(); let (code, headers) = get(memory_router.clone(), "/index.html", "accept", "text/html").await; let etag: &str = headers.get(header::ETAG).unwrap().to_str().unwrap(); @@ -354,7 +352,7 @@ mod tests { #[tokio::test] async fn brotli_compression() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .enable_brotli(true) .into_router(); let (code, headers) = get( @@ -372,7 +370,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 178); // check disable compression - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .enable_brotli(false) .into_router(); let (code, headers) = get( @@ -390,7 +388,7 @@ mod tests { #[tokio::test] async fn gzip_compression() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .enable_gzip(true) .into_router(); let (code, headers) = get( @@ -408,7 +406,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 274); // check disable compression - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .enable_gzip(false) .into_router(); let (code, headers) = get( @@ -426,14 +424,14 @@ mod tests { #[tokio::test] async fn index_file() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .index_file(None) .into_router(); let (code, _) = get(memory_router.clone(), "/", "accept", "*").await; assert_eq!(code, 404); - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .index_file(Some("/index.html")) .into_router(); @@ -443,7 +441,7 @@ mod tests { #[tokio::test] async fn index_file_on_subdirs() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .index_file(Some("/index.html")) .index_on_subdirectories(false) .into_router(); @@ -451,7 +449,7 @@ mod tests { let (code, _) = get(memory_router.clone(), "/blog", "accept", "*").await; assert_eq!(code, 404); - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .index_file(Some("/index.html")) .index_on_subdirectories(true) .into_router(); @@ -462,7 +460,7 @@ mod tests { #[tokio::test] async fn clean_url() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .enable_clean_url(true) .into_router(); @@ -475,11 +473,11 @@ mod tests { #[tokio::test] async fn fallback() { - let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); + let memory_router = MemoryServe::new().into_router(); let (code, _) = get(memory_router.clone(), "/foobar", "accept", "*").await; assert_eq!(code, 404); - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .fallback(Some("/index.html")) .into_router(); let (code, headers) = get(memory_router.clone(), "/foobar", "accept", "*").await; @@ -487,7 +485,7 @@ mod tests { assert_eq!(code, 404); assert_eq!(length.parse::().unwrap(), 437); - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .fallback(Some("/index.html")) .fallback_status(StatusCode::OK) .into_router(); @@ -500,7 +498,7 @@ mod tests { #[tokio::test] async fn cache_control() { async fn check_cache_control(cache_control: CacheControl, expected: &str) { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .cache_control(cache_control) .into_router(); @@ -534,7 +532,7 @@ mod tests { .await; async fn check_html_cache_control(cache_control: CacheControl, expected: &str) { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .html_cache_control(cache_control) .into_router(); @@ -568,7 +566,7 @@ mod tests { #[tokio::test] async fn aliases() { - let memory_router = MemoryServe::new(load_assets!("../static")) + let memory_router = MemoryServe::new() .add_alias("/foobar", "/index.html") .add_alias("/baz", "/index.html") .into_router(); diff --git a/memory-serve-macros/src/utils.rs b/memory-serve/utils.rs similarity index 81% rename from memory-serve-macros/src/utils.rs rename to memory-serve/utils.rs index 639ca5d..1f19ce1 100644 --- a/memory-serve-macros/src/utils.rs +++ b/memory-serve/utils.rs @@ -1,11 +1,39 @@ use mime_guess::mime; -use proc_macro2::Span; +use serde::Serialize; use std::{io::Write, path::Path}; -use syn::LitByteStr; use tracing::{info, warn}; use walkdir::WalkDir; -use crate::Asset; +/// Internal data structure +#[derive(Serialize)] +pub(crate) struct Asset { + pub(crate) route: String, + pub(crate) path: String, + pub(crate) etag: String, + pub(crate) content_type: String, + pub(crate) bytes: Vec, + pub(crate) brotli_bytes: Vec, +} + +impl PartialEq for Asset { + fn eq(&self, other: &Self) -> bool { + self.route == other.route + } +} + +impl Eq for Asset {} + +impl PartialOrd for Asset { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Asset { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.route.cmp(&other.route) + } +} const COMPRESS_TYPES: &[&str] = &[ "text/html", @@ -54,10 +82,6 @@ fn compress_brotli(input: &[u8]) -> Option> { Some(writer.into_inner()) } -fn literal_bytes(bytes: Vec) -> LitByteStr { - LitByteStr::new(&bytes, Span::call_site()) -} - // skip if compressed data is larger than the original fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { if compressed.len() >= original.len() { @@ -107,8 +131,8 @@ pub(crate) fn list_assets(base_path: &Path) -> Vec { path: path.to_owned(), content_type, etag: Default::default(), - bytes: literal_bytes(Default::default()), - brotli_bytes: literal_bytes(Default::default()), + bytes: Default::default(), + brotli_bytes: Default::default(), }); } @@ -142,12 +166,12 @@ pub(crate) fn list_assets(base_path: &Path) -> Vec { path: path.to_owned(), content_type, etag, - bytes: literal_bytes(if brotli_bytes.is_empty() { + bytes: if brotli_bytes.is_empty() { bytes } else { Default::default() - }), - brotli_bytes: literal_bytes(brotli_bytes), + }, + brotli_bytes: brotli_bytes, }) }) .collect(); From 51e85f8da15e8f418d5728cf453625894491029f Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Tue, 8 Oct 2024 19:52:53 +0200 Subject: [PATCH 2/7] Implemented build.rs flow, for single directory --- README.md | 15 +-- memory-serve/Cargo.toml | 8 +- memory-serve/build.rs | 227 ++++++++++++++++++++++++++++++++++++-- memory-serve/src/asset.rs | 10 +- memory-serve/src/lib.rs | 163 ++++++++++++++------------- memory-serve/utils.rs | 182 ------------------------------ 6 files changed, 309 insertions(+), 296 deletions(-) delete mode 100644 memory-serve/utils.rs diff --git a/README.md b/README.md index 71574b7..66bace2 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ memory-serve is designed to work with [axum](https://github.com/tokio-rs/axum) ## Usage Provide a relative path to the directory containing your static assets -to the [`load_assets!`] macro. This macro creates a data structure intended to +usinf the `ASSET_PATH` environment variable. This macro creates a data structure intended to be consumed by [`MemoryServe::new`]. Calling [`MemoryServe::into_router()`] on the resulting instance produces a axum [`Router`](https://docs.rs/axum/latest/axum/routing/struct.Router.html) that @@ -45,12 +45,12 @@ calling [`Router::into_make_service()`](https://docs.rs/axum/latest/axum/routing ```rust,no_run use axum::{response::Html, routing::get, Router}; -use memory_serve::{load_assets, MemoryServe}; +use memory_serve::MemoryServe; use std::net::SocketAddr; #[tokio::main] async fn main() { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new() .index_file(Some("/index.html")) .into_router(); @@ -84,15 +84,6 @@ the following configuration methods: See [`Cache control`](#cache-control) for the cache control options. -## Asset path - -The `load_assets!` macro accepts relative (from `CARGO_MANIFEST_DIR`) or absolute paths. When the environment variable `MEMORY_SERVE_ROOT` -is set, the path is constructed relative from the path provided by `MEMORY_SERVE_ROOT`. - -## Build cache - - - ## Logging During compilation, problems that occur with the inclusion or compression diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index c408278..31b36f4 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -13,8 +13,7 @@ flate2 = "1.0" axum = { version = "0.7", default-features = false } tracing = "0.1" sha256 = "1.4" -postcard = { version = "1.0", features = ["use-std"] } -serde = "1.0" +tower = "0.4" [build-dependencies] sha256 = "1.4" @@ -23,10 +22,9 @@ tracing = "0.1" mime_guess = "2.0" walkdir = "2" tracing-subscriber = { version = "0.3", features = ["fmt", "ansi"], default-features = false } -postcard = { version = "1.0", features = ["use-std"] } -serde = "1.0" +quote = { version = "1.0", default-features = false } [dev-dependencies] tokio = { version = "1", features = ["full"] } -tower = "0.4" +tower = { version = "0.4", features = ["util"] } axum = { version = "0.7" } diff --git a/memory-serve/build.rs b/memory-serve/build.rs index c5cc855..9a18c54 100644 --- a/memory-serve/build.rs +++ b/memory-serve/build.rs @@ -1,28 +1,235 @@ -use std::path::Path; +use mime_guess::mime; +use std::{io::Write, path::Path}; +use tracing::{info, warn}; +use walkdir::WalkDir; -mod utils; +/// Internal data structure +pub struct Asset { + pub route: String, + pub path: String, + pub etag: String, + pub content_type: String, + pub bytes: Vec, + pub is_compressed: bool, +} -const ASSET_FILE: &str = "memory_serve_assets.bin"; +impl PartialEq for Asset { + fn eq(&self, other: &Self) -> bool { + self.route == other.route + } +} + +impl Eq for Asset {} + +impl PartialOrd for Asset { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Asset { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.route.cmp(&other.route) + } +} + +const COMPRESS_TYPES: &[&str] = &[ + "text/html", + "text/css", + "application/json", + "text/javascript", + "application/javascript", + "application/xml", + "text/xml", + "image/svg+xml", + "application/wasm", +]; + +fn path_to_route(base: &Path, path: &Path) -> String { + let relative_path = path + .strip_prefix(base) + .expect("Could not strap prefix from path"); + + let route = relative_path + .components() + .filter_map(|c| match c { + std::path::Component::Normal(s) => s.to_str(), + _ => None, + }) + .collect::>() + .join("/"); + + format!("/{route}") +} + +fn path_to_content_type(path: &Path) -> Option { + let ext = path.extension()?; + + Some( + mime_guess::from_ext(&ext.to_string_lossy()) + .first_raw() + .unwrap_or(mime::APPLICATION_OCTET_STREAM.to_string().as_str()) + .to_owned(), + ) +} + +fn compress_brotli(input: &[u8]) -> Option> { + let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); + writer.write_all(input).ok()?; + + Some(writer.into_inner()) +} + +// skip if compressed data is larger than the original +fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { + if compressed.len() >= original.len() { + Default::default() + } else { + compressed + } +} + +pub fn list_assets(base_path: &Path) -> Vec { + let mut assets: Vec = WalkDir::new(base_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + let Some(path) = entry.path().to_str() else { + warn!("invalid file path {:?}", entry.path()); + return None; + }; + + let route = path_to_route(base_path, entry.path()); + + let Ok(metadata) = entry.metadata() else { + warn!("skipping file {route}, could not get file metadata"); + return None; + }; + + // skip directories + if !metadata.is_file() { + return None; + }; + + // skip empty + if metadata.len() == 0 { + warn!("skipping file {route}: file empty"); + return None; + } + + let Some(content_type) = path_to_content_type(entry.path()) else { + warn!("skipping file {route}, could not determine file extension"); + return None; + }; + + // do not load assets into the binary in debug / development mode + if cfg!(debug_assertions) { + return Some(Asset { + route, + path: path.to_owned(), + content_type, + etag: Default::default(), + bytes: Default::default(), + is_compressed: false, + }); + } + + let Ok(bytes) = std::fs::read(entry.path()) else { + warn!("skipping file {route}: file is not readable"); + return None; + }; + + let etag = sha256::digest(&bytes); + let original_size = bytes.len(); + + let (bytes, is_compressed) = if COMPRESS_TYPES.contains(&content_type.as_str()) { + let brotli_bytes = compress_brotli(&bytes) + .map(|v| skip_larger(v, &bytes)) + .unwrap_or_default(); + + (brotli_bytes, true) + } else { + (bytes, false) + }; + + if is_compressed { + info!( + "including {route} {original_size} -> {} bytes (compressed)", + bytes.len() + ); + } else { + info!("including {route} {} bytes", bytes.len()); + }; + + Some(Asset { + route, + path: path.to_owned(), + content_type, + etag, + bytes, + is_compressed, + }) + }) + .collect(); + + assets.sort(); + + assets +} + +const ASSET_FILE: &str = "memory_serve_assets.rs"; fn main() { let out_dir: String = std::env::var("OUT_DIR").unwrap(); + let pkg_name: String = std::env::var("CARGO_PKG_NAME").unwrap(); - let Ok(memory_serve_dir) = std::env::var("ASSET_DIR") else { - panic!("Please specify the ASSET_DIR environment variable."); + let memory_serve_dir = match std::env::var("ASSET_DIR") { + Ok(dir) => dir, + Err(_) if pkg_name == "memory-serve" => "../static".to_string(), + Err(_) => { + panic!("Please specify the ASSET_DIR environment variable."); + } }; let path = Path::new(&memory_serve_dir); - let path = path.canonicalize().expect("Unable to canonicalize the path specified by ASSET_DIR."); + let path = path + .canonicalize() + .expect("Unable to canonicalize the path specified by ASSET_DIR."); if !path.exists() { panic!("The path {memory_serve_dir} specified by ASSET_DIR does not exists!"); } let target = Path::new(&out_dir).join(ASSET_FILE); + let assets = list_assets(&path); + + let route = assets.iter().map(|a| &a.route); + let path = assets.iter().map(|a| &a.path); + let content_type = assets.iter().map(|a| &a.content_type); + let etag = assets.iter().map(|a| &a.etag); + let is_compressed = assets.iter().map(|a| &a.is_compressed); + let bytes = assets.iter().map(|a| { + let file_name = Path::new(&a.path).file_name().unwrap().to_str().unwrap(); + let target = Path::new(&out_dir).join(file_name); + std::fs::write(&target, &a.bytes).expect("Unable to write file to out dir."); + + target.to_str().unwrap().to_string() + }); + + let code = quote::quote! { + &[ + #(Asset { + route: #route, + path: #path, + content_type: #content_type, + etag: #etag, + bytes: include_bytes!(#bytes), + is_compressed: #is_compressed, + }),* + ] + }; - let assets = utils::list_assets(&path); - let data = postcard::to_allocvec(&assets).expect("Unable to serialize memory-serve assets."); - std::fs::write(target, data).expect("Unable to write memory-serve asset file."); + std::fs::write(target, code.to_string()).expect("Unable to write memory-serve asset file."); println!("cargo::rerun-if-changed={memory_serve_dir}"); -} \ No newline at end of file +} diff --git a/memory-serve/src/asset.rs b/memory-serve/src/asset.rs index 0f3e4b3..e68b2a4 100644 --- a/memory-serve/src/asset.rs +++ b/memory-serve/src/asset.rs @@ -5,7 +5,6 @@ use axum::{ }, response::{IntoResponse, Response}, }; -use serde::{Deserialize, Serialize}; use tracing::{debug, error}; use crate::{ @@ -34,14 +33,14 @@ const GZIP_ENCODING: &str = "gzip"; const GZIP_HEADER: (HeaderName, HeaderValue) = (CONTENT_ENCODING, HeaderValue::from_static(GZIP_ENCODING)); -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug)] pub struct Asset { pub route: &'static str, pub path: &'static str, pub etag: &'static str, pub content_type: &'static str, pub bytes: &'static [u8], - pub brotli_bytes: &'static [u8], + pub is_compressed: bool, } struct AssetResponse<'t, B> { @@ -182,6 +181,7 @@ impl Asset { headers: &HeaderMap, status: StatusCode, bytes: &'static [u8], + brotli_bytes: &'static [u8], gzip_bytes: &'static [u8], options: &ServeOptions, ) -> Response { @@ -199,8 +199,8 @@ impl Asset { etag: self.etag, bytes_len: bytes.len(), bytes, - brotli_bytes_len: self.brotli_bytes.len(), - brotli_bytes: self.brotli_bytes, + brotli_bytes_len: brotli_bytes.len(), + brotli_bytes, gzip_bytes_len: gzip_bytes.len(), gzip_bytes, } diff --git a/memory-serve/src/lib.rs b/memory-serve/src/lib.rs index a63884a..ea89ac9 100644 --- a/memory-serve/src/lib.rs +++ b/memory-serve/src/lib.rs @@ -45,8 +45,6 @@ impl Default for ServeOptions { /// Helper struct to create and configure an axum to serve static files from /// memory. -/// Initiate an instance with the `MemoryServe::new` method and pass a call -/// to the `load_assets!` macro as the single argument. #[derive(Debug, Default)] pub struct MemoryServe { options: ServeOptions, @@ -59,18 +57,17 @@ impl MemoryServe { /// created at build time. /// Specify which asset directory to include using the environment variable `ASSET_DIR`. pub fn new() -> Self { - let asset_bytes = include_bytes!("../memory_serve_assets.bin"); - let assets: Vec = postcard::from_bytes(asset_bytes).expect("Could not deserialize assets"); + let assets: &[Asset] = include!(concat!(env!("OUT_DIR"), "/memory_serve_assets.rs")); Self { - assets: assets.leak(), + assets, ..Default::default() } } /// Which static file to serve on the route "/" (the index) - /// The path (or route) should be relative to the directory passed to - /// the `load_assets!` macro, but prepended with a slash. + /// The path (or route) should be relative to the directory set with + /// the `ASSET_DIR` variable, but prepended with a slash. /// By default this is `Some("/index.html")` pub fn index_file(mut self, index_file: Option<&'static str>) -> Self { self.options.index_file = index_file; @@ -88,8 +85,8 @@ impl MemoryServe { /// Which static file to serve when no other routes are matched, also see /// [fallback](https://docs.rs/axum/latest/axum/routing/struct.Router.html#method.fallback) - /// The path (or route) should be relative to the directory passed to - /// the `load_assets!` macro, but prepended with a slash. + /// The path (or route) should be relative to the directory set with + /// the `ASSET_DIR` variable, but prepended with a slash. /// By default this is `None`, which means axum will return an empty /// response with a HTTP 404 status code when no route matches. pub fn fallback(mut self, fallback: Option<&'static str>) -> Self { @@ -165,25 +162,31 @@ impl MemoryServe { let options = Box::leak(Box::new(self.options)); for asset in self.assets { - let bytes = if asset.bytes.is_empty() && !asset.brotli_bytes.is_empty() { - Box::new(decompress_brotli(asset.brotli_bytes).unwrap_or_default()).leak() + let bytes = if asset.is_compressed { + Box::new(decompress_brotli(asset.bytes).unwrap_or_default()).leak() } else { asset.bytes }; - let gzip_bytes = if !asset.brotli_bytes.is_empty() && options.enable_gzip { + let gzip_bytes = if asset.is_compressed && options.enable_gzip { Box::new(compress_gzip(bytes).unwrap_or_default()).leak() } else { Default::default() }; + let brotli_bytes = if asset.is_compressed { + asset.bytes + } else { + Default::default() + }; + if !bytes.is_empty() { - if !asset.brotli_bytes.is_empty() { + if !asset.is_compressed { info!( "serving {} {} -> {} bytes (compressed)", asset.route, bytes.len(), - asset.brotli_bytes.len() + brotli_bytes.len() ); } else { info!("serving {} {} bytes", asset.route, bytes.len()); @@ -191,7 +194,14 @@ impl MemoryServe { } let handler = |headers: HeaderMap| { - ready(asset.handler(&headers, StatusCode::OK, bytes, gzip_bytes, options)) + ready(asset.handler( + &headers, + StatusCode::OK, + bytes, + brotli_bytes, + gzip_bytes, + options, + )) }; if Some(asset.route) == options.fallback { @@ -202,6 +212,7 @@ impl MemoryServe { &headers, options.fallback_status, bytes, + brotli_bytes, gzip_bytes, options, )) @@ -244,7 +255,7 @@ impl MemoryServe { #[cfg(test)] mod tests { - use crate::{CacheControl, MemoryServe}; + use crate::{Asset, CacheControl, MemoryServe}; use axum::{ body::Body, http::{ @@ -281,54 +292,54 @@ mod tests { headers.get(name).unwrap().to_str().unwrap() } - // #[test] - // fn test_load_assets() { - // let assets: &'static [Asset] = load_assets!("../static"); - // let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); - // let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); - // let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); - - // assert_eq!( - // routes, - // [ - // "/about.html", - // "/assets/icon.jpg", - // "/assets/index.css", - // "/assets/index.js", - // "/assets/stars.svg", - // "/blog/index.html", - // "/index.html" - // ] - // ); - // assert_eq!( - // content_types, - // [ - // "text/html", - // "image/jpeg", - // "text/css", - // "text/javascript", - // "image/svg+xml", - // "text/html", - // "text/html" - // ] - // ); - // if cfg!(debug_assertions) { - // assert_eq!(etags, ["", "", "", "", "", "", ""]); - // } else { - // assert_eq!( - // etags, - // [ - // "56a0dcb83ec56b6c967966a1c06c7b1392e261069d0844aa4e910ca5c1e8cf58", - // "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", - // "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", - // "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", - // "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", - // "89e9873a8e49f962fe83ad2bfe6ac9b21ef7c1b4040b99c34eb783dccbadebc5", - // "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" - // ] - // ); - // } - // } + #[test] + fn test_load_assets() { + let assets: &[Asset] = include!(concat!(env!("OUT_DIR"), "/memory_serve_assets.rs")); + let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); + let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); + let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); + + assert_eq!( + routes, + [ + "/about.html", + "/assets/icon.jpg", + "/assets/index.css", + "/assets/index.js", + "/assets/stars.svg", + "/blog/index.html", + "/index.html" + ] + ); + assert_eq!( + content_types, + [ + "text/html", + "image/jpeg", + "text/css", + "text/javascript", + "image/svg+xml", + "text/html", + "text/html" + ] + ); + if cfg!(debug_assertions) { + assert_eq!(etags, ["", "", "", "", "", "", ""]); + } else { + assert_eq!( + etags, + [ + "56a0dcb83ec56b6c967966a1c06c7b1392e261069d0844aa4e910ca5c1e8cf58", + "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", + "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", + "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", + "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", + "89e9873a8e49f962fe83ad2bfe6ac9b21ef7c1b4040b99c34eb783dccbadebc5", + "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" + ] + ); + } + } #[tokio::test] async fn if_none_match_handling() { @@ -352,9 +363,7 @@ mod tests { #[tokio::test] async fn brotli_compression() { - let memory_router = MemoryServe::new() - .enable_brotli(true) - .into_router(); + let memory_router = MemoryServe::new().enable_brotli(true).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -370,9 +379,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 178); // check disable compression - let memory_router = MemoryServe::new() - .enable_brotli(false) - .into_router(); + let memory_router = MemoryServe::new().enable_brotli(false).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -388,9 +395,7 @@ mod tests { #[tokio::test] async fn gzip_compression() { - let memory_router = MemoryServe::new() - .enable_gzip(true) - .into_router(); + let memory_router = MemoryServe::new().enable_gzip(true).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -406,9 +411,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 274); // check disable compression - let memory_router = MemoryServe::new() - .enable_gzip(false) - .into_router(); + let memory_router = MemoryServe::new().enable_gzip(false).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -424,9 +427,7 @@ mod tests { #[tokio::test] async fn index_file() { - let memory_router = MemoryServe::new() - .index_file(None) - .into_router(); + let memory_router = MemoryServe::new().index_file(None).into_router(); let (code, _) = get(memory_router.clone(), "/", "accept", "*").await; assert_eq!(code, 404); @@ -460,9 +461,7 @@ mod tests { #[tokio::test] async fn clean_url() { - let memory_router = MemoryServe::new() - .enable_clean_url(true) - .into_router(); + let memory_router = MemoryServe::new().enable_clean_url(true).into_router(); let (code, _) = get(memory_router.clone(), "/about.html", "accept", "*").await; assert_eq!(code, 404); diff --git a/memory-serve/utils.rs b/memory-serve/utils.rs deleted file mode 100644 index 1f19ce1..0000000 --- a/memory-serve/utils.rs +++ /dev/null @@ -1,182 +0,0 @@ -use mime_guess::mime; -use serde::Serialize; -use std::{io::Write, path::Path}; -use tracing::{info, warn}; -use walkdir::WalkDir; - -/// Internal data structure -#[derive(Serialize)] -pub(crate) struct Asset { - pub(crate) route: String, - pub(crate) path: String, - pub(crate) etag: String, - pub(crate) content_type: String, - pub(crate) bytes: Vec, - pub(crate) brotli_bytes: Vec, -} - -impl PartialEq for Asset { - fn eq(&self, other: &Self) -> bool { - self.route == other.route - } -} - -impl Eq for Asset {} - -impl PartialOrd for Asset { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Asset { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.route.cmp(&other.route) - } -} - -const COMPRESS_TYPES: &[&str] = &[ - "text/html", - "text/css", - "application/json", - "text/javascript", - "application/javascript", - "application/xml", - "text/xml", - "image/svg+xml", - "application/wasm", -]; - -fn path_to_route(base: &Path, path: &Path) -> String { - let relative_path = path - .strip_prefix(base) - .expect("Could not strap prefix from path"); - - let route = relative_path - .components() - .filter_map(|c| match c { - std::path::Component::Normal(s) => s.to_str(), - _ => None, - }) - .collect::>() - .join("/"); - - format!("/{route}") -} - -fn path_to_content_type(path: &Path) -> Option { - let ext = path.extension()?; - - Some( - mime_guess::from_ext(&ext.to_string_lossy()) - .first_raw() - .unwrap_or(mime::APPLICATION_OCTET_STREAM.to_string().as_str()) - .to_owned(), - ) -} - -fn compress_brotli(input: &[u8]) -> Option> { - let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); - writer.write_all(input).ok()?; - - Some(writer.into_inner()) -} - -// skip if compressed data is larger than the original -fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { - if compressed.len() >= original.len() { - Default::default() - } else { - compressed - } -} - -pub(crate) fn list_assets(base_path: &Path) -> Vec { - let mut assets: Vec = WalkDir::new(base_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| { - let Some(path) = entry.path().to_str() else { - warn!("invalid file path {:?}", entry.path()); - return None; - }; - - let route = path_to_route(base_path, entry.path()); - - let Ok(metadata) = entry.metadata() else { - warn!("skipping file {route}, could not get file metadata"); - return None; - }; - - // skip directories - if !metadata.is_file() { - return None; - }; - - // skip empty - if metadata.len() == 0 { - warn!("skipping file {route}: file empty"); - return None; - } - - let Some(content_type) = path_to_content_type(entry.path()) else { - warn!("skipping file {route}, could not determine file extension"); - return None; - }; - - // do not load assets into the binary in debug / development mode - if cfg!(debug_assertions) { - return Some(Asset { - route, - path: path.to_owned(), - content_type, - etag: Default::default(), - bytes: Default::default(), - brotli_bytes: Default::default(), - }); - } - - let Ok(bytes) = std::fs::read(entry.path()) else { - warn!("skipping file {route}: file is not readable"); - return None; - }; - - let etag = sha256::digest(&bytes); - - let brotli_bytes = if COMPRESS_TYPES.contains(&content_type.as_str()) { - compress_brotli(&bytes) - .map(|v| skip_larger(v, &bytes)) - .unwrap_or_default() - } else { - Default::default() - }; - - if brotli_bytes.is_empty() { - info!("including {route} {} bytes", bytes.len()); - } else { - info!( - "including {route} {} -> {} bytes (compressed)", - bytes.len(), - brotli_bytes.len() - ); - }; - - Some(Asset { - route, - path: path.to_owned(), - content_type, - etag, - bytes: if brotli_bytes.is_empty() { - bytes - } else { - Default::default() - }, - brotli_bytes: brotli_bytes, - }) - }) - .collect(); - - assets.sort(); - - assets -} From d8b3221f090739803f32098e2b1ea019c8455db4 Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Thu, 10 Oct 2024 12:27:33 +0200 Subject: [PATCH 3/7] Improve logging, require environment variable --- .github/workflows/test.yml | 1 + examples/test/src/main.rs | 6 +- memory-serve/Cargo.toml | 3 - memory-serve/build.rs | 195 ++++++++++++++++++++----------------- memory-serve/src/asset.rs | 5 +- memory-serve/src/lib.rs | 18 ++-- 6 files changed, 125 insertions(+), 103 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cf3945..04dc3f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,7 @@ on: env: CARGO_TERM_COLOR: always + ASSET_DIR: ../static jobs: format: diff --git a/examples/test/src/main.rs b/examples/test/src/main.rs index ac00377..733225e 100644 --- a/examples/test/src/main.rs +++ b/examples/test/src/main.rs @@ -1,11 +1,13 @@ use axum::{response::Html, routing::get, Router}; use memory_serve::MemoryServe; use std::net::SocketAddr; -use tracing::info; +use tracing::{info, Level}; #[tokio::main] async fn main() { - tracing_subscriber::fmt().init(); + tracing_subscriber::fmt() + .with_max_level(Level::TRACE) + .init(); let memory_router = MemoryServe::new() .index_file(Some("/index.html")) diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index 31b36f4..d73ef52 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -18,11 +18,8 @@ tower = "0.4" [build-dependencies] sha256 = "1.4" brotli = "6.0" -tracing = "0.1" mime_guess = "2.0" walkdir = "2" -tracing-subscriber = { version = "0.3", features = ["fmt", "ansi"], default-features = false } -quote = { version = "1.0", default-features = false } [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/memory-serve/build.rs b/memory-serve/build.rs index 9a18c54..932ce71 100644 --- a/memory-serve/build.rs +++ b/memory-serve/build.rs @@ -1,16 +1,23 @@ use mime_guess::mime; -use std::{io::Write, path::Path}; -use tracing::{info, warn}; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; use walkdir::WalkDir; +macro_rules! log { + ($($tokens: tt)*) => { + println!("cargo:warning={}", format!($($tokens)*)) + } +} + /// Internal data structure pub struct Asset { pub route: String, - pub path: String, + pub path: PathBuf, pub etag: String, pub content_type: String, - pub bytes: Vec, - pub is_compressed: bool, + pub compressed_bytes: Option>, } impl PartialEq for Asset { @@ -80,29 +87,16 @@ fn compress_brotli(input: &[u8]) -> Option> { Some(writer.into_inner()) } -// skip if compressed data is larger than the original -fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { - if compressed.len() >= original.len() { - Default::default() - } else { - compressed - } -} - pub fn list_assets(base_path: &Path) -> Vec { let mut assets: Vec = WalkDir::new(base_path) .into_iter() .filter_map(|entry| entry.ok()) .filter_map(|entry| { - let Some(path) = entry.path().to_str() else { - warn!("invalid file path {:?}", entry.path()); - return None; - }; - + let path = entry.path().to_owned(); let route = path_to_route(base_path, entry.path()); let Ok(metadata) = entry.metadata() else { - warn!("skipping file {route}, could not get file metadata"); + log!("skipping file {route}, could not get file metadata"); return None; }; @@ -113,62 +107,72 @@ pub fn list_assets(base_path: &Path) -> Vec { // skip empty if metadata.len() == 0 { - warn!("skipping file {route}: file empty"); + log!("skipping file {route}: file empty"); return None; } let Some(content_type) = path_to_content_type(entry.path()) else { - warn!("skipping file {route}, could not determine file extension"); + log!("skipping file {route}, could not determine file extension"); return None; }; // do not load assets into the binary in debug / development mode if cfg!(debug_assertions) { + log!("including {route} (dynamically)"); + return Some(Asset { route, path: path.to_owned(), content_type, etag: Default::default(), - bytes: Default::default(), - is_compressed: false, + compressed_bytes: None, }); } let Ok(bytes) = std::fs::read(entry.path()) else { - warn!("skipping file {route}: file is not readable"); + log!("skipping file {route}: file is not readable"); return None; }; - let etag = sha256::digest(&bytes); + let etag: String = sha256::digest(&bytes); let original_size = bytes.len(); - - let (bytes, is_compressed) = if COMPRESS_TYPES.contains(&content_type.as_str()) { - let brotli_bytes = compress_brotli(&bytes) - .map(|v| skip_larger(v, &bytes)) - .unwrap_or_default(); - - (brotli_bytes, true) - } else { - (bytes, false) - }; - - if is_compressed { - info!( - "including {route} {original_size} -> {} bytes (compressed)", - bytes.len() - ); + let is_compress_type = COMPRESS_TYPES.contains(&content_type.as_str()); + let brotli_bytes = if is_compress_type { + compress_brotli(&bytes) } else { - info!("including {route} {} bytes", bytes.len()); + None }; - Some(Asset { - route, + let mut asset = Asset { + route: route.clone(), path: path.to_owned(), content_type, etag, - bytes, - is_compressed, - }) + compressed_bytes: None, + }; + + if is_compress_type { + match brotli_bytes { + Some(brotli_bytes) if brotli_bytes.len() >= original_size => { + log!("including {route} {original_size} bytes (compression unnecessary)"); + } + Some(brotli_bytes) => { + log!( + "including {route} {original_size} -> {} bytes (compressed)", + brotli_bytes.len() + ); + + asset.compressed_bytes = Some(brotli_bytes); + } + None => { + log!("including {route} {original_size} bytes (compression failed)"); + } + } + } else { + log!("including {route} {original_size} bytes"); + } + + Some(asset) }) .collect(); @@ -180,56 +184,73 @@ pub fn list_assets(base_path: &Path) -> Vec { const ASSET_FILE: &str = "memory_serve_assets.rs"; fn main() { - let out_dir: String = std::env::var("OUT_DIR").unwrap(); - let pkg_name: String = std::env::var("CARGO_PKG_NAME").unwrap(); - - let memory_serve_dir = match std::env::var("ASSET_DIR") { - Ok(dir) => dir, - Err(_) if pkg_name == "memory-serve" => "../static".to_string(), - Err(_) => { - panic!("Please specify the ASSET_DIR environment variable."); - } + let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set."); + let target = Path::new(&out_dir).join(ASSET_FILE); + + let Ok(asset_dir) = std::env::var("ASSET_DIR") else { + log!("Please specify the `ASSET_DIR` environment variable."); + + std::fs::write(target, "&[]").expect("Unable to write memory-serve asset file."); + + println!("cargo::rerun-if-env-changed=ASSET_DIR"); + return; }; - let path = Path::new(&memory_serve_dir); + let path = Path::new(&asset_dir); let path = path .canonicalize() .expect("Unable to canonicalize the path specified by ASSET_DIR."); if !path.exists() { - panic!("The path {memory_serve_dir} specified by ASSET_DIR does not exists!"); + panic!("The path {asset_dir} specified by ASSET_DIR does not exists!"); } - let target = Path::new(&out_dir).join(ASSET_FILE); + log!("Loading static assets from {asset_dir}..."); let assets = list_assets(&path); - let route = assets.iter().map(|a| &a.route); - let path = assets.iter().map(|a| &a.path); - let content_type = assets.iter().map(|a| &a.content_type); - let etag = assets.iter().map(|a| &a.etag); - let is_compressed = assets.iter().map(|a| &a.is_compressed); - let bytes = assets.iter().map(|a| { - let file_name = Path::new(&a.path).file_name().unwrap().to_str().unwrap(); - let target = Path::new(&out_dir).join(file_name); - std::fs::write(&target, &a.bytes).expect("Unable to write file to out dir."); - - target.to_str().unwrap().to_string() - }); - - let code = quote::quote! { - &[ - #(Asset { - route: #route, - path: #path, - content_type: #content_type, - etag: #etag, - bytes: include_bytes!(#bytes), - is_compressed: #is_compressed, - }),* - ] - }; + // using a string is faster than using quote ;) + let mut code = "&[".to_string(); + + for asset in assets { + let Asset { + route, + path, + etag, + content_type, + compressed_bytes, + } = asset; + + let bytes = if cfg!(debug_assertions) { + "None".to_string() + } else if let Some(compressed_bytes) = &compressed_bytes { + let file_name = path.file_name().expect("Unable to get file name."); + let file_path = Path::new(&out_dir).join(file_name); + std::fs::write(&file_path, compressed_bytes).expect("Unable to write file to out dir."); + + format!("Some(include_bytes!(\"{}\"))", file_path.to_string_lossy()) + } else { + format!("Some(include_bytes!(\"{}\"))", path.to_string_lossy()) + }; + + let is_compressed = compressed_bytes.is_some(); + + code.push_str(&format!( + " + Asset {{ + route: \"{route}\", + path: {path:?}, + content_type: \"{content_type}\", + etag: \"{etag}\", + bytes: {bytes}, + is_compressed: {is_compressed}, + }}," + )); + } + + code.push(']'); - std::fs::write(target, code.to_string()).expect("Unable to write memory-serve asset file."); + std::fs::write(target, code).expect("Unable to write memory-serve asset file."); - println!("cargo::rerun-if-changed={memory_serve_dir}"); + println!("cargo::rerun-if-changed={asset_dir}"); + println!("cargo::rerun-if-env-changed=ASSET_DIR"); } diff --git a/memory-serve/src/asset.rs b/memory-serve/src/asset.rs index e68b2a4..0616daf 100644 --- a/memory-serve/src/asset.rs +++ b/memory-serve/src/asset.rs @@ -5,7 +5,7 @@ use axum::{ }, response::{IntoResponse, Response}, }; -use tracing::{debug, error}; +use tracing::debug; use crate::{ util::{compress_brotli, compress_gzip, content_length, supports_encoding}, @@ -39,7 +39,7 @@ pub struct Asset { pub path: &'static str, pub etag: &'static str, pub content_type: &'static str, - pub bytes: &'static [u8], + pub bytes: Option<&'static [u8]>, pub is_compressed: bool, } @@ -142,7 +142,6 @@ impl Asset { options: &ServeOptions, ) -> Response { let Ok(bytes) = std::fs::read(self.path) else { - error!("File not found {}", self.path); return StatusCode::NOT_FOUND.into_response(); }; diff --git a/memory-serve/src/lib.rs b/memory-serve/src/lib.rs index ea89ac9..8a7d0eb 100644 --- a/memory-serve/src/lib.rs +++ b/memory-serve/src/lib.rs @@ -5,7 +5,6 @@ use axum::{ }; use std::future::ready; use tracing::info; - mod asset; mod cache_control; mod util; @@ -162,11 +161,11 @@ impl MemoryServe { let options = Box::leak(Box::new(self.options)); for asset in self.assets { - let bytes = if asset.is_compressed { - Box::new(decompress_brotli(asset.bytes).unwrap_or_default()).leak() - } else { - asset.bytes - }; + let mut bytes = asset.bytes.unwrap_or_default(); + + if asset.is_compressed { + bytes = Box::new(decompress_brotli(bytes).unwrap_or_default()).leak() + } let gzip_bytes = if asset.is_compressed && options.enable_gzip { Box::new(compress_gzip(bytes).unwrap_or_default()).leak() @@ -175,13 +174,13 @@ impl MemoryServe { }; let brotli_bytes = if asset.is_compressed { - asset.bytes + asset.bytes.unwrap_or_default() } else { Default::default() }; if !bytes.is_empty() { - if !asset.is_compressed { + if asset.is_compressed { info!( "serving {} {} -> {} bytes (compressed)", asset.route, @@ -191,6 +190,8 @@ impl MemoryServe { } else { info!("serving {} {} bytes", asset.route, bytes.len()); } + } else { + info!("serving {} (dynamically)", asset.route); } let handler = |headers: HeaderMap| { @@ -403,6 +404,7 @@ mod tests { "gzip", ) .await; + let encoding = get_header(&headers, &CONTENT_ENCODING); let length = get_header(&headers, &CONTENT_LENGTH); From 4c8ec6288913f7429635d93ce5b79e10ab219985 Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Thu, 10 Oct 2024 12:47:54 +0200 Subject: [PATCH 4/7] Update deps --- memory-serve/Cargo.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index d73ef52..1b84cfe 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -8,20 +8,19 @@ publish.workspace = true description.workspace = true [dependencies] -brotli = "6.0" +brotli = "7.0" flate2 = "1.0" axum = { version = "0.7", default-features = false } tracing = "0.1" sha256 = "1.4" -tower = "0.4" [build-dependencies] sha256 = "1.4" -brotli = "6.0" +brotli = "7.0" mime_guess = "2.0" walkdir = "2" [dev-dependencies] tokio = { version = "1", features = ["full"] } -tower = { version = "0.4", features = ["util"] } +tower = { version = "0.5", features = ["util"] } axum = { version = "0.7" } From e2da8781286656c2c3a636716b3d21a8119755aa Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Thu, 10 Oct 2024 15:27:08 +0200 Subject: [PATCH 5/7] Add debugging statements --- memory-serve/build.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/memory-serve/build.rs b/memory-serve/build.rs index 932ce71..4872a65 100644 --- a/memory-serve/build.rs +++ b/memory-serve/build.rs @@ -187,6 +187,11 @@ fn main() { let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set."); let target = Path::new(&out_dir).join(ASSET_FILE); + // prin tenvironment variables + for (key, value) in std::env::vars() { + log!("{}: {}", key, value); + } + let Ok(asset_dir) = std::env::var("ASSET_DIR") else { log!("Please specify the `ASSET_DIR` environment variable."); From d1700a4f4e7673b39b79212cf61246d8e126f596 Mon Sep 17 00:00:00 2001 From: Marlon Baeten Date: Thu, 10 Oct 2024 15:36:53 +0200 Subject: [PATCH 6/7] Hack to resolve crate root --- memory-serve/build.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/memory-serve/build.rs b/memory-serve/build.rs index 4872a65..ef908b7 100644 --- a/memory-serve/build.rs +++ b/memory-serve/build.rs @@ -187,11 +187,6 @@ fn main() { let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set."); let target = Path::new(&out_dir).join(ASSET_FILE); - // prin tenvironment variables - for (key, value) in std::env::vars() { - log!("{}: {}", key, value); - } - let Ok(asset_dir) = std::env::var("ASSET_DIR") else { log!("Please specify the `ASSET_DIR` environment variable."); @@ -202,6 +197,23 @@ fn main() { }; let path = Path::new(&asset_dir); + + let path = if path.is_relative() { + // assume the out dit is in the target directory + let crate_root = target + .parent() // memory-serve + .and_then(|p| p.parent()) // build + .and_then(|p| p.parent()) // debug/release + .and_then(|p| p.parent()) // target + .and_then(|p| p.parent()) // crate root + .expect("Unable to get crate root directory."); + + crate_root.join(path) + } else { + path.to_path_buf() + }; + + let path = path .canonicalize() .expect("Unable to canonicalize the path specified by ASSET_DIR."); From 638807b881dbd0ae70e1a6ef47198ec7ccffe17c Mon Sep 17 00:00:00 2001 From: cikzh Date: Fri, 11 Oct 2024 10:38:46 +0200 Subject: [PATCH 7/7] cargo fmt and typo fix --- README.md | 2 +- memory-serve/build.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 66bace2..51b45e8 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ memory-serve is designed to work with [axum](https://github.com/tokio-rs/axum) ## Usage Provide a relative path to the directory containing your static assets -usinf the `ASSET_PATH` environment variable. This macro creates a data structure intended to +using the `ASSET_PATH` environment variable. This macro creates a data structure intended to be consumed by [`MemoryServe::new`]. Calling [`MemoryServe::into_router()`] on the resulting instance produces a axum [`Router`](https://docs.rs/axum/latest/axum/routing/struct.Router.html) that diff --git a/memory-serve/build.rs b/memory-serve/build.rs index ef908b7..337f8db 100644 --- a/memory-serve/build.rs +++ b/memory-serve/build.rs @@ -213,7 +213,6 @@ fn main() { path.to_path_buf() }; - let path = path .canonicalize() .expect("Unable to canonicalize the path specified by ASSET_DIR.");