Merge branch 'main' into openssl-feature

CODE_OF_CONDUCT.md

@@ -8,5 +8,3 @@ For example, the political statements in the pages below will be rejected.
 + <https://blog.rust-lang.org/2020/06/04/Rust-1.44.0.html>
 + <https://blog.rust-lang.org/2022/02/24/Rust-1.59.0.html>
 + <https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html>
-+ <https://github.com/apps/bors>
-

codegen/src/ops.rs

@@ -24,6 +24,8 @@ pub struct Operation {
     pub smithy_input: String,
     pub smithy_output: String,
 
+    pub s3_unwrapped_xml_output: bool,
+
     pub doc: Option<String>,
 
     pub http_method: String,
@@ -77,6 +79,12 @@ pub fn collect_operations(model: &smithy::Model) -> Operations {
             smithy_http_code
         };
 
+        // https://smithy.io/2.0/aws/customizations/s3-customizations.html
+        // https://github.com/Nugine/s3s/pull/127
+        if sh.traits.s3_unwrapped_xml_output() {
+            assert_eq!(op_name, "GetBucketLocation");
+        }
+
         let op = Operation {
             name: op_name.clone(),
 
@@ -86,6 +94,8 @@ pub fn collect_operations(model: &smithy::Model) -> Operations {
             smithy_input,
             smithy_output,
 
+            s3_unwrapped_xml_output: sh.traits.s3_unwrapped_xml_output(),
+
             doc: sh.traits.doc().map(o),
 
             http_method: sh.traits.http_method().unwrap().to_owned(),

codegen/src/smithy.rs

@@ -209,6 +209,10 @@ impl Traits {
         self.get("smithy.api#xmlFlattened").is_some()
     }
 
+    pub fn s3_unwrapped_xml_output(&self) -> bool {
+        self.get("aws.customizations#s3UnwrappedXmlOutput").is_some()
+    }
+
     pub fn http_label(&self) -> Option<&Value> {
         self.get("smithy.api#httpLabel")
     }

codegen/src/xml.rs

@@ -154,8 +154,15 @@ fn codegen_xml_ser(ops: &Operations, rust_types: &RustTypes) {
                             g!("s.timestamp(\"{}\", &self.{}, TimestampFormat::{})?;", xml_name, field.name, fmt);
                         }
                     } else if field.option_type {
+                        let s3_unwrapped_xml_output =
+                            ops.iter().any(|(_, op)| op.s3_unwrapped_xml_output && op.output == ty.name);
+
                         g!("if let Some(ref val) = self.{} {{", field.name);
-                        g!("s.content(\"{xml_name}\", val)?;");
+                        if s3_unwrapped_xml_output {
+                            g!("val.serialize_content(s)?;");
+                        } else {
+                            g!("s.content(\"{xml_name}\", val)?;");
+                        }
                         g!("}}");
                     } else {
                         let default_is_zero = match field.default_value.as_ref() {

crates/s3s-fs/src/fs.rs

@@ -74,9 +74,10 @@ impl FileSystem {
     }
 
     /// resolve metadata path under the virtual root (custom format)
-    pub(crate) fn get_metadata_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
+    pub(crate) fn get_metadata_path(&self, bucket: &str, key: &str, upload_id: Option<Uuid>) -> Result<PathBuf> {
         let encode = |s: &str| base64_simd::URL_SAFE_NO_PAD.encode_to_string(s);
-        let file_path = format!(".bucket-{}.object-{}.metadata.json", encode(bucket), encode(key));
+        let u_ext = upload_id.map(|u| format!(".upload-{u}")).unwrap_or_default();
+        let file_path = format!(".bucket-{}.object-{}{u_ext}.metadata.json", encode(bucket), encode(key));
         self.resolve_abs_path(file_path)
     }
 
@@ -87,8 +88,8 @@ impl FileSystem {
     }
 
     /// load metadata from fs
-    pub(crate) async fn load_metadata(&self, bucket: &str, key: &str) -> Result<Option<dto::Metadata>> {
-        let path = self.get_metadata_path(bucket, key)?;
+    pub(crate) async fn load_metadata(&self, bucket: &str, key: &str, upload_id: Option<Uuid>) -> Result<Option<dto::Metadata>> {
+        let path = self.get_metadata_path(bucket, key, upload_id)?;
         if path.exists().not() {
             return Ok(None);
         }
@@ -98,13 +99,26 @@ impl FileSystem {
     }
 
     /// save metadata to fs
-    pub(crate) async fn save_metadata(&self, bucket: &str, key: &str, metadata: &dto::Metadata) -> Result<()> {
-        let path = self.get_metadata_path(bucket, key)?;
+    pub(crate) async fn save_metadata(
+        &self,
+        bucket: &str,
+        key: &str,
+        metadata: &dto::Metadata,
+        upload_id: Option<Uuid>,
+    ) -> Result<()> {
+        let path = self.get_metadata_path(bucket, key, upload_id)?;
         let content = serde_json::to_vec(metadata)?;
         fs::write(&path, &content).await?;
         Ok(())
     }
 
+    /// remove metadata from fs
+    pub(crate) fn delete_metadata(&self, bucket: &str, key: &str, upload_id: Option<Uuid>) -> Result<()> {
+        let path = self.get_metadata_path(bucket, key, upload_id)?;
+        std::fs::remove_file(path)?;
+        Ok(())
+    }
+
     pub(crate) async fn load_internal_info(&self, bucket: &str, key: &str) -> Result<Option<InternalInfo>> {
         let path = self.get_internal_info_path(bucket, key)?;
         if path.exists().not() {

crates/s3s-fs/src/main.rs

@@ -7,10 +7,11 @@ use s3s_fs::Result;
 use s3s::auth::SimpleAuth;
 use s3s::service::S3ServiceBuilder;
 
+use std::io::IsTerminal;
 use std::net::TcpListener;
 use std::path::PathBuf;
 
-use clap::Parser;
+use clap::{CommandFactory, Parser};
 use hyper::server::Server;
 use tracing::info;
 
@@ -26,11 +27,11 @@ struct Opt {
     port: u16,
 
     /// Access key used for authentication.
-    #[arg(long, requires("secret-key"))]
+    #[arg(long)]
     access_key: Option<String>,
 
     /// Secret key used for authentication.
-    #[arg(long, requires("access-key"))]
+    #[arg(long)]
     secret_key: Option<String>,
 
     /// Domain name used for virtual-hosted-style requests.
@@ -45,8 +46,7 @@ fn setup_tracing() {
     use tracing_subscriber::EnvFilter;
 
     let env_filter = EnvFilter::from_default_env();
-    // let enable_color = std::io::stdout().is_terminal(); // TODO
-    let enable_color = false;
+    let enable_color = std::io::stdout().is_terminal();
 
     tracing_subscriber::fmt()
         .pretty()
@@ -55,18 +55,28 @@ fn setup_tracing() {
         .init();
 }
 
-fn check_cli_args(opt: &Opt) -> Result<(), String> {
+fn check_cli_args(opt: &Opt) {
+    use clap::error::ErrorKind;
+
+    let mut cmd = Opt::command();
+
+    // TODO: how to specify the requirements with clap derive API?
+    if let (Some(_), None) | (None, Some(_)) = (&opt.access_key, &opt.secret_key) {
+        let msg = "access key and secret key must be specified together";
+        cmd.error(ErrorKind::MissingRequiredArgument, msg).exit();
+    }
+
     if let Some(ref s) = opt.domain_name {
         if s.contains('/') {
-            return Err(format!("expected domain name, found URL-like string: {s:?}"));
+            let msg = format!("expected domain name, found URL-like string: {s:?}");
+            cmd.error(ErrorKind::InvalidValue, msg).exit();
         }
     }
-    Ok(())
 }
 
 fn main() -> Result {
     let opt = Opt::parse();
-    check_cli_args(&opt).map_err(s3s_fs::Error::from_string)?;
+    check_cli_args(&opt);
 
     setup_tracing();
 
@@ -85,11 +95,13 @@ async fn run(opt: Opt) -> Result {
         // Enable authentication
         if let (Some(ak), Some(sk)) = (opt.access_key, opt.secret_key) {
             b.set_auth(SimpleAuth::from_single(ak, sk));
+            info!("authentication is enabled");
         }
 
         // Enable parsing virtual-hosted-style requests
         if let Some(domain_name) = opt.domain_name {
             b.set_base_domain(domain_name);
+            info!("virtual-hosted-style requests are enabled");
         }
 
         b.build()

crates/s3s-fs/src/s3.rs

@@ -48,6 +48,11 @@ fn normalize_path(path: &Path, delimiter: &str) -> Option<String> {
     Some(normalized)
 }
 
+/// <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range>
+fn fmt_content_range(start: u64, end_inclusive: u64, size: u64) -> String {
+    format!("bytes {start}-{end_inclusive}/{size}")
+}
+
 #[async_trait::async_trait]
 impl S3 for FileSystem {
     #[tracing::instrument]
@@ -95,9 +100,9 @@ impl S3 for FileSystem {
 
         debug!(from = %src_path.display(), to = %dst_path.display(), "copy file");
 
-        let src_metadata_path = self.get_metadata_path(bucket, key)?;
+        let src_metadata_path = self.get_metadata_path(bucket, key, None)?;
         if src_metadata_path.exists() {
-            let dst_metadata_path = self.get_metadata_path(&input.bucket, &input.key)?;
+            let dst_metadata_path = self.get_metadata_path(&input.bucket, &input.key, None)?;
             let _ = try_!(fs::copy(src_metadata_path, dst_metadata_path).await);
         }
 
@@ -195,11 +200,13 @@ impl S3 for FileSystem {
         let last_modified = Timestamp::from(try_!(file_metadata.modified()));
         let file_len = file_metadata.len();
 
-        let content_length = match input.range {
-            None => file_len,
+        let (content_length, content_range) = match input.range {
+            None => (file_len, None),
             Some(range) => {
                 let file_range = range.check(file_len)?;
-                file_range.end - file_range.start
+                let content_length = file_range.end - file_range.start;
+                let content_range = fmt_content_range(file_range.start, file_range.end - 1, file_len);
+                (content_length, Some(content_range))
             }
         };
         let content_length_usize = try_!(usize::try_from(content_length));
@@ -218,7 +225,7 @@ impl S3 for FileSystem {
 
         let body = bytes_stream(ReaderStream::with_capacity(file, 4096), content_length_usize);
 
-        let object_metadata = self.load_metadata(&input.bucket, &input.key).await?;
+        let object_metadata = self.load_metadata(&input.bucket, &input.key, None).await?;
 
         let md5_sum = self.get_md5_sum(&input.bucket, &input.key).await?;
         let e_tag = format!("\"{md5_sum}\"");
@@ -232,6 +239,7 @@ impl S3 for FileSystem {
         let output = GetObjectOutput {
             body: Some(StreamingBlob::wrap(body)),
             content_length: content_length_i64,
+            content_range,
             last_modified: Some(last_modified),
             metadata: object_metadata,
             e_tag: Some(e_tag),
@@ -269,7 +277,7 @@ impl S3 for FileSystem {
         let last_modified = Timestamp::from(try_!(file_metadata.modified()));
         let file_len = file_metadata.len();
 
-        let object_metadata = self.load_metadata(&input.bucket, &input.key).await?;
+        let object_metadata = self.load_metadata(&input.bucket, &input.key, None).await?;
 
         // TODO: detect content type
         let content_type = mime::APPLICATION_OCTET_STREAM;
@@ -496,7 +504,7 @@ impl S3 for FileSystem {
         debug!(path = %object_path.display(), ?size, %md5_sum, ?checksum, "write file");
 
         if let Some(ref metadata) = metadata {
-            self.save_metadata(&bucket, &key, metadata).await?;
+            self.save_metadata(&bucket, &key, metadata, None).await?;
         }
 
         let mut info: InternalInfo = default();
@@ -524,6 +532,11 @@ impl S3 for FileSystem {
         let input = req.input;
         let upload_id = self.create_upload_id(req.credentials.as_ref()).await?;
 
+        if let Some(ref metadata) = input.metadata {
+            self.save_metadata(&input.bucket, &input.key, metadata, Some(upload_id))
+                .await?;
+        }
+
         let output = CreateMultipartUploadOutput {
             bucket: Some(input.bucket),
             key: Some(input.key),
@@ -707,6 +720,11 @@ impl S3 for FileSystem {
 
         self.delete_upload_id(&upload_id).await?;
 
+        if let Ok(Some(metadata)) = self.load_metadata(&bucket, &key, Some(upload_id)).await {
+            self.save_metadata(&bucket, &key, &metadata, None).await?;
+            let _ = self.delete_metadata(&bucket, &key, Some(upload_id));
+        }
+
         let object_path = self.get_object_path(&bucket, &key)?;
         let mut file_writer = self.prepare_file_write(&object_path).await?;
 
@@ -756,6 +774,8 @@ impl S3 for FileSystem {
             return Err(s3_error!(AccessDenied));
         }
 
+        let _ = self.delete_metadata(&bucket, &key, Some(upload_id));
+
         let prefix = format!(".upload_id-{upload_id}");
         let mut iter = try_!(fs::read_dir(&self.root).await);
         while let Some(entry) = try_!(iter.next_entry().await) {

crates/s3s-proxy/src/main.rs

@@ -5,6 +5,7 @@ use s3s::auth::SimpleAuth;
 use s3s::service::S3ServiceBuilder;
 
 use std::error::Error;
+use std::io::IsTerminal;
 use std::net::TcpListener;
 
 use aws_credential_types::provider::ProvideCredentials;
@@ -32,8 +33,7 @@ fn setup_tracing() {
     use tracing_subscriber::EnvFilter;
 
     let env_filter = EnvFilter::from_default_env();
-    // let enable_color = std::io::stdout().is_terminal(); // TODO
-    let enable_color = false;
+    let enable_color = std::io::stdout().is_terminal();
 
     tracing_subscriber::fmt()
         .pretty()