diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml index 512fa305960c..f3aaf35fbb02 100644 --- a/object_store/Cargo.toml +++ b/object_store/Cargo.toml @@ -54,6 +54,7 @@ reqwest = { version = "0.11", default-features = false, features = ["rustls-tls- ring = { version = "0.17", default-features = false, features = ["std"], optional = true } rustls-pemfile = { version = "2.0", default-features = false, features = ["std"], optional = true } tokio = { version = "1.25.0", features = ["sync", "macros", "rt", "time", "io-util"] } +md-5 = { version = "0.10.6", default-features = false, optional = true } [target.'cfg(target_family="unix")'.dev-dependencies] nix = { version = "0.27.1", features = ["fs"] } @@ -62,7 +63,7 @@ nix = { version = "0.27.1", features = ["fs"] } cloud = ["serde", "serde_json", "quick-xml", "hyper", "reqwest", "reqwest/json", "reqwest/stream", "chrono/serde", "base64", "rand", "ring"] azure = ["cloud"] gcp = ["cloud", "rustls-pemfile"] -aws = ["cloud"] +aws = ["cloud", "md-5"] http = ["cloud"] tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"] diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs index fed6911e6f04..a31350fad4d8 100644 --- a/object_store/src/aws/client.rs +++ b/object_store/src/aws/client.rs @@ -43,6 +43,7 @@ use bytes::{Buf, Bytes}; use hyper::http; use hyper::http::HeaderName; use itertools::Itertools; +use md5::{Digest, Md5}; use percent_encoding::{utf8_percent_encode, PercentEncode}; use quick_xml::events::{self as xml_events}; use reqwest::{ @@ -438,6 +439,14 @@ impl S3Client { None }; + // S3 *requires* DeleteObjects to include a Content-MD5 header: + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + // > "The Content-MD5 request header is required for all Multi-Object Delete requests" + // Some platforms, like MinIO, enforce this requirement and fail requests without the header. + let mut hasher = Md5::new(); + hasher.update(&body); + builder = builder.header("Content-MD5", BASE64_STANDARD.encode(hasher.finalize())); + let response = builder .header(CONTENT_TYPE, "application/xml") .body(body)