Add more attributes (#5690)

Signed-off-by: netthier <[email protected]>

object_store/src/attributes.rs

@@ -23,6 +23,18 @@ use std::ops::Deref;
 #[non_exhaustive]
 #[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub enum Attribute {
+    /// Specifies how the object should be handled by a browser
+    ///
+    /// See [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition)
+    ContentDisposition,
+    /// Specifies the encodings applied to the object
+    ///
+    /// See [Content-Encoding](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding)
+    ContentEncoding,
+    /// Specifies the language of the object
+    ///
+    /// See [Content-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language)
+    ContentLanguage,
     /// Specifies the MIME type of the object
     ///
     /// This takes precedence over any [ClientOptions](crate::ClientOptions) configuration
@@ -177,12 +189,15 @@ mod tests {
     #[test]
     fn test_attributes_basic() {
         let mut attributes = Attributes::from_iter([
+            (Attribute::ContentDisposition, "inline"),
+            (Attribute::ContentEncoding, "gzip"),
+            (Attribute::ContentLanguage, "en-US"),
             (Attribute::ContentType, "test"),
             (Attribute::CacheControl, "control"),
         ]);
 
         assert!(!attributes.is_empty());
-        assert_eq!(attributes.len(), 2);
+        assert_eq!(attributes.len(), 5);
 
         assert_eq!(
             attributes.get(&Attribute::ContentType),
@@ -195,17 +210,30 @@ mod tests {
             attributes.insert(Attribute::CacheControl, "v1".into()),
             Some(metav)
         );
-        assert_eq!(attributes.len(), 2);
+        assert_eq!(attributes.len(), 5);
 
         assert_eq!(
             attributes.remove(&Attribute::CacheControl).unwrap(),
             "v1".into()
         );
-        assert_eq!(attributes.len(), 1);
+        assert_eq!(attributes.len(), 4);
 
         let metav: AttributeValue = "v2".into();
         attributes.insert(Attribute::CacheControl, metav.clone());
         assert_eq!(attributes.get(&Attribute::CacheControl), Some(&metav));
-        assert_eq!(attributes.len(), 2);
+        assert_eq!(attributes.len(), 5);
+
+        assert_eq!(
+            attributes.get(&Attribute::ContentDisposition),
+            Some(&"inline".into())
+        );
+        assert_eq!(
+            attributes.get(&Attribute::ContentEncoding),
+            Some(&"gzip".into())
+        );
+        assert_eq!(
+            attributes.get(&Attribute::ContentLanguage),
+            Some(&"en-US".into())
+        );
     }
 }

object_store/src/aws/client.rs

@@ -42,14 +42,17 @@ use async_trait::async_trait;
 use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
 use bytes::{Buf, Bytes};
-use hyper::header::{CACHE_CONTROL, CONTENT_LENGTH};
+use hyper::header::{
+    CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH,
+    CONTENT_TYPE,
+};
 use hyper::http::HeaderName;
 use hyper::{http, HeaderMap};
 use itertools::Itertools;
 use md5::{Digest, Md5};
 use percent_encoding::{utf8_percent_encode, PercentEncode};
 use quick_xml::events::{self as xml_events};
-use reqwest::{header::CONTENT_TYPE, Client as ReqwestClient, Method, RequestBuilder, Response};
+use reqwest::{Client as ReqwestClient, Method, RequestBuilder, Response};
 use ring::digest;
 use ring::digest::Context;
 use serde::{Deserialize, Serialize};
@@ -322,6 +325,9 @@ impl<'a> Request<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(CACHE_CONTROL, v.as_ref()),
+                Attribute::ContentDisposition => builder.header(CONTENT_DISPOSITION, v.as_ref()),
+                Attribute::ContentEncoding => builder.header(CONTENT_ENCODING, v.as_ref()),
+                Attribute::ContentLanguage => builder.header(CONTENT_LANGUAGE, v.as_ref()),
                 Attribute::ContentType => {
                     has_content_type = true;
                     builder.header(CONTENT_TYPE, v.as_ref())

object_store/src/azure/client.rs

@@ -50,6 +50,10 @@ use url::Url;
 const VERSION_HEADER: &str = "x-ms-version-id";
 static MS_CACHE_CONTROL: HeaderName = HeaderName::from_static("x-ms-blob-cache-control");
 static MS_CONTENT_TYPE: HeaderName = HeaderName::from_static("x-ms-blob-content-type");
+static MS_CONTENT_DISPOSITION: HeaderName =
+    HeaderName::from_static("x-ms-blob-content-disposition");
+static MS_CONTENT_ENCODING: HeaderName = HeaderName::from_static("x-ms-blob-content-encoding");
+static MS_CONTENT_LANGUAGE: HeaderName = HeaderName::from_static("x-ms-blob-content-language");
 
 static TAGS_HEADER: HeaderName = HeaderName::from_static("x-ms-tags");
 
@@ -206,6 +210,11 @@ impl<'a> PutRequest<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(&MS_CACHE_CONTROL, v.as_ref()),
+                Attribute::ContentDisposition => {
+                    builder.header(&MS_CONTENT_DISPOSITION, v.as_ref())
+                }
+                Attribute::ContentEncoding => builder.header(&MS_CONTENT_ENCODING, v.as_ref()),
+                Attribute::ContentLanguage => builder.header(&MS_CONTENT_LANGUAGE, v.as_ref()),
                 Attribute::ContentType => {
                     has_content_type = true;
                     builder.header(&MS_CONTENT_TYPE, v.as_ref())

object_store/src/client/get.rs

@@ -22,7 +22,10 @@ use crate::path::Path;
 use crate::{Attribute, Attributes, GetOptions, GetRange, GetResult, GetResultPayload, Result};
 use async_trait::async_trait;
 use futures::{StreamExt, TryStreamExt};
-use hyper::header::{CACHE_CONTROL, CONTENT_RANGE, CONTENT_TYPE};
+use hyper::header::{
+    CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_RANGE,
+    CONTENT_TYPE,
+};
 use hyper::StatusCode;
 use reqwest::header::ToStrError;
 use reqwest::Response;
@@ -120,6 +123,15 @@ enum GetResultError {
     #[snafu(display("Cache-Control header contained non UTF-8 characters"))]
     InvalidCacheControl { source: ToStrError },
 
+    #[snafu(display("Content-Disposition header contained non UTF-8 characters"))]
+    InvalidContentDisposition { source: ToStrError },
+
+    #[snafu(display("Content-Encoding header contained non UTF-8 characters"))]
+    InvalidContentEncoding { source: ToStrError },
+
+    #[snafu(display("Content-Language header contained non UTF-8 characters"))]
+    InvalidContentLanguage { source: ToStrError },
+
     #[snafu(display("Content-Type header contained non UTF-8 characters"))]
     InvalidContentType { source: ToStrError },
 
@@ -167,16 +179,48 @@ fn get_result<T: GetClient>(
         0..meta.size
     };
 
-    let mut attributes = Attributes::new();
-    if let Some(x) = response.headers().get(CACHE_CONTROL) {
-        let x = x.to_str().context(InvalidCacheControlSnafu)?;
-        attributes.insert(Attribute::CacheControl, x.to_string().into());
-    }
-    if let Some(x) = response.headers().get(CONTENT_TYPE) {
-        let x = x.to_str().context(InvalidContentTypeSnafu)?;
-        attributes.insert(Attribute::ContentType, x.to_string().into());
+    macro_rules! parse_attributes {
+        ($headers:expr, $(($header:expr, $attr:expr, $err:expr)),*) => {{
+            let mut attributes = Attributes::new();
+            $(
+            if let Some(x) = $headers.get($header) {
+                let x = x.to_str().context($err)?;
+                attributes.insert($attr, x.to_string().into());
+            }
+            )*
+            attributes
+        }}
     }
 
+    let attributes = parse_attributes!(
+        response.headers(),
+        (
+            CACHE_CONTROL,
+            Attribute::CacheControl,
+            InvalidCacheControlSnafu
+        ),
+        (
+            CONTENT_DISPOSITION,
+            Attribute::ContentDisposition,
+            InvalidContentDispositionSnafu
+        ),
+        (
+            CONTENT_ENCODING,
+            Attribute::ContentEncoding,
+            InvalidContentEncodingSnafu
+        ),
+        (
+            CONTENT_LANGUAGE,
+            Attribute::ContentLanguage,
+            InvalidContentLanguageSnafu
+        ),
+        (
+            CONTENT_TYPE,
+            Attribute::ContentType,
+            InvalidContentTypeSnafu
+        )
+    );
+
     let stream = response
         .bytes_stream()
         .map_err(|source| crate::Error::Generic {

object_store/src/gcp/client.rs

@@ -36,7 +36,10 @@ use async_trait::async_trait;
 use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
 use bytes::Buf;
-use hyper::header::{CACHE_CONTROL, CONTENT_LENGTH, CONTENT_TYPE};
+use hyper::header::{
+    CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH,
+    CONTENT_TYPE,
+};
 use percent_encoding::{percent_encode, utf8_percent_encode, NON_ALPHANUMERIC};
 use reqwest::header::HeaderName;
 use reqwest::{Client, Method, RequestBuilder, Response, StatusCode};
@@ -195,6 +198,9 @@ impl<'a> Request<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(CACHE_CONTROL, v.as_ref()),
+                Attribute::ContentDisposition => builder.header(CONTENT_DISPOSITION, v.as_ref()),
+                Attribute::ContentEncoding => builder.header(CONTENT_ENCODING, v.as_ref()),
+                Attribute::ContentLanguage => builder.header(CONTENT_LANGUAGE, v.as_ref()),
                 Attribute::ContentType => {
                     has_content_type = true;
                     builder.header(CONTENT_TYPE, v.as_ref())

object_store/src/http/client.rs

@@ -25,9 +25,11 @@ use crate::{Attribute, Attributes, ClientOptions, GetOptions, ObjectMeta, PutPay
 use async_trait::async_trait;
 use bytes::Buf;
 use chrono::{DateTime, Utc};
-use hyper::header::{CACHE_CONTROL, CONTENT_LENGTH};
+use hyper::header::{
+    CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH,
+    CONTENT_TYPE,
+};
 use percent_encoding::percent_decode_str;
-use reqwest::header::CONTENT_TYPE;
 use reqwest::{Method, Response, StatusCode};
 use serde::Deserialize;
 use snafu::{OptionExt, ResultExt, Snafu};
@@ -172,6 +174,11 @@ impl Client {
             for (k, v) in &attributes {
                 builder = match k {
                     Attribute::CacheControl => builder.header(CACHE_CONTROL, v.as_ref()),
+                    Attribute::ContentDisposition => {
+                        builder.header(CONTENT_DISPOSITION, v.as_ref())
+                    }
+                    Attribute::ContentEncoding => builder.header(CONTENT_ENCODING, v.as_ref()),
+                    Attribute::ContentLanguage => builder.header(CONTENT_LANGUAGE, v.as_ref()),
                     Attribute::ContentType => {
                         has_content_type = true;
                         builder.header(CONTENT_TYPE, v.as_ref())

object_store/src/lib.rs

@@ -1744,8 +1744,14 @@ mod tests {
     pub(crate) async fn put_get_attributes(integration: &dyn ObjectStore) {
         // Test handling of attributes
         let attributes = Attributes::from_iter([
-            (Attribute::ContentType, "text/html; charset=utf-8"),
             (Attribute::CacheControl, "max-age=604800"),
+            (
+                Attribute::ContentDisposition,
+                r#"attachment; filename="test.html""#,
+            ),
+            (Attribute::ContentEncoding, "gzip"),
+            (Attribute::ContentLanguage, "en-US"),
+            (Attribute::ContentType, "text/html; charset=utf-8"),
         ]);
 
         let path = Path::from("attributes");