[DRAFT] Extension Type Registry Draft

Cargo.toml

@@ -94,6 +94,7 @@ apache-avro = { version = "0.20", default-features = false }
 arrow = { version = "57.0.0", features = [
     "prettyprint",
     "chrono-tz",
+    "canonical_extension_types"
 ] }
 arrow-buffer = { version = "57.0.0", default-features = false }
 arrow-flight = { version = "57.0.0", features = [
@@ -103,7 +104,7 @@ arrow-ipc = { version = "57.0.0", default-features = false, features = [
     "lz4",
 ] }
 arrow-ord = { version = "57.0.0", default-features = false }
-arrow-schema = { version = "57.0.0", default-features = false }
+arrow-schema = { version = "57.0.0", default-features = false, features = ["canonical_extension_types"] }
 async-trait = "0.1.89"
 bigdecimal = "0.4.8"
 bytes = "1.10"
@@ -182,6 +183,7 @@ testcontainers = { version = "0.25.2", features = ["default"] }
 testcontainers-modules = { version = "0.13" }
 tokio = { version = "1.48", features = ["macros", "rt", "sync"] }
 url = "2.5.7"
+uuid = { version = "1.18", features = ["v4"] }
 
 [workspace.lints.clippy]
 # Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml)

datafusion/common/Cargo.toml

@@ -61,6 +61,7 @@ apache-avro = { version = "0.20", default-features = false, features = [
 ], optional = true }
 arrow = { workspace = true }
 arrow-ipc = { workspace = true }
+arrow-schema = { workspace = true }
 chrono = { workspace = true }
 half = { workspace = true }
 hashbrown = { workspace = true }
@@ -75,6 +76,7 @@ pyo3 = { version = "0.26", optional = true }
 recursive = { workspace = true, optional = true }
 sqlparser = { workspace = true, optional = true }
 tokio = { workspace = true }
+uuid = { version = "1.18.1", features = ["v4"] }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
 web-time = "1.1.0"

datafusion/common/src/types/canonical.rs

@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::_internal_err;
+use crate::types::{
+    LogicalType, NativeType, TypeParameter, TypeSignature, ValuePrettyPrinter,
+};
+use crate::ScalarValue;
+use crate::{Result, _internal_datafusion_err};
+use arrow_schema::extension::{ExtensionType, Opaque, Uuid};
+use std::sync::{Arc, LazyLock};
+use uuid::Bytes;
+
+impl LogicalType for Uuid {
+    fn native(&self) -> &NativeType {
+        &NativeType::FixedSizeBinary(16)
+    }
+
+    fn signature(&self) -> TypeSignature<'_> {
+        TypeSignature::Extension {
+            name: Uuid::NAME,
+            parameters: vec![],
+        }
+    }
+
+    fn pretty_printer(&self) -> &Arc<dyn ValuePrettyPrinter> {
+        static PRETTY_PRINTER: LazyLock<Arc<dyn ValuePrettyPrinter>> =
+            LazyLock::new(|| Arc::new(UuidValuePrettyPrinter {}));
+        &PRETTY_PRINTER
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)]
+struct UuidValuePrettyPrinter;
+
+impl ValuePrettyPrinter for UuidValuePrettyPrinter {
+    fn pretty_print_scalar(&self, value: &ScalarValue) -> Result<String> {
+        match value {
+            ScalarValue::FixedSizeBinary(16, value) => match value {
+                Some(value) => {
+                    let bytes = Bytes::try_from(value.as_slice()).map_err(|_| {
+                        _internal_datafusion_err!(
+                            "Invalid UUID bytes even though type is correct."
+                        )
+                    })?;
+                    let uuid = uuid::Uuid::from_bytes(bytes);
+                    Ok(format!("arrow.uuid({uuid})"))
+                }
+                None => Ok("arrow.uuid(NULL)".to_owned()),
+            },
+            _ => _internal_err!("Wrong scalar given to "),
+        }
+    }
+}
+
+/// Represents the canonical [Opaque extension type](https://arrow.apache.org/docs/format/CanonicalExtensions.html#opaque).
+///
+/// In the context of DataFusion, a common use case of the opaque type is when an extension type
+/// is unknown to DataFusion. Contrary to [UnresolvedExtensionType], the extension type has
+/// already been checked against the extension type registry and was not found.
+impl LogicalType for Opaque {
+    fn native(&self) -> &NativeType {
+        &NativeType::FixedSizeBinary(16)
+    }
+
+    fn signature(&self) -> TypeSignature<'_> {
+        let parameter = TypeParameter::Type(TypeSignature::Extension {
+            name: self.metadata().type_name(),
+            parameters: vec![],
+        });
+        TypeSignature::Extension {
+            name: Opaque::NAME,
+            parameters: vec![parameter],
+        }
+    }
+
+    fn pretty_printer(&self) -> &Arc<dyn ValuePrettyPrinter> {
+        static PRETTY_PRINTER: LazyLock<Arc<dyn ValuePrettyPrinter>> =
+            LazyLock::new(|| Arc::new(OpaqueValuePrettyPrinter {}));
+        &PRETTY_PRINTER
+    }
+}
+
+// TODO Other canonical extension types.
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)]
+struct OpaqueValuePrettyPrinter;
+
+impl ValuePrettyPrinter for OpaqueValuePrettyPrinter {
+    fn pretty_print_scalar(&self, value: &ScalarValue) -> Result<String> {
+        Ok(format!("arrow.opaque({value})"))
+    }
+}
+
+/// Represents an unresolved extension type with a given native type and name.
+///
+/// This does not necessarily indicate that DataFusion does not understand the extension type. For
+/// this purpose, see [OpaqueType]. However, it does indicate that the extension type was not yet
+/// checked against the extension type registry.
+///
+/// This extension type exists because it is often challenging to gain access to an extension type
+/// registry. Especially because extension type support is relatively new, and therefore this
+/// consideration was not taken into account by users. This provides a workaround such that
+/// unresolved extension types can be resolved at a later point in time where access to the registry
+/// is available.
+pub struct UnresolvedExtensionType {
+    /// The name of the underlying extension type.
+    name: String,
+    /// The metadata of the underlying extension type.
+    metadata: Option<String>,
+    /// The underlying native type.
+    native_type: NativeType,
+}
+
+impl UnresolvedExtensionType {
+    /// Creates a new [UnresolvedExtensionType].
+    pub fn new(name: String, metadata: Option<String>, native_type: NativeType) -> Self {
+        Self {
+            name,
+            metadata,
+            native_type,
+        }
+    }
+
+    /// The name of the unresolved extension type.
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    /// The metadata of the unresolved extension type.
+    pub fn metadata(&self) -> Option<&str> {
+        self.metadata.as_deref()
+    }
+}
+
+impl LogicalType for UnresolvedExtensionType {
+    fn native(&self) -> &NativeType {
+        &self.native_type
+    }
+
+    fn signature(&self) -> TypeSignature<'_> {
+        let inner_type = TypeParameter::Type(TypeSignature::Extension {
+            name: &self.name,
+            parameters: vec![],
+        });
+        TypeSignature::Extension {
+            name: "datafusion.unresolved",
+            parameters: vec![inner_type],
+        }
+    }
+
+    fn pretty_printer(&self) -> &Arc<dyn ValuePrettyPrinter> {
+        static PRETTY_PRINTER: LazyLock<Arc<dyn ValuePrettyPrinter>> =
+            LazyLock::new(|| Arc::new(UnresolvedValuePrettyPrinter {}));
+        &PRETTY_PRINTER
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)]
+struct UnresolvedValuePrettyPrinter {}
+
+impl ValuePrettyPrinter for UnresolvedValuePrettyPrinter {
+    fn pretty_print_scalar(&self, value: &ScalarValue) -> Result<String> {
+        Ok(format!("datafusion.unresolved({value})"))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    pub fn test_pretty_print_uuid() {
+        let my_uuid = uuid::Uuid::nil();
+        let uuid = ScalarValue::FixedSizeBinary(16, Some(my_uuid.as_bytes().to_vec()));
+
+        let printer = UuidValuePrettyPrinter::default();
+        let pretty_printed = printer.pretty_print_scalar(&uuid).unwrap();
+        assert_eq!(
+            pretty_printed,
+            "arrow.uuid(00000000-0000-0000-0000-000000000000)"
+        );
+    }
+}

datafusion/common/src/types/extensions.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::Result;
+use crate::ScalarValue;
+use arrow::array::Array;
+use std::fmt::Debug;
+
+/// Implements pretty printing for a set of types.
+///
+/// For example, the default pretty-printer for a byte array might not be adequate for a UUID type,
+/// which is physically stored as a fixed-length byte array. This extension allows the user to
+/// override the default pretty-printer for a given type.
+pub trait ValuePrettyPrinter: Debug + Sync + Send {
+    /// Pretty print a scalar value.
+    ///
+    /// # Error
+    ///
+    /// Will return an error if the given `df_type` is not supported by this pretty printer.
+    fn pretty_print_scalar(&self, value: &ScalarValue) -> Result<String>;
+
+    /// Pretty print a specific value of a given array.
+    ///
+    /// # Error
+    ///
+    /// Will return an error if the given `df_type` is not supported by this pretty printer.
+    fn pretty_print_array(&self, array: &dyn Array, index: usize) -> Result<String> {
+        let value = ScalarValue::try_from_array(array, index)?;
+        self.pretty_print_scalar(&value)
+    }
+}
+
+/// The default pretty printer.
+///
+/// Uses the arrow implementation of printing values.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DefaultValuePrettyPrinter;
+
+impl ValuePrettyPrinter for DefaultValuePrettyPrinter {
+    fn pretty_print_scalar(&self, value: &ScalarValue) -> Result<String> {
+        Ok(value.to_string())
+    }
+}

datafusion/common/src/types/logical.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::NativeType;
+use super::{NativeType, ValuePrettyPrinter};
 use crate::error::Result;
 use arrow::datatypes::DataType;
 use core::fmt;
@@ -32,7 +32,7 @@ pub enum TypeSignature<'a> {
     /// The `name` should contain the same value as 'ARROW:extension:name'.
     Extension {
         name: &'a str,
-        parameters: &'a [TypeParameter<'a>],
+        parameters: Vec<TypeParameter<'a>>,
     },
 }
 
@@ -87,6 +87,9 @@ pub trait LogicalType: Sync + Send {
     fn default_cast_for(&self, origin: &DataType) -> Result<DataType> {
         self.native().default_cast_for(origin)
     }
+
+    /// Returns a pretty-printer that can format values of this type.
+    fn pretty_printer(&self) -> &Arc<dyn ValuePrettyPrinter>;
 }
 
 impl fmt::Debug for dyn LogicalType {

datafusion/common/src/types/mod.rs

@@ -16,11 +16,15 @@
 // under the License.
 
 mod builtin;
+mod canonical;
+mod extensions;
 mod field;
 mod logical;
 mod native;
 
 pub use builtin::*;
+pub use canonical::*;
+pub use extensions::*;
 pub use field::*;
 pub use logical::*;
 pub use native::*;

datafusion/common/src/types/native.rs

@@ -17,14 +17,16 @@
 
 use super::{
     LogicalField, LogicalFieldRef, LogicalFields, LogicalType, LogicalUnionFields,
-    TypeSignature,
+    TypeSignature, ValuePrettyPrinter,
 };
 use crate::error::{Result, _internal_err};
+use crate::types::DefaultValuePrettyPrinter;
 use arrow::compute::can_cast_types;
 use arrow::datatypes::{
     DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields,
     DECIMAL128_MAX_PRECISION, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION,
 };
+use std::sync::LazyLock;
 use std::{fmt::Display, sync::Arc};
 
 /// Representation of a type that DataFusion can handle natively. It is a subset
@@ -368,6 +370,12 @@ impl LogicalType for NativeType {
             }
         })
     }
+
+    fn pretty_printer(&self) -> &Arc<dyn ValuePrettyPrinter> {
+        static PRETTY_PRINTER: LazyLock<Arc<dyn ValuePrettyPrinter>> =
+            LazyLock::new(|| Arc::new(DefaultValuePrettyPrinter {}));
+        &PRETTY_PRINTER
+    }
 }
 
 // The following From<DataType>, From<Field>, ... implementations are temporary