Add support for byte translation

core/data/tests/test_byte_translation/input.rs

@@ -0,0 +1,5 @@
+#[typeshare]
+#[serde(rename_all = "camelCase")]
+pub struct Foo {
+    pub this_is_bits: Vec<u8>,
+}

core/data/tests/test_byte_translation/output.go

@@ -0,0 +1,7 @@
+package proto
+
+import "encoding/json"
+
+type Foo struct {
+	ThisIsBits []byte `json:"thisIsBits"`
+}

core/data/tests/test_byte_translation/output.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PlainSerializer
+from typing import Annotated
+
+
+def deserialize_binary_data(value):
+    if isinstance(value, list):
+        if all(isinstance(x, int) and 0 <= x <= 255 for x in value):
+            return bytes(value)
+        raise ValueError("All elements must be integers in the range 0-255 (u8).")
+    elif isinstance(value, bytes):
+            return value
+    raise TypeError("Content must be a list of integers (0-255) or bytes.")
+
+
+def serialize_binary_data(value: bytes) -> list[int]:
+    return list(value)
+
+class Foo(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    this_is_bits: Annotated[bytes, BeforeValidator(deserialize_binary_data), PlainSerializer(serialize_binary_data)] = Field(alias="thisIsBits")
+

core/data/tests/test_byte_translation/output.ts

@@ -0,0 +1,16 @@
+export interface Foo {
+	thisIsBits: Uint8Array;
+}
+
+export function ReviverFunc(key: string, value: unknown): unknown {
+    return Array.isArray(value) && value.every(v => Number.isInteger(v) && v >= 0 && v <= 255)  
+        ? new Uint8Array(value) 
+        : value;
+}
+
+export function ReplacerFunc(key: string, value: unknown): unknown {
+    if (value instanceof Uint8Array) {
+        return Array.from(value);
+    }
+    return value;
+}

core/data/tests/test_serde_iso8601/output.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pydantic import BaseModel
+
+
+class Foo(BaseModel):
+    time: datetime
+

core/src/language/go.rs

@@ -118,8 +118,14 @@ impl Language for Go {
         special_ty: &SpecialRustType,
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
+        if let Some(mapped) = self.type_map().get(&special_ty.to_string()) {
+            return Ok(mapped.to_owned());
+        };
+
         Ok(match special_ty {
-            SpecialRustType::Vec(rtype) => format!("[]{}", self.format_type(rtype, generic_types)?),
+            SpecialRustType::Vec(rtype) => {
+                format!("[]{}", self.format_type(rtype, generic_types)?)
+            }
             SpecialRustType::Array(rtype, len) => {
                 format!("[{}]{}", len, self.format_type(rtype, generic_types)?)
             }
@@ -491,14 +497,12 @@ func ({short_name} {full_name}) MarshalJSON() ([]byte, error) {{
         }
 
         write_comments(w, 1, &field.comments)?;
-
         let type_name = match field.type_override(SupportedLanguage::Go) {
             Some(type_override) => type_override.to_owned(),
             None => self
                 .format_type(&field.ty, generic_types)
                 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?,
         };
-
         let go_type = self.acronyms_to_uppercase(&type_name);
         let is_optional = field.ty.is_optional() || field.has_default;
         let formatted_renamed_id = format!("{:?}", &field.id.renamed);

core/src/language/python.rs

@@ -83,12 +83,17 @@ pub struct Python {
     /// Whether or not to exclude the version header that normally appears at the top of generated code.
     /// If you aren't generating a snapshot test, this setting can just be left as a default (false)
     pub no_version_header: bool,
+    /// Whether or not to include the serialization/deserialzation functions for bytes.
+    /// This by default should be false as unless the user expclitly wants to translate to its bytes
+    /// representation
+    pub should_translate_bytes: bool,
 }
 
 impl Language for Python {
     fn type_map(&mut self) -> &HashMap<String, String> {
         &self.type_mappings
     }
+
     fn generate_types(
         &mut self,
         w: &mut dyn Write,
@@ -124,9 +129,27 @@ impl Language for Python {
                 RustItem::Const(c) => self.write_const(&mut body, &c)?,
             };
         }
-
         self.write_all_imports(w)?;
 
+        if self.should_translate_bytes {
+            writeln!(
+                w,
+                r#"def deserialize_binary_data(value):
+    if isinstance(value, list):
+        if all(isinstance(x, int) and 0 <= x <= 255 for x in value):
+            return bytes(value)
+        raise ValueError("All elements must be integers in the range 0-255 (u8).")
+    elif isinstance(value, bytes):
+            return value
+    raise TypeError("Content must be a list of integers (0-255) or bytes.")
+
+
+def serialize_binary_data(value: bytes) -> list[int]:
+    return list(value)
+"#
+            )?;
+        }
+
         w.write_all(&body)?;
         Ok(())
     }
@@ -137,6 +160,7 @@ impl Language for Python {
         parameters: &[RustType],
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
+        self.add_imports(base);
         if let Some(mapped) = self.type_map().get(base) {
             Ok(mapped.into())
         } else {
@@ -173,10 +197,22 @@ impl Language for Python {
         special_ty: &SpecialRustType,
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
+        let mapped = if let Some(mapped) = self.type_map().get(&special_ty.to_string()) {
+            mapped.to_owned()
+        } else {
+            String::new()
+        };
         match special_ty {
-            SpecialRustType::Vec(rtype)
-            | SpecialRustType::Array(rtype, _)
-            | SpecialRustType::Slice(rtype) => {
+            SpecialRustType::Vec(rtype) => {
+                // TODO: https://github.com/1Password/typeshare/issues/231
+                if rtype.contains_type(SpecialRustType::U8.id()) && !mapped.is_empty() {
+                    self.should_translate_bytes = true;
+                    return Ok(mapped);
+                }
+                self.add_import("typing".to_string(), "List".to_string());
+                Ok(format!("List[{}]", self.format_type(rtype, generic_types)?))
+            }
+            SpecialRustType::Array(rtype, _) | SpecialRustType::Slice(rtype) => {
                 self.add_import("typing".to_string(), "List".to_string());
                 Ok(format!("List[{}]", self.format_type(rtype, generic_types)?))
             }
@@ -267,6 +303,7 @@ impl Language for Python {
     }
 
     fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> {
+        self.add_import("pydantic".to_string(), "BaseModel".to_string());
         {
             rs.generic_types
                 .iter()
@@ -293,9 +330,7 @@ impl Language for Python {
         if rs.fields.is_empty() {
             write!(w, "    pass")?
         }
-        writeln!(w)?;
-        self.add_import("pydantic".to_string(), "BaseModel".to_string());
-        Ok(())
+        writeln!(w)
     }
 
     fn write_enum(&mut self, w: &mut dyn Write, e: &RustEnum) -> std::io::Result<()> {
@@ -376,6 +411,25 @@ impl Python {
         }
     }
 
+    fn add_common_imports(
+        &mut self,
+        is_optional: bool,
+        requires_custom_translation: bool,
+        is_aliased: bool,
+    ) {
+        if is_optional {
+            self.add_import("typing".to_string(), "Optional".to_string());
+        }
+        if requires_custom_translation {
+            self.add_import("pydantic".to_string(), "BeforeValidator".to_string());
+            self.add_import("pydantic".to_string(), "PlainSerializer".to_string());
+            self.add_import("typing".to_string(), "Annotated".to_string());
+        }
+        if is_aliased || is_optional {
+            self.add_import("pydantic".to_string(), "Field".to_string());
+        }
+    }
+
     fn write_field(
         &mut self,
         w: &mut dyn Write,
@@ -390,48 +444,33 @@ impl Python {
             .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
         let python_field_name = python_property_aware_rename(&field.id.original);
         let is_aliased = python_field_name != field.id.renamed;
-        match (not_optional_but_default, is_aliased) {
-            (true, true) => {
-                self.add_import("typing".to_string(), "Optional".to_string());
-                self.add_import("pydantic".to_string(), "Field".to_string());
-                write!(w, "    {python_field_name}: Optional[{python_type}] = Field(alias=\"{renamed}\", default=None)", renamed=field.id.renamed)?;
-            }
-            (true, false) => {
-                self.add_import("typing".to_string(), "Optional".to_string());
-                self.add_import("pydantic".to_string(), "Field".to_string());
-                writeln!(
-                    w,
-                    "    {python_field_name}: Optional[{python_type}] = Field(default=None)"
-                )?
-            }
-            (false, true) => {
-                self.add_import("pydantic".to_string(), "Field".to_string());
-                write!(
-                    w,
-                    "    {python_field_name}: {python_type} = Field(alias=\"{renamed}\"",
-                    renamed = field.id.renamed
-                )?;
-                if is_optional {
-                    writeln!(w, ", default=None)")?;
-                } else {
-                    writeln!(w, ")")?;
-                }
-            }
-            (false, false) => {
-                write!(
-                    w,
-                    "    {python_field_name}: {python_type}",
-                    python_field_name = python_field_name,
-                    python_type = python_type
-                )?;
-                if is_optional {
-                    self.add_import("pydantic".to_string(), "Field".to_string());
-                    writeln!(w, " = Field(default=None)")?;
-                } else {
-                    writeln!(w)?;
-                }
+        let custom_translations = json_translation_for_type(&python_type);
+        // Adds all the required imports needed based off whether its optional ,aliased, or needs a byte translation
+        self.add_common_imports(is_optional, custom_translations.is_some(), is_aliased);
+
+        let python_field_type = match (not_optional_but_default, custom_translations) {
+            (true, _) => format!("Optional[{python_type}]") ,
+            (false, Some((serialize_function, deserialize_function))) => format!(
+                "Annotated[{python_type}, BeforeValidator({deserialize_function}), PlainSerializer({serialize_function})]"
+            ),
+            _ => python_type,
+        };
+
+        let python_return_value = match (not_optional_but_default, is_aliased, is_optional) {
+            (true, true, _) => format!(" = Field(alias=\"{}\", default=None)", field.id.renamed),
+            (true, false, _) => " = Field(default=None)".to_owned(),
+            (false, true, true) => {
+                format!(" = Field(alias=\"{}\", default=None)", field.id.renamed)
             }
-        }
+            (false, true, false) => format!(" = Field(alias=\"{}\")", field.id.renamed),
+            (false, false, true) => " = Field(default=None)".to_owned(),
+            (false, false, false) => String::new(),
+        };
+
+        writeln!(
+            w,
+            r#"    {python_field_name}: {python_field_type}{python_return_value}"#
+        )?;
 
         self.write_comments(w, true, &field.comments, 1)?;
         Ok(())
@@ -700,6 +739,22 @@ fn handle_model_config(w: &mut dyn Write, python_module: &mut Python, fields: &[
     };
 }
 
+/// acquires custom translation function names if custom serialize/deserialize functions are needed
+fn json_translation_for_type(python_type: &str) -> Option<(String, String)> {
+    // if more custom serialization/deserialization is needed, we can add it here and in the hashmap below
+    let custom_translations = HashMap::from([(
+        "bytes".to_owned(),
+        (
+            "serialize_binary_data".to_owned(),
+            "deserialize_binary_data".to_owned(),
+        ),
+    )]);
+
+    custom_translations
+        .get(python_type)
+        .map(|(s, d)| (s.to_owned(), d.to_owned()))
+}
+
 #[cfg(test)]
 mod test {
     use crate::rust_types::Id;

core/src/language/typescript.rs

@@ -24,20 +24,55 @@ pub struct TypeScript {
     /// Whether or not to exclude the version header that normally appears at the top of generated code.
     /// If you aren't generating a snapshot test, this setting can just be left as a default (false)
     pub no_version_header: bool,
+    /// Whether or not to include the reviver/replacer functions for Uint8Array.
+    /// This by default should be false as unless the user expclitly wants to translate to its Uint8Array
+    /// representation
+    pub should_translate_bytes: bool,
 }
 
 impl Language for TypeScript {
     fn type_map(&mut self) -> &HashMap<String, String> {
         &self.type_mappings
     }
 
+    fn end_file(&mut self, w: &mut dyn Write) -> std::io::Result<()> {
+        if self.should_translate_bytes {
+            return writeln!(
+                w,
+                r#"export function ReviverFunc(key: string, value: unknown): unknown {{
+    return Array.isArray(value) && value.every(v => Number.isInteger(v) && v >= 0 && v <= 255)  
+        ? new Uint8Array(value) 
+        : value;
+}}
+
+export function ReplacerFunc(key: string, value: unknown): unknown {{
+    if (value instanceof Uint8Array) {{
+        return Array.from(value);
+    }}
+    return value;
+}}"#
+            );
+        }
+        Ok(())
+    }
+
     fn format_special_type(
         &mut self,
         special_ty: &SpecialRustType,
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
+        let mapped = if let Some(mapped) = self.type_map().get(&special_ty.to_string()) {
+            mapped.to_owned()
+        } else {
+            String::new()
+        };
         match special_ty {
             SpecialRustType::Vec(rtype) => {
+                // TODO: https://github.com/1Password/typeshare/issues/231
+                if rtype.contains_type(SpecialRustType::U8.id()) && !mapped.is_empty() {
+                    self.should_translate_bytes = true;
+                    return Ok(mapped);
+                }
                 Ok(format!("{}[]", self.format_type(rtype, generic_types)?))
             }
             SpecialRustType::Array(rtype, len) => {