From 5849237fe7a06241d221c7fa164f85ff57b85a2d Mon Sep 17 00:00:00 2001 From: Naveen Narayanan Date: Tue, 5 Nov 2024 15:35:40 -0800 Subject: [PATCH 1/3] Add linux arm64 support to release --- .github/workflows/release.yml | 37 ++++++++++++++++++++++++++++++----- Cargo.toml | 1 + 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3fb612bc..34071ec3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,15 +87,23 @@ jobs: include: - os: ubuntu-20.04 dist-args: --artifacts=global + target: '' install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.4/cargo-dist-v0.0.4-installer.sh | sh - os: macos-11 dist-args: --artifacts=local --target=aarch64-apple-darwin --target=x86_64-apple-darwin + target: 'aarch64-apple-darwin x86_64-apple-darwin' install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.4/cargo-dist-v0.0.4-installer.sh | sh - os: ubuntu-20.04 dist-args: --artifacts=local --target=x86_64-unknown-linux-gnu + target: 'x86_64-unknown-linux-gnu' install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.4/cargo-dist-v0.0.4-installer.sh | sh + - os: ubuntu-20.04 + dist-args: --artifacts=local --target=aarch64-unknown-linux-gnu + target: 'aarch64-unknown-linux-gnu' + install-dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.4/cargo-dist-v0.0.4-installer.sh | sh - os: windows-2019 dist-args: --artifacts=local --target=x86_64-pc-windows-msvc + target: 'x86_64-pc-windows-msvc' install-dist: irm https://github.com/axodotdev/cargo-dist/releases/download/v0.0.4/cargo-dist-v0.0.4-installer.ps1 | iex runs-on: ${{ matrix.os }} @@ -108,17 +116,36 @@ jobs: - name: Install cargo-dist run: ${{ matrix.install-dist }} - name: Run cargo-dist + if: ${{ matrix.target != '' }} # This logic is a bit janky because it's trying to be a polyglot between # powershell and bash since this will run on windows, macos, and linux! # The two platforms don't agree on how to talk about env vars but they # do agree on 'cat' and '$()' so we use that to marshal values between commands. run: | - # Actually do builds and make zips and whatnot - cargo dist build --tag=${{ github.ref_name }} --output-format=json ${{ matrix.dist-args }} > dist-manifest.json - echo "dist ran successfully" + pip3 install ziglang + cargo install cargo-zigbuild + rustup target add ${{ matrix.target }} + cargo zigbuild --target ${{ matrix.target }} --release + + # Set binary name to typeshare + BINARY_NAME="typeshare" + TARGET_DIR="target/${{ matrix.target }}/release" + + # Create zip directory + mkdir -p "dist" + + # Create zip file with binary + ZIP_NAME="${BINARY_NAME}-${{ github.ref_name }}-${{ matrix.target }}.zip" + cd ${TARGET_DIR} && zip "../../../dist/${ZIP_NAME}" "${BINARY_NAME}${BINARY_SUFFIX}" + cd ../../.. + + # Create manifest file similar to cargo-dist + echo "{\"artifacts\": [{\"path\": \"dist/${ZIP_NAME}\"}]}" > dist-manifest.json + + echo "Build complete, contents of dist-manifest.json:" cat dist-manifest.json - - # Parse out what we just built and upload it to the Github Release™ + + # Upload to release cat dist-manifest.json | jq --raw-output ".artifacts[]?.path | select( . != null )" > uploads.txt echo "uploading..." cat uploads.txt diff --git a/Cargo.toml b/Cargo.toml index c11bcd32..17c5b553 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ ci = ["github"] installers = ["shell", "powershell"] # Target platforms to build apps for (Rust target-triple syntax) targets = [ + "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", From 16dc8973513064478d4ee4f8fa6ce0ae5042179f Mon Sep 17 00:00:00 2001 From: Naveen Narayanan Date: Tue, 5 Nov 2024 15:32:12 -0800 Subject: [PATCH 2/3] Support 64 bit integer --- .../can_override_disallowed_types/input.rs | 10 +++ .../can_override_disallowed_types/output.go | 9 +++ .../can_override_disallowed_types/output.kt | 11 ++++ .../output.scala | 19 ++++++ .../output.swift | 13 ++++ .../can_override_disallowed_types/output.ts | 5 ++ core/src/language/typescript.rs | 5 +- core/src/rust_types.rs | 7 ++- core/tests/agnostic_tests.rs | 63 ------------------- core/tests/snapshot_tests.rs | 1 + docs/src/usage/annotations.md | 56 +++++++++++++++++ 11 files changed, 132 insertions(+), 67 deletions(-) create mode 100644 core/data/tests/can_override_disallowed_types/input.rs create mode 100644 core/data/tests/can_override_disallowed_types/output.go create mode 100644 core/data/tests/can_override_disallowed_types/output.kt create mode 100644 core/data/tests/can_override_disallowed_types/output.scala create mode 100644 core/data/tests/can_override_disallowed_types/output.swift create mode 100644 core/data/tests/can_override_disallowed_types/output.ts diff --git a/core/data/tests/can_override_disallowed_types/input.rs b/core/data/tests/can_override_disallowed_types/input.rs new file mode 100644 index 00000000..01548305 --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/input.rs @@ -0,0 +1,10 @@ +#[typeshare] +struct DisallowedType { + #[typeshare(typescript(type = "bigint"))] + disallowed_type: u64, + #[typeshare(typescript(type = "number"))] + another_disallowed_type: i64, + #[typeshare(typescript(type = "string"))] + #[serde(with = "my_string_serde_impl")] + disallowed_type_serde_with: u64, +} diff --git a/core/data/tests/can_override_disallowed_types/output.go b/core/data/tests/can_override_disallowed_types/output.go new file mode 100644 index 00000000..69181b9c --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/output.go @@ -0,0 +1,9 @@ +package proto + +import "encoding/json" + +type DisallowedType struct { + DisallowedType uint64 `json:"disallowed_type"` + AnotherDisallowedType int64 `json:"another_disallowed_type"` + DisallowedTypeSerdeWith uint64 `json:"disallowed_type_serde_with"` +} diff --git a/core/data/tests/can_override_disallowed_types/output.kt b/core/data/tests/can_override_disallowed_types/output.kt new file mode 100644 index 00000000..0ad00b7e --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/output.kt @@ -0,0 +1,11 @@ +package com.agilebits.onepassword + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +data class DisallowedType ( + val disallowed_type: ULong, + val another_disallowed_type: Long, + val disallowed_type_serde_with: ULong +) diff --git a/core/data/tests/can_override_disallowed_types/output.scala b/core/data/tests/can_override_disallowed_types/output.scala new file mode 100644 index 00000000..43e81a32 --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/output.scala @@ -0,0 +1,19 @@ +package com.agilebits + +package object onepassword { + +type UByte = Byte +type UShort = Short +type UInt = Int +type ULong = Int + +} +package onepassword { + +case class DisallowedType ( + disallowed_type: ULong, + another_disallowed_type: Long, + disallowed_type_serde_with: ULong +) + +} diff --git a/core/data/tests/can_override_disallowed_types/output.swift b/core/data/tests/can_override_disallowed_types/output.swift new file mode 100644 index 00000000..385028a3 --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/output.swift @@ -0,0 +1,13 @@ +import Foundation + +public struct DisallowedType: Codable { + public let disallowed_type: UInt64 + public let another_disallowed_type: Int64 + public let disallowed_type_serde_with: UInt64 + + public init(disallowed_type: UInt64, another_disallowed_type: Int64, disallowed_type_serde_with: UInt64) { + self.disallowed_type = disallowed_type + self.another_disallowed_type = another_disallowed_type + self.disallowed_type_serde_with = disallowed_type_serde_with + } +} diff --git a/core/data/tests/can_override_disallowed_types/output.ts b/core/data/tests/can_override_disallowed_types/output.ts new file mode 100644 index 00000000..c8c24f42 --- /dev/null +++ b/core/data/tests/can_override_disallowed_types/output.ts @@ -0,0 +1,5 @@ +export interface DisallowedType { + disallowed_type: bigint; + another_disallowed_type: number; + disallowed_type_serde_with: string; +} diff --git a/core/src/language/typescript.rs b/core/src/language/typescript.rs index 184dc9db..dc71e908 100644 --- a/core/src/language/typescript.rs +++ b/core/src/language/typescript.rs @@ -81,7 +81,10 @@ impl Language for TypeScript { | SpecialRustType::I64 | SpecialRustType::ISize | SpecialRustType::USize => { - panic!("64 bit types not allowed in Typeshare") + panic!( + "64 bit integer types require an explicit output type. \ + See: https://1password.github.io/typeshare/usage/annotations.html#special-note-on-64-bit-integer-types for more information." + ) } } } diff --git a/core/src/rust_types.rs b/core/src/rust_types.rs index 4585b05f..eaeabec0 100644 --- a/core/src/rust_types.rs +++ b/core/src/rust_types.rs @@ -305,12 +305,13 @@ impl TryFrom<&syn::Type> for RustType { "u16" => Self::Special(SpecialRustType::U16), "u32" => Self::Special(SpecialRustType::U32), "U53" => Self::Special(SpecialRustType::U53), - "u64" | "i64" | "usize" | "isize" => { - return Err(RustTypeParseError::UnsupportedType(vec![id])) - } + "u64" => Self::Special(SpecialRustType::U64), + "usize" => Self::Special(SpecialRustType::USize), "i8" => Self::Special(SpecialRustType::I8), "i16" => Self::Special(SpecialRustType::I16), "i32" => Self::Special(SpecialRustType::I32), + "i64" => Self::Special(SpecialRustType::I64), + "isize" => Self::Special(SpecialRustType::ISize), "I54" => Self::Special(SpecialRustType::I54), "f32" => Self::Special(SpecialRustType::F32), "f64" => Self::Special(SpecialRustType::F64), diff --git a/core/tests/agnostic_tests.rs b/core/tests/agnostic_tests.rs index 347c742b..c4bddb81 100644 --- a/core/tests/agnostic_tests.rs +++ b/core/tests/agnostic_tests.rs @@ -2,7 +2,6 @@ use std::io::Write; use typeshare_core::{ language::{CrateTypes, Language, TypeScript}, parser::{self, ParseError}, - rust_types::RustTypeParseError, ProcessInputError, }; /// Parse and generate types for a single Rust input file. @@ -33,68 +32,6 @@ pub fn process_input( Ok(()) } -mod blocklisted_types { - use std::collections::HashMap; - - use super::*; - - fn assert_type_is_blocklisted(ty: &str, blocklisted_type: &str) { - let source = format!( - r##" - #[typeshare] - #[serde(default, rename_all = "camelCase")] - pub struct Foo {{ - pub bar: {ty}, - }} - "##, - ty = ty - ); - - let mut out: Vec = Vec::new(); - assert!(matches!( - process_input(&source, &mut TypeScript::default(), &HashMap::new(), &mut out), - Err(ProcessInputError::ParseError( - ParseError::RustTypeParseError(RustTypeParseError::UnsupportedType(contents)) - )) if contents == vec![blocklisted_type.to_owned()] - )); - } - - #[test] - fn test_i64_blocklisted_struct() { - assert_type_is_blocklisted("i64", "i64"); - } - - #[test] - fn test_u64_blocklisted_struct() { - assert_type_is_blocklisted("u64", "u64"); - } - - #[test] - fn test_isize_blocklisted_struct() { - assert_type_is_blocklisted("isize", "isize"); - } - - #[test] - fn test_usize_blocklisted_in_struct() { - assert_type_is_blocklisted("usize", "usize"); - } - - #[test] - fn test_optional_blocklisted_struct() { - assert_type_is_blocklisted("Option", "i64"); - } - - #[test] - fn test_vec_blocklisted_struct() { - assert_type_is_blocklisted("Vec", "i64"); - } - - #[test] - fn test_hashmap_blocklisted_struct() { - assert_type_is_blocklisted("HashMap", "i64"); - } -} - mod serde_attributes_on_enums { use std::collections::HashMap; diff --git a/core/tests/snapshot_tests.rs b/core/tests/snapshot_tests.rs index 6d9b887f..a8b8491b 100644 --- a/core/tests/snapshot_tests.rs +++ b/core/tests/snapshot_tests.rs @@ -502,6 +502,7 @@ tests! { go ]; can_override_types: [swift, kotlin, scala, typescript, go]; + can_override_disallowed_types: [swift, kotlin, scala, typescript, go]; /// Structs can_generate_simple_struct_with_a_comment: [kotlin, swift, typescript, scala, go]; diff --git a/docs/src/usage/annotations.md b/docs/src/usage/annotations.md index 7227619f..d7094489 100644 --- a/docs/src/usage/annotations.md +++ b/docs/src/usage/annotations.md @@ -103,6 +103,62 @@ This would generate the following Kotlin code: typealias Options = String ``` +### Override Type for a Field + +You can also use language-specific arguments to tell Typeshare to treat +a field as a type in a particular output language. For example, +```rust +#[typeshare] +struct MyStruct { + #[typeshare(typescript(type = "0 | 1"))] + oneOrZero: u8, +} +``` +would generate the following Typescript code: +```typescript +export interface MyStruct { + oneOrZero: 0 | 1; +} +``` +The `type` argument is supported for all output languages, however Typescript +also supports the optional `readonly` argument (e.g. `typescript(readonly, type= "0 | 1")`) +to make the output property readonly. + +### Special Note on 64 Bit Integer Types + +The default behavior for 64 bit integer types when outputting TypeScript is to +panic. The reasoning behind this is that in JavaScript runtimes integers are not +sufficient to fully represent the set of all 64 bit integers, that is, +`Number.MIN_SAFE_INTEGER` and `Number.MAX_SAFE_INTEGER` are less in magnitude +than `i64::MIN` and `u64::MAX`, respectively. There are a few ways one can still +use 64 bit integer types, however, and a Typeshare attribute to override the +field type can be applied to accommodate the particular approach one chooses to +take. Here are a few examples: + +**Serializing 64 bit integer fields to strings using `serde(with = ...)`** +```rust +struct MyStruct { + #[typeshare(typescript(type = "string"))] + #[serde(with = "my_string_serde_impl")] + my_field: u64 +} +``` + +**Using a third-party JSON parser that provides support for larger integer types via `bigint`** +```rust +struct MyStruct { + #[typeshare(typescript(type = "bigint"))] + my_field: u64 +} +``` + +**Throwing all caution to the wind and just using `number`** +```rust +struct MyStruct { + #[typeshare(typescript(type = "number"))] + my_field: u64 +} +``` ## The `#[serde]` Attribute From 4df7cc34aad39b7ab9e710cd4973951f3e170f98 Mon Sep 17 00:00:00 2001 From: Naveen Narayanan Date: Wed, 6 Nov 2024 13:33:04 -0800 Subject: [PATCH 3/3] [typeshare] Allow unsupported large numbers --- .../tests/can_override_disallowed_types/output.kt | 1 + .../tests/can_override_disallowed_types/output.ts | 1 + core/src/language/go.rs | 1 + core/src/language/kotlin.rs | 1 + core/src/language/scala.rs | 1 + core/src/language/swift.rs | 1 + core/src/language/typescript.rs | 11 ++++------- core/src/rust_types.rs | 8 ++++++++ core/src/visitors.rs | 15 +++++++++++---- 9 files changed, 29 insertions(+), 11 deletions(-) diff --git a/core/data/tests/can_override_disallowed_types/output.kt b/core/data/tests/can_override_disallowed_types/output.kt index 0ad00b7e..54159a8c 100644 --- a/core/data/tests/can_override_disallowed_types/output.kt +++ b/core/data/tests/can_override_disallowed_types/output.kt @@ -9,3 +9,4 @@ data class DisallowedType ( val another_disallowed_type: Long, val disallowed_type_serde_with: ULong ) + diff --git a/core/data/tests/can_override_disallowed_types/output.ts b/core/data/tests/can_override_disallowed_types/output.ts index c8c24f42..87b53f36 100644 --- a/core/data/tests/can_override_disallowed_types/output.ts +++ b/core/data/tests/can_override_disallowed_types/output.ts @@ -3,3 +3,4 @@ export interface DisallowedType { another_disallowed_type: number; disallowed_type_serde_with: string; } + diff --git a/core/src/language/go.rs b/core/src/language/go.rs index 1aeab1a4..6ac937d6 100644 --- a/core/src/language/go.rs +++ b/core/src/language/go.rs @@ -148,6 +148,7 @@ impl Language for Go { SpecialRustType::U32 => "uint32".into(), SpecialRustType::I54 | SpecialRustType::I64 => "int64".into(), SpecialRustType::U53 | SpecialRustType::U64 => "uint64".into(), + SpecialRustType::U128 => "uint128".into(), SpecialRustType::Bool => "bool".into(), SpecialRustType::F32 => "float32".into(), SpecialRustType::F64 => "float64".into(), diff --git a/core/src/language/kotlin.rs b/core/src/language/kotlin.rs index 011efe2d..1370d15a 100644 --- a/core/src/language/kotlin.rs +++ b/core/src/language/kotlin.rs @@ -90,6 +90,7 @@ impl Language for Kotlin { SpecialRustType::Bool => "Boolean".into(), SpecialRustType::F32 => "Float".into(), SpecialRustType::F64 => "Double".into(), + SpecialRustType::U128 => "BigInteger".into(), }) } diff --git a/core/src/language/scala.rs b/core/src/language/scala.rs index 9ba736d7..b7e3256f 100644 --- a/core/src/language/scala.rs +++ b/core/src/language/scala.rs @@ -111,6 +111,7 @@ impl Language for Scala { SpecialRustType::Bool => "Boolean".into(), SpecialRustType::F32 => "Float".into(), SpecialRustType::F64 => "Double".into(), + SpecialRustType::U128 => "BigInt".into(), }) } diff --git a/core/src/language/swift.rs b/core/src/language/swift.rs index 984ebe14..6b3a33ab 100644 --- a/core/src/language/swift.rs +++ b/core/src/language/swift.rs @@ -215,6 +215,7 @@ impl Language for Swift { SpecialRustType::Bool => "Bool".into(), SpecialRustType::F32 => "Float".into(), SpecialRustType::F64 => "Double".into(), + SpecialRustType::U128 => "UInt128".into(), }) } diff --git a/core/src/language/typescript.rs b/core/src/language/typescript.rs index dc71e908..149dc539 100644 --- a/core/src/language/typescript.rs +++ b/core/src/language/typescript.rs @@ -80,12 +80,8 @@ impl Language for TypeScript { SpecialRustType::U64 | SpecialRustType::I64 | SpecialRustType::ISize - | SpecialRustType::USize => { - panic!( - "64 bit integer types require an explicit output type. \ - See: https://1password.github.io/typeshare/usage/annotations.html#special-note-on-64-bit-integer-types for more information." - ) - } + | SpecialRustType::U128 + | SpecialRustType::USize => Ok("number".into()) } } @@ -181,8 +177,9 @@ impl Language for TypeScript { w: &mut dyn Write, imports: ScopedCrateTypes<'_>, ) -> std::io::Result<()> { + writeln!(w, "import type {{ HashSet }} from \"./base\";")?; for (path, ty) in imports { - write!(w, "import {{ ")?; + write!(w, "import type {{ ")?; let ty_list = ty.iter().join(", "); write!(w, "{ty_list}")?; writeln!(w, " }} from \"./{path}\";")?; diff --git a/core/src/rust_types.rs b/core/src/rust_types.rs index eaeabec0..fa91f5dc 100644 --- a/core/src/rust_types.rs +++ b/core/src/rust_types.rs @@ -1,3 +1,4 @@ +use log::debug; use quote::ToTokens; use std::collections::BTreeSet; use std::str::FromStr; @@ -206,6 +207,8 @@ pub enum SpecialRustType { I32, /// Represents `i64` I64, + /// Represents `u128` + U128, /// Represents `u8` U8, /// Represents `u16` @@ -257,6 +260,7 @@ impl TryFrom<&syn::Type> for RustType { type Error = RustTypeParseError; fn try_from(ty: &syn::Type) -> Result { + debug!("Parsing type: {}", ty.to_token_stream()); Ok(match ty { syn::Type::Tuple(tuple) if tuple.elems.iter().count() == 0 => { Self::Special(SpecialRustType::Unit) @@ -307,6 +311,7 @@ impl TryFrom<&syn::Type> for RustType { "U53" => Self::Special(SpecialRustType::U53), "u64" => Self::Special(SpecialRustType::U64), "usize" => Self::Special(SpecialRustType::USize), + "u128" => Self::Special(SpecialRustType::U128), "i8" => Self::Special(SpecialRustType::I8), "i16" => Self::Special(SpecialRustType::I16), "i32" => Self::Special(SpecialRustType::I32), @@ -489,6 +494,7 @@ impl SpecialRustType { | Self::F32 | Self::F64 | Self::I54 + | Self::U128 | Self::U53 => ty == self.id(), } } @@ -519,6 +525,7 @@ impl SpecialRustType { Self::USize => "usize", Self::U53 => "U53", Self::I54 => "I54", + Self::U128 => "u128", } } /// Iterate over the generic parameters for this type. Returns an empty iterator @@ -542,6 +549,7 @@ impl SpecialRustType { | Self::U16 | Self::U32 | Self::U64 + | Self::U128 | Self::ISize | Self::USize | Self::Bool diff --git a/core/src/visitors.rs b/core/src/visitors.rs index 131ccb05..639ae2c7 100644 --- a/core/src/visitors.rs +++ b/core/src/visitors.rs @@ -265,33 +265,38 @@ impl<'ast, 'a> Visit<'ast> for TypeShareVisitor<'a> { /// Collect rust structs. fn visit_item_struct(&mut self, i: &'ast syn::ItemStruct) { - debug!("Visiting {}", i.ident); + debug!("Visiting item {}", i.ident); if has_typeshare_annotation(&i.attrs) && self.target_os_accepted(&i.attrs) { debug!("\tParsing {}", i.ident); self.collect_result(parse_struct(i, self.target_os)); + debug!("\tParsed {}", i.ident); } - + debug!("Visited item {}", i.ident); syn::visit::visit_item_struct(self, i); } /// Collect rust enums. fn visit_item_enum(&mut self, i: &'ast syn::ItemEnum) { - debug!("Visiting {}", i.ident); + debug!("Visiting enum {}", i.ident); if has_typeshare_annotation(&i.attrs) && self.target_os_accepted(&i.attrs) { debug!("\tParsing {}", i.ident); self.collect_result(parse_enum(i, self.target_os)); + debug!("\tParsed {}", i.ident); } + debug!("Visited enum {}", i.ident); syn::visit::visit_item_enum(self, i); } /// Collect rust type aliases. fn visit_item_type(&mut self, i: &'ast syn::ItemType) { - debug!("Visiting {}", i.ident); + debug!("Visiting type {}", i.ident); if has_typeshare_annotation(&i.attrs) && self.target_os_accepted(&i.attrs) { debug!("\tParsing {}", i.ident); self.collect_result(parse_type_alias(i)); + debug!("\tParsed {}", i.ident); } + debug!("Visited type {}", i.ident); syn::visit::visit_item_type(self, i); } @@ -308,9 +313,11 @@ impl<'ast, 'a> Visit<'ast> for TypeShareVisitor<'a> { // } fn visit_file(&mut self, i: &'ast syn::File) { + debug!("Visiting file {}", self.parsed_data.file_name); if self.target_os_accepted(&i.attrs) { syn::visit::visit_file(self, i); } + debug!("Visited file {}", self.parsed_data.file_name); } }