From ff3d2ede5b9204140d2209179982c2ef94cb9cf4 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 10 Jun 2024 16:41:06 +0200 Subject: [PATCH 01/25] form from type functions --- src/form_utils.jl | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/form_utils.jl diff --git a/src/form_utils.jl b/src/form_utils.jl new file mode 100644 index 0000000..b356da1 --- /dev/null +++ b/src/form_utils.jl @@ -0,0 +1,54 @@ +using AwkwardArray + +function type_to_form(::Type{T}, form_key_id::Int64) where {T} + # Helper function to generate the form key string + function generate_form_key!(form_key_id_ref) + form_key = "node" * string(form_key_id_ref[]) + form_key_id_ref[] += 1 + return form_key + end + + # Initialize form_key_id reference for mutability + form_key_id_ref = Ref(form_key_id) + + form_key = generate_form_key!(form_key_id_ref) + + if T <: Union{Int, Float64, Bool} + parameters = string(T, "\", ") + if T == Char + parameters = "uint8\", \"parameters\": { \"__array__\": \"char\" }, " + end + return "{\"class\": \"NumpyArray\", \"primitive\": \"" * parameters * + "\"form_key\": \"" * form_key * "\"}" + elseif T <: Complex + return "{\"class\": \"NumpyArray\", \"primitive\": \"" * string(T) * + "\", \"form_key\": \"" * form_key * "\"}" + else + error("Type not supported") + end +end + +function type_to_form(::Type{T}) where T <: Union{Int, Float64, String, Bool} + return AwkwardArray.check_primitive_type(T) +end + +function type_to_form(::Type{Vector{T}}) where T + return Vector{type_to_form(T)} +end + +function type_to_form(::Type{SubArray{T, N, A, I, L}}) where {T, N, A, I, L} + return SubArray{type_to_form(T), N, type_to_form(A), I, L} +end + +function tree_branches_type(tree) + for name in propertynames(tree) + branch = getproperty(tree, name) + branch_type = eltype(branch) + println("Is it a primitive type? ", AwkwardArray.isprimitive(branch_type)) + if branch_type <: SubArray + println(name, " ==>>> ", branch_type.parameters[3]) + else + println(name, " ==> ", branch_type) + end + end +end \ No newline at end of file From e712951dad8c138210c4fd931f976235e2c64669 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 10 Jun 2024 20:58:08 +0200 Subject: [PATCH 02/25] add check on iterable type --- src/form_utils.jl | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/form_utils.jl b/src/form_utils.jl index b356da1..697910f 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -1,5 +1,35 @@ using AwkwardArray +is_iterable(obj) = try + iterate(obj) !== nothing +catch + false +end + +# Function to check if a type is iterable +function is_iterable_type(T::Type) + try + iterate(T()) + return true + catch + return false + end +end + +function object_type_iterable(obj) + # Get the type of the object + T = typeof(obj) + + # Create an instance of the specified type + obj_instance = T() + + # Attempt to iterate over the object + iter_result = iterate(obj_instance) + + # Return true if iteration succeeds, false otherwise + return iter_result !== nothing +end + function type_to_form(::Type{T}, form_key_id::Int64) where {T} # Helper function to generate the form key string function generate_form_key!(form_key_id_ref) @@ -23,9 +53,31 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T} elseif T <: Complex return "{\"class\": \"NumpyArray\", \"primitive\": \"" * string(T) * "\", \"form_key\": \"" * form_key * "\"}" + end + + # Check if the type T is iterable + if is_iterable_type(T) + parameters = "" + + # Get the value type of the iterable T + value_type = eltype(T) + + if value_type == Char + parameters = " \"parameters\": { \"__array__\": \"string\" }, " + end + + form_key = "node$(form_key_id)" + + offsets = "offsets_placeholder" # Replace this with the actual logic to get offsets + + content = type_to_form(value_type, form_key_id + 1) # Recursive call for value_type + + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"$offsets\", \"content\": $content, $parameters \"form_key\": \"$form_key\"}" else error("Type not supported") end + + error("Type not supported") end function type_to_form(::Type{T}) where T <: Union{Int, Float64, String, Bool} From a0d377aad8d67408dd1387d5a55439d1c741c034 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 12:14:37 +0200 Subject: [PATCH 03/25] add test --- src/AwkwardArray.jl | 1 + test/runtests.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 3d3cbd5..d08dfbf 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -9,6 +9,7 @@ import Tables include("./all_implementations.jl") include("./tables.jl") +include("./form_utils.jl") include("./AwkwardPythonCallExt.jl") using .AwkwardPythonCallExt: convert diff --git a/test/runtests.jl b/test/runtests.jl index d6d2b25..f4977ff 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3721,3 +3721,4 @@ end # @testset "AwkwardArray.jl" end # @testset "Tables.jl" include("./runpytests.jl") +include("./test_106_form_from_type.jl") From afaaae328251270d71468e2dec1b48bc9d995142 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 12:28:56 +0200 Subject: [PATCH 04/25] correct numpy type --- src/form_utils.jl | 4 ++-- test/test_106_form_from_type.jl | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 test/test_106_form_from_type.jl diff --git a/src/form_utils.jl b/src/form_utils.jl index 697910f..8e55382 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -44,14 +44,14 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T} form_key = generate_form_key!(form_key_id_ref) if T <: Union{Int, Float64, Bool} - parameters = string(T, "\", ") + parameters = string(AwkwardArray.check_primitive_type(T), "\", ") if T == Char parameters = "uint8\", \"parameters\": { \"__array__\": \"char\" }, " end return "{\"class\": \"NumpyArray\", \"primitive\": \"" * parameters * "\"form_key\": \"" * form_key * "\"}" elseif T <: Complex - return "{\"class\": \"NumpyArray\", \"primitive\": \"" * string(T) * + return "{\"class\": \"NumpyArray\", \"primitive\": \"" * AwkwardArray.check_primitive_type(T) * "\", \"form_key\": \"" * form_key * "\"}" end diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl new file mode 100644 index 0000000..dde523a --- /dev/null +++ b/test/test_106_form_from_type.jl @@ -0,0 +1,6 @@ +using AwkwardArray + +@testset "Form from type" begin + form = type_to_form(Int, 1) + @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" +end \ No newline at end of file From fd0f81b4d1dd7d388af7edbec23e4565eaed5994 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 12:42:43 +0200 Subject: [PATCH 05/25] fix offsets type --- src/form_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index 8e55382..de5f867 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -68,7 +68,7 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T} form_key = "node$(form_key_id)" - offsets = "offsets_placeholder" # Replace this with the actual logic to get offsets + offsets = "i64" # Replace this with the actual logic to get offsets content = type_to_form(value_type, form_key_id + 1) # Recursive call for value_type From 11cd24908a53b41b4732764d950bb799488a1648 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 12:45:08 +0200 Subject: [PATCH 06/25] add test --- test/test_106_form_from_type.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index dde523a..af91a79 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -1,6 +1,8 @@ using AwkwardArray @testset "Form from type" begin - form = type_to_form(Int, 1) - @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" + begin + form = type_to_form(Int, 1) + @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" + end end \ No newline at end of file From 9974d57261f570ecc908bfa49792bbee99888054 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 13:00:51 +0200 Subject: [PATCH 07/25] add module --- test/test_106_form_from_type.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index af91a79..1dfdc09 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -2,7 +2,7 @@ using AwkwardArray @testset "Form from type" begin begin - form = type_to_form(Int, 1) + form = AwkwardArray.type_to_form(Int, 1) @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" end end \ No newline at end of file From 965a33e4f8e410cf90c11c0aceb5bdecb1442355 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 11 Jun 2024 14:18:51 +0200 Subject: [PATCH 08/25] move function out --- src/form_utils.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index de5f867..e8e08a0 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -30,14 +30,14 @@ function object_type_iterable(obj) return iter_result !== nothing end -function type_to_form(::Type{T}, form_key_id::Int64) where {T} - # Helper function to generate the form key string - function generate_form_key!(form_key_id_ref) - form_key = "node" * string(form_key_id_ref[]) - form_key_id_ref[] += 1 - return form_key - end +# Function to generate form key +function generate_form_key!(form_key_id_ref::Base.RefValue{Int64}) + form_key_id = form_key_id_ref[] + form_key_id_ref[] += 1 + return "node$form_key_id" +end +function type_to_form(::Type{T}, form_key_id::Int64) where {T} # Initialize form_key_id reference for mutability form_key_id_ref = Ref(form_key_id) From 88f92a24b3eb5704cd37e35687cc56ec31220d0d Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 13 Jun 2024 16:05:41 +0200 Subject: [PATCH 09/25] add more functions --- src/form_utils.jl | 152 ++++++++++++++++++++------------ test/runtests.jl | 4 +- test/test_106_form_from_type.jl | 18 +++- 3 files changed, 114 insertions(+), 60 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index e8e08a0..25aad2c 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -1,21 +1,9 @@ -using AwkwardArray - is_iterable(obj) = try iterate(obj) !== nothing catch false end -# Function to check if a type is iterable -function is_iterable_type(T::Type) - try - iterate(T()) - return true - catch - return false - end -end - function object_type_iterable(obj) # Get the type of the object T = typeof(obj) @@ -30,6 +18,30 @@ function object_type_iterable(obj) return iter_result !== nothing end +# Define a dictionary mapping Julia types to NumPy types +julia_to_numpy = Dict( + Int8 => "int8", + UInt8 => "uint8", + Int16 => "int16", + UInt16 => "uint16", + Int32 => "int32", + UInt32 => "uint32", + Int64 => "int64", + UInt64 => "uint64", + Float16 => "float16", + Float32 => "float32", + Float64 => "float64", + Bool => "bool", + Complex{Float32} => "complex64", + Complex{Float64} => "complex128", + String => "str" +) + +# Function to get the corresponding NumPy type +function julia_to_numpy_type(julia_type::Type) + return get(julia_to_numpy, julia_type, "unknown") +end + # Function to generate form key function generate_form_key!(form_key_id_ref::Base.RefValue{Int64}) form_key_id = form_key_id_ref[] @@ -37,59 +49,87 @@ function generate_form_key!(form_key_id_ref::Base.RefValue{Int64}) return "node$form_key_id" end -function type_to_form(::Type{T}, form_key_id::Int64) where {T} - # Initialize form_key_id reference for mutability - form_key_id_ref = Ref(form_key_id) +function json_numpy_form(parameters::String, form_key::String) + return "{\"class\": \"NumpyArray\", \"primitive\": \"" * parameters * + "\"form_key\": \"" * form_key * "\"}" +end - form_key = generate_form_key!(form_key_id_ref) +# Method for handling primitive types +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Integer} + form_key = "node$(form_key_id)" + form_key_id += 1 - if T <: Union{Int, Float64, Bool} - parameters = string(AwkwardArray.check_primitive_type(T), "\", ") - if T == Char - parameters = "uint8\", \"parameters\": { \"__array__\": \"char\" }, " - end - return "{\"class\": \"NumpyArray\", \"primitive\": \"" * parameters * - "\"form_key\": \"" * form_key * "\"}" - elseif T <: Complex - return "{\"class\": \"NumpyArray\", \"primitive\": \"" * AwkwardArray.check_primitive_type(T) * - "\", \"form_key\": \"" * form_key * "\"}" - end + parameters = string(julia_to_numpy_type(T), "\", ") - # Check if the type T is iterable - if is_iterable_type(T) - parameters = "" - - # Get the value type of the iterable T - value_type = eltype(T) - - if value_type == Char - parameters = " \"parameters\": { \"__array__\": \"string\" }, " - end - - form_key = "node$(form_key_id)" - - offsets = "i64" # Replace this with the actual logic to get offsets - - content = type_to_form(value_type, form_key_id + 1) # Recursive call for value_type - - return "{\"class\": \"ListOffsetArray\", \"offsets\": \"$offsets\", \"content\": $content, $parameters \"form_key\": \"$form_key\"}" - else - error("Type not supported") - end - - error("Type not supported") + return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{T}) where T <: Union{Int, Float64, String, Bool} - return AwkwardArray.check_primitive_type(T) +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractFloat} + form_key = "node$(form_key_id)" + form_key_id += 1 + + parameters = string(julia_to_numpy_type(T), "\", ") + return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{Vector{T}}) where T - return Vector{type_to_form(T)} +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Bool} + form_key = "node$(form_key_id)" + form_key_id += 1 + + parameters = string(julia_to_numpy_type(T), "\", ") + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Char} + form_key = "node$(form_key_id)" + form_key_id += 1 + + parameters = "uint8\", \"parameters\": { \"__array__\": \"char\" }, " + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: String} + value_type = eltype(T) + form_key = "node$(form_key_id)" + form_key_id += 1 + + parameters = " \"parameters\": { \"__array__\": \"string\" }, " + + content = type_to_form(value_type, form_key_id) + + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * + "\"content\":" * content * ", " * parameters * + "\"form_key\": \"" * form_key * "\"}" +end + +# Method for handling iterable types +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Vector} + value_type = eltype(T) + form_key = "node$(form_key_id)" + form_key_id += 1 + + parameters = "" + if value_type == Char + parameters = " \"parameters\": { \"__array__\": \"string\" }, " + end + + content = type_to_form(value_type, form_key_id) + + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * + "\"content\":" * content * ", " * parameters * + "\"form_key\": \"" * form_key * "\"}" +end + +# Fallback method for unsupported types +function type_to_form(::Type{T}, form_key_id::Int64) where {T} + error("Type '$T' is not supported yet.") end -function type_to_form(::Type{SubArray{T, N, A, I, L}}) where {T, N, A, I, L} - return SubArray{type_to_form(T), N, type_to_form(A), I, L} +# Helper function for type_to_numpy_like (placeholder implementation) +function type_to_numpy_like(::Type{T}) where {T} + return "int64" # Placeholder implementation end function tree_branches_type(tree) diff --git a/test/runtests.jl b/test/runtests.jl index f4977ff..7571131 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using JSON using Test using Tables +include("./test_106_form_from_type.jl") ### PrimitiveArray ####################################################### @testset "PrimitiveArray" begin @@ -3720,5 +3721,4 @@ end # @testset "AwkwardArray.jl" end end # @testset "Tables.jl" -include("./runpytests.jl") -include("./test_106_form_from_type.jl") +# include("./runpytests.jl") diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 1dfdc09..0a3b301 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -1,8 +1,22 @@ -using AwkwardArray +using AwkwardArray: type_to_form @testset "Form from type" begin begin - form = AwkwardArray.type_to_form(Int, 1) + id = 1 + ref = Ref(id) + out = AwkwardArray.generate_form_key!(ref) + @test out == "node1" + end + begin + form = type_to_form(Int, 1) @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" end + begin + form = type_to_form(Int32, 1) + @test form == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node1"}""" + end + begin + form = type_to_form(Vector{Int32}, 1) + @test form == """{"class": "ListOffsetArray", "offsets": "int64", "content":{"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}""" + end end \ No newline at end of file From d7b676633e45769a9d099d3ae58bd36b7b0b70ac Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 13 Jun 2024 16:37:19 +0200 Subject: [PATCH 10/25] try to fix Symbol issue --- src/form_utils.jl | 5 +++-- test/runtests.jl | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index 25aad2c..5fae47e 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -19,7 +19,7 @@ function object_type_iterable(obj) end # Define a dictionary mapping Julia types to NumPy types -julia_to_numpy = Dict( +const julia_to_numpy = Dict( Int8 => "int8", UInt8 => "uint8", Int16 => "int16", @@ -39,7 +39,8 @@ julia_to_numpy = Dict( # Function to get the corresponding NumPy type function julia_to_numpy_type(julia_type::Type) - return get(julia_to_numpy, julia_type, "unknown") + result = get(julia_to_numpy, julia_type, "unknown") + return String(result) end # Function to generate form key diff --git a/test/runtests.jl b/test/runtests.jl index 7571131..727b064 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,8 +3,6 @@ using JSON using Test using Tables -include("./test_106_form_from_type.jl") - ### PrimitiveArray ####################################################### @testset "PrimitiveArray" begin @@ -3721,4 +3719,5 @@ end # @testset "AwkwardArray.jl" end end # @testset "Tables.jl" -# include("./runpytests.jl") +include("./runpytests.jl") +include("./test_106_form_from_type.jl") From e7967d5d504812cef3395eb5d873ca26a35e3282 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 13 Jun 2024 17:51:35 +0200 Subject: [PATCH 11/25] add more tests --- src/form_utils.jl | 19 +++++++++-------- test/test_106_form_from_type.jl | 37 ++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index 5fae47e..07d3650 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -44,7 +44,7 @@ function julia_to_numpy_type(julia_type::Type) end # Function to generate form key -function generate_form_key!(form_key_id_ref::Base.RefValue{Int64}) +function _generate_form_key!(form_key_id_ref::Base.RefValue{Int64}) form_key_id = form_key_id_ref[] form_key_id_ref[] += 1 return "node$form_key_id" @@ -55,12 +55,12 @@ function json_numpy_form(parameters::String, form_key::String) "\"form_key\": \"" * form_key * "\"}" end -# Method for handling primitive types +# Function for handling primitive types function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Integer} form_key = "node$(form_key_id)" form_key_id += 1 - parameters = string(julia_to_numpy_type(T), "\", ") + parameters = julia_to_numpy_type(T) * "\", " return json_numpy_form(parameters, form_key) end @@ -69,7 +69,8 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractFloat} form_key = "node$(form_key_id)" form_key_id += 1 - parameters = string(julia_to_numpy_type(T), "\", ") + parameters = julia_to_numpy_type(T) * "\", " + return json_numpy_form(parameters, form_key) end @@ -77,7 +78,7 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Bool} form_key = "node$(form_key_id)" form_key_id += 1 - parameters = string(julia_to_numpy_type(T), "\", ") + parameters = julia_to_numpy_type(T) * "\", " return json_numpy_form(parameters, form_key) end @@ -101,11 +102,11 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: String} content = type_to_form(value_type, form_key_id) return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * - "\"content\":" * content * ", " * parameters * + "\"content\": " * content * ", " * parameters * "\"form_key\": \"" * form_key * "\"}" end -# Method for handling iterable types +# Function for handling iterable types function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Vector} value_type = eltype(T) form_key = "node$(form_key_id)" @@ -119,11 +120,11 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Vector} content = type_to_form(value_type, form_key_id) return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * - "\"content\":" * content * ", " * parameters * + "\"content\": " * content * ", " * parameters * "\"form_key\": \"" * form_key * "\"}" end -# Fallback method for unsupported types +# Fallback function for unsupported types function type_to_form(::Type{T}, form_key_id::Int64) where {T} error("Type '$T' is not supported yet.") end diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 0a3b301..35609ab 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -1,22 +1,29 @@ -using AwkwardArray: type_to_form @testset "Form from type" begin + @test AwkwardArray.type_to_form(Bool, 1) == """{"class": "NumpyArray", "primitive": "bool", "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Int, 1) == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Int32, 1) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Int32, 0) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Int64, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node1\"}" + + @test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Int64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}""" + + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" +end + +@testset "Key generate" begin begin id = 1 ref = Ref(id) - out = AwkwardArray.generate_form_key!(ref) + out = AwkwardArray._generate_form_key!(ref) @test out == "node1" end - begin - form = type_to_form(Int, 1) - @test form == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" - end - begin - form = type_to_form(Int32, 1) - @test form == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node1"}""" - end - begin - form = type_to_form(Vector{Int32}, 1) - @test form == """{"class": "ListOffsetArray", "offsets": "int64", "content":{"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}""" - end -end \ No newline at end of file +end From be8e72a1ff7dcd6d024fb5b690a92d659a77c860 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 13 Jun 2024 21:13:47 +0200 Subject: [PATCH 12/25] cleanup and add more tests --- src/form_utils.jl | 19 ------------------- test/test_106_form_from_type.jl | 2 ++ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index 07d3650..cdf67bd 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -1,22 +1,3 @@ -is_iterable(obj) = try - iterate(obj) !== nothing -catch - false -end - -function object_type_iterable(obj) - # Get the type of the object - T = typeof(obj) - - # Create an instance of the specified type - obj_instance = T() - - # Attempt to iterate over the object - iter_result = iterate(obj_instance) - - # Return true if iteration succeeds, false otherwise - return iter_result !== nothing -end # Define a dictionary mapping Julia types to NumPy types const julia_to_numpy = Dict( diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 35609ab..9d630b0 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -5,6 +5,8 @@ @test AwkwardArray.type_to_form(Int32, 1) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node1"}""" @test AwkwardArray.type_to_form(Int32, 0) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node0"}""" @test AwkwardArray.type_to_form(Int64, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(Char, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(String, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node2\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node1\"}" @test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" @test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}""" From 25c936d6c62b8b47ec5f69f18b8a63e3cfb5fef1 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 13 Jun 2024 22:30:42 +0200 Subject: [PATCH 13/25] add tree form test function --- src/form_utils.jl | 31 +++++++++++++++++++++++++------ test/test_106_form_from_type.jl | 1 + 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index cdf67bd..eada25f 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -87,8 +87,17 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: String} "\"form_key\": \"" * form_key * "\"}" end +# Method to handle specific Vector types +function type_to_form(::Type{Vector{T}}, form_key_id::Int64) where {T} + element_type = T + content_form = type_to_form(element_type, form_key_id + 1) + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " * + "\"content\": " * content_form * ", " * + "\"form_key\": \"node$(form_key_id)\"}" +end + # Function for handling iterable types -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Vector} +function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractVector} value_type = eltype(T) form_key = "node$(form_key_id)" form_key_id += 1 @@ -115,15 +124,25 @@ function type_to_numpy_like(::Type{T}) where {T} return "int64" # Placeholder implementation end -function tree_branches_type(tree) +# Print forms of all tree brunches +function tree_branches_type(tree, form_key_id::Int64=0) + println("""{"class": "RecordArray", "fields": [""") + for name in propertynames(tree) + println(""""$name", """) + end + println("""], "contents": [""") + for name in propertynames(tree) branch = getproperty(tree, name) branch_type = eltype(branch) - println("Is it a primitive type? ", AwkwardArray.isprimitive(branch_type)) + if branch_type <: SubArray - println(name, " ==>>> ", branch_type.parameters[3]) + form_type = branch_type.parameters[3] + println(type_to_form(form_type, 0), ", ") else - println(name, " ==> ", branch_type) + println(type_to_form(branch_type, 0), ", ") end end -end \ No newline at end of file + println(" ]") +end + diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 9d630b0..4b50646 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -27,5 +27,6 @@ end ref = Ref(id) out = AwkwardArray._generate_form_key!(ref) @test out == "node1" + @test ref[] == 2 end end From 4b890cec211fc2e054b5e58e7b7e0c506a914fd1 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 12:49:41 +0200 Subject: [PATCH 14/25] add tests --- src/form_utils.jl | 136 ++++++++++++++++++++++++++------ test/test_106_form_from_type.jl | 69 +++++++++++++++- 2 files changed, 175 insertions(+), 30 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index eada25f..abb2fe0 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -33,11 +33,11 @@ end function json_numpy_form(parameters::String, form_key::String) return "{\"class\": \"NumpyArray\", \"primitive\": \"" * parameters * - "\"form_key\": \"" * form_key * "\"}" + "\"form_key\": \"" * form_key * "\"}" end # Function for handling primitive types -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Integer} +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: Integer} form_key = "node$(form_key_id)" form_key_id += 1 @@ -46,7 +46,15 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Integer} return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractFloat} +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: Integer} + form_key = _generate_form_key!(form_key_id_ref) + + parameters = julia_to_numpy_type(T) * "\", " + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: AbstractFloat} form_key = "node$(form_key_id)" form_key_id += 1 @@ -55,7 +63,15 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractFloat} return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Bool} +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: AbstractFloat} + form_key = _generate_form_key!(form_key_id_ref) + + parameters = julia_to_numpy_type(T) * "\", " + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: Bool} form_key = "node$(form_key_id)" form_key_id += 1 @@ -64,7 +80,15 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Bool} return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Char} +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: Bool} + form_key = _generate_form_key!(form_key_id_ref) + + parameters = julia_to_numpy_type(T) * "\", " + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: Char} form_key = "node$(form_key_id)" form_key_id += 1 @@ -73,7 +97,15 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: Char} return json_numpy_form(parameters, form_key) end -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: String} +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: Char} + form_key = _generate_form_key!(form_key_id_ref) + + parameters = "uint8\", \"parameters\": { \"__array__\": \"char\" }, " + + return json_numpy_form(parameters, form_key) +end + +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: String} value_type = eltype(T) form_key = "node$(form_key_id)" form_key_id += 1 @@ -82,13 +114,28 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: String} content = type_to_form(value_type, form_key_id) - return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * + type_to_numpy_like(T) * "\", " * + "\"content\": " * content * ", " * parameters * + "\"form_key\": \"" * form_key * "\"}" +end + +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: String} + value_type = eltype(T) + form_key = _generate_form_key!(form_key_id_ref) + + parameters = " \"parameters\": { \"__array__\": \"string\" }, " + + content = type_to_form(value_type, form_key_id_ref) + + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * + type_to_numpy_like(T) * "\", " * "\"content\": " * content * ", " * parameters * "\"form_key\": \"" * form_key * "\"}" end -# Method to handle specific Vector types -function type_to_form(::Type{Vector{T}}, form_key_id::Int64) where {T} +# Function to handle specific Vector types +function type_to_form(::Type{Vector{T}}, form_key_id::Int64=0) where {T} element_type = T content_form = type_to_form(element_type, form_key_id + 1) return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " * @@ -96,8 +143,18 @@ function type_to_form(::Type{Vector{T}}, form_key_id::Int64) where {T} "\"form_key\": \"node$(form_key_id)\"}" end +function type_to_form(::Type{Vector{T}}, form_key_id_ref::Base.RefValue{Int64}) where {T} + element_type = T + form_key = _generate_form_key!(form_key_id_ref) + + content_form = type_to_form(element_type, form_key_id_ref) + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " * + "\"content\": " * content_form * ", " * + "\"form_key\": \"" * form_key * "\"}" +end + # Function for handling iterable types -function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractVector} +function type_to_form(::Type{T}, form_key_id::Int64=0) where {T <: AbstractVector} value_type = eltype(T) form_key = "node$(form_key_id)" form_key_id += 1 @@ -109,13 +166,35 @@ function type_to_form(::Type{T}, form_key_id::Int64) where {T <: AbstractVector} content = type_to_form(value_type, form_key_id) - return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * type_to_numpy_like(T) * "\", " * + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * + type_to_numpy_like(T) * "\", " * + "\"content\": " * content * ", " * parameters * + "\"form_key\": \"" * form_key * "\"}" +end + +function type_to_form(::Type{T}, form_key_id_ref::Base.RefValue{Int64}) where {T <: AbstractVector} + value_type = eltype(T) + form_key = _generate_form_key!(form_key_id_ref) + + parameters = "" + if value_type == Char + parameters = " \"parameters\": { \"__array__\": \"string\" }, " + end + + content = type_to_form(value_type, form_key_id_ref) + + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"" * + type_to_numpy_like(T) * "\", " * "\"content\": " * content * ", " * parameters * "\"form_key\": \"" * form_key * "\"}" end # Fallback function for unsupported types -function type_to_form(::Type{T}, form_key_id::Int64) where {T} +function type_to_form(::Type{T}, ::Int64) where {T} + error("Type '$T' is not supported yet.") +end + +function type_to_form(::Type{T}, ::Base.RefValue{Int64}) where {T} error("Type '$T' is not supported yet.") end @@ -124,25 +203,30 @@ function type_to_numpy_like(::Type{T}) where {T} return "int64" # Placeholder implementation end -# Print forms of all tree brunches +# A RecordArray form of all tree brunches function tree_branches_type(tree, form_key_id::Int64=0) - println("""{"class": "RecordArray", "fields": [""") - for name in propertynames(tree) - println(""""$name", """) - end - println("""], "contents": [""") + form = """{"class": "RecordArray", "fields": [""" + form_fields = "" + form_contents = "" + + id = form_key_id + id_ref = Ref(id) for name in propertynames(tree) + form_fields *= """$name, """ branch = getproperty(tree, name) branch_type = eltype(branch) - - if branch_type <: SubArray - form_type = branch_type.parameters[3] - println(type_to_form(form_type, 0), ", ") - else - println(type_to_form(branch_type, 0), ", ") - end + form_contents *= type_to_form(branch_type, id_ref) * ", " end - println(" ]") + + # Removing the trailing comma and space + form_fields = replace(rstrip(form_fields), r",\s*$" => "") + form_contents = replace(rstrip(form_contents), r",\s*$" => "") + + form *= form_fields * """], "contents": [""" * form_contents + form *= """], "parameters": {}, "form_key": \"""" * + _generate_form_key!(id_ref) * "\"}" + + return form end diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 4b50646..a552d80 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -1,5 +1,6 @@ @testset "Form from type" begin + @test AwkwardArray.type_to_form(Bool, 1) == """{"class": "NumpyArray", "primitive": "bool", "form_key": "node1"}""" @test AwkwardArray.type_to_form(Int, 1) == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}""" @test AwkwardArray.type_to_form(Int32, 1) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node1"}""" @@ -19,14 +20,74 @@ @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + + @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" + +end + +@testset "Form from type with an id reference" begin + id = 1 + id_ref = Ref(id) + + @test AwkwardArray.type_to_form(Bool, id_ref) == """{"class": "NumpyArray", "primitive": "bool", "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Int, id_ref) == """{"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}""" + @test AwkwardArray.type_to_form(Int32, id_ref) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node3"}""" + @test AwkwardArray.type_to_form(Int32, id_ref) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node4"}""" + @test AwkwardArray.type_to_form(Int64, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node5\"}" + @test AwkwardArray.type_to_form(Char, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node6\"}" + @test AwkwardArray.type_to_form(String, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node8\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node7\"}" + + @test AwkwardArray.type_to_form(Vector{Int}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node10\"}, \"form_key\": \"node9\"}" + @test AwkwardArray.type_to_form(Vector{Int32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node12"}, "form_key": "node11"}""" + @test AwkwardArray.type_to_form(Vector{Int64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node14"}, "form_key": "node13"}""" + @test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}""" + @test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}""" + + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}""" + + @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}" + +end + +@testset "type_to_form error tests" begin + try + AwkwardArray.type_to_form(Any, 1) + @test false # This line should never be reached + catch e + @test isa(e, ErrorException) + @test occursin("Type 'Any' is not supported yet.", e.msg) + end + + try + AwkwardArray.type_to_form(Nothing, 1) + @test false # This line should never be reached + catch e + @test isa(e, ErrorException) + @test occursin("Type 'Nothing' is not supported yet.", e.msg) + end end @testset "Key generate" begin begin id = 1 - ref = Ref(id) - out = AwkwardArray._generate_form_key!(ref) - @test out == "node1" - @test ref[] == 2 + id_ref = Ref(id) + node_key = AwkwardArray._generate_form_key!(id_ref) + @test node_key == "node1" + @test id_ref[] == 2 + end +end + +@testset "tree_branches_type tests" begin + mutable struct TestTree + field1::Vector{Int} + field2::Vector{Int} end + + tree = TestTree([1, 2, 3], [4, 5, 6]) + expected_form = """{"class": "RecordArray", "fields": [field1, field2], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node0"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}], "parameters": {}, "form_key": "node2"}""" + @test AwkwardArray.tree_branches_type(tree) == expected_form end From 6711f70fe198a484e1f015f3cefb445d4f7c4373 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 16:30:43 +0200 Subject: [PATCH 15/25] fix FieldError --- test/runtests.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 727b064..12faab0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using JSON using Test using Tables + ### PrimitiveArray ####################################################### @testset "PrimitiveArray" begin @@ -312,7 +313,7 @@ end layout = AwkwardArray.ListOffsetArray([1, 2, 5], content_layout) @test layout[:a] == [[2], [3, 4, 5]] - @test_throws ErrorException getindex(layout, :invalid) + @test_throws FieldError getindex(layout, :invalid) @test_throws AssertionError getindex(layout[:a], :invalid) end From 296b6d682588e0554e1662343026b975bf3403b0 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 16:39:30 +0200 Subject: [PATCH 16/25] use FieldError for nightly only --- test/runtests.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 12faab0..70fc9c9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -313,7 +313,12 @@ end layout = AwkwardArray.ListOffsetArray([1, 2, 5], content_layout) @test layout[:a] == [[2], [3, 4, 5]] - @test_throws FieldError getindex(layout, :invalid) + + if VERSION >= v"1.12.0-DEV" + @test_throws FieldError getindex(layout, :invalid) + else + @test_throws ErrorException getindex(layout, :invalid) + @test_throws AssertionError getindex(layout[:a], :invalid) end From 5fb586c4ab59fb8dbd42e3654135117320df3fc9 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 16:42:28 +0200 Subject: [PATCH 17/25] fix typo --- test/runtests.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 70fc9c9..121da5c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -318,7 +318,8 @@ end @test_throws FieldError getindex(layout, :invalid) else @test_throws ErrorException getindex(layout, :invalid) - + end + @test_throws AssertionError getindex(layout[:a], :invalid) end From 9d4889a11070f684cd27464491fc39ba99acc965 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 16:47:48 +0200 Subject: [PATCH 18/25] one more --- test/runtests.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 121da5c..089e4c4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -319,7 +319,7 @@ end else @test_throws ErrorException getindex(layout, :invalid) end - + @test_throws AssertionError getindex(layout[:a], :invalid) end @@ -496,7 +496,12 @@ end layout = AwkwardArray.ListArray([1, 2, 5], [2, 5, 5], content_layout) @test layout[:a] == [[2], [3, 4, 5], []] - @test_throws ErrorException getindex(layout, :invalid) + if VERSION >= v"1.12.0-DEV" + @test_throws FieldError getindex(layout, :invalid) + else + @test_throws ErrorException getindex(layout, :invalid) + end + @test_throws AssertionError getindex(layout[:a], :invalid) end end From 59e803da5554a3486a0b551033e01b8254308df2 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 14 Jun 2024 17:18:01 +0200 Subject: [PATCH 19/25] last one --- test/runtests.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 089e4c4..2f3a7f3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -720,7 +720,12 @@ end layout = AwkwardArray.RegularArray(content_layout, 2) @test layout[:a] == [[1, 2], [3, 4]] - @test_throws ErrorException getindex(layout, :invalid) + if VERSION >= v"1.12.0-DEV" + @test_throws FieldError getindex(layout, :invalid) + else + @test_throws ErrorException getindex(layout, :invalid) + end + @test_throws AssertionError getindex(layout[:a], :invalid) end end From 708120f39f777726b999bb32778aa7312eb93388 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 20:50:42 +0200 Subject: [PATCH 20/25] fix correct fields and node counting --- src/form_utils.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index abb2fe0..61bdcc9 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -200,20 +200,21 @@ end # Helper function for type_to_numpy_like (placeholder implementation) function type_to_numpy_like(::Type{T}) where {T} - return "int64" # Placeholder implementation + return "i64" # Placeholder implementation end # A RecordArray form of all tree brunches function tree_branches_type(tree, form_key_id::Int64=0) - form = """{"class": "RecordArray", "fields": [""" - form_fields = "" - form_contents = "" - id = form_key_id id_ref = Ref(id) + form = """{"class": "RecordArray", "fields": [""" + form_key = _generate_form_key!(id_ref) + + form_fields = "" + form_contents = "" for name in propertynames(tree) - form_fields *= """$name, """ + form_fields = form_fields * """\"$name\", """ branch = getproperty(tree, name) branch_type = eltype(branch) form_contents *= type_to_form(branch_type, id_ref) * ", " @@ -225,7 +226,7 @@ function tree_branches_type(tree, form_key_id::Int64=0) form *= form_fields * """], "contents": [""" * form_contents form *= """], "parameters": {}, "form_key": \"""" * - _generate_form_key!(id_ref) * "\"}" + form_key * "\"}" return form end From 36a1b89f9e73d31754d67c35cbefad39ea880f58 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 21:50:49 +0200 Subject: [PATCH 21/25] fix: offsets from int64 to i64 --- test/test_106_form_from_type.jl | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index a552d80..87d33d1 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -7,21 +7,21 @@ @test AwkwardArray.type_to_form(Int32, 0) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node0"}""" @test AwkwardArray.type_to_form(Int64, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node1\"}" @test AwkwardArray.type_to_form(Char, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node1\"}" - @test AwkwardArray.type_to_form(String, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node2\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(String, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node2\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node1\"}" - @test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" - @test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}""" - @test AwkwardArray.type_to_form(Vector{Int64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}""" - @test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}""" - @test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Int64}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}""" + @test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" + @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" end @@ -35,21 +35,21 @@ end @test AwkwardArray.type_to_form(Int32, id_ref) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node4"}""" @test AwkwardArray.type_to_form(Int64, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node5\"}" @test AwkwardArray.type_to_form(Char, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node6\"}" - @test AwkwardArray.type_to_form(String, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node8\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node7\"}" + @test AwkwardArray.type_to_form(String, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node8\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node7\"}" - @test AwkwardArray.type_to_form(Vector{Int}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node10\"}, \"form_key\": \"node9\"}" - @test AwkwardArray.type_to_form(Vector{Int32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node12"}, "form_key": "node11"}""" - @test AwkwardArray.type_to_form(Vector{Int64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node14"}, "form_key": "node13"}""" - @test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}""" - @test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}""" + @test AwkwardArray.type_to_form(Vector{Int}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node10\"}, \"form_key\": \"node9\"}" + @test AwkwardArray.type_to_form(Vector{Int32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node12"}, "form_key": "node11"}""" + @test AwkwardArray.type_to_form(Vector{Int64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node14"}, "form_key": "node13"}""" + @test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}""" + @test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}""" - @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}" + @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}" end From a57515723b11eead985ea12d0f8678681ffa5fa4 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 21:54:24 +0200 Subject: [PATCH 22/25] fix: more offsets --- test/test_106_form_from_type.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index 87d33d1..d036f74 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -15,11 +15,11 @@ @test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}""" @test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}""" @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}" @@ -43,11 +43,11 @@ end @test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}""" @test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}""" - @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}""" + @test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}""" @test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}" From 8f4287eb4b48b8a9d4799419f0917b0bcfe695e3 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 21:59:46 +0200 Subject: [PATCH 23/25] fix: one more offset type --- src/form_utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/form_utils.jl b/src/form_utils.jl index 61bdcc9..a5c54d5 100644 --- a/src/form_utils.jl +++ b/src/form_utils.jl @@ -138,7 +138,7 @@ end function type_to_form(::Type{Vector{T}}, form_key_id::Int64=0) where {T} element_type = T content_form = type_to_form(element_type, form_key_id + 1) - return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " * + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", " * "\"content\": " * content_form * ", " * "\"form_key\": \"node$(form_key_id)\"}" end @@ -148,7 +148,7 @@ function type_to_form(::Type{Vector{T}}, form_key_id_ref::Base.RefValue{Int64}) form_key = _generate_form_key!(form_key_id_ref) content_form = type_to_form(element_type, form_key_id_ref) - return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " * + return "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", " * "\"content\": " * content_form * ", " * "\"form_key\": \"" * form_key * "\"}" end From 8c1513de1551d63ea4582fc35fdfd40d49239e77 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 22:05:04 +0200 Subject: [PATCH 24/25] fix: record form node count --- test/test_106_form_from_type.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index d036f74..f83ca32 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -88,6 +88,6 @@ end end tree = TestTree([1, 2, 3], [4, 5, 6]) - expected_form = """{"class": "RecordArray", "fields": [field1, field2], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node0"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}], "parameters": {}, "form_key": "node2"}""" + expected_form = """{"class": "RecordArray", "fields": [field1, field2], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}], "parameters": {}, "form_key": "node0"}""" @test AwkwardArray.tree_branches_type(tree) == expected_form end From 9a26c4e6badd0649b9f14e60efe8ce0f65e0f027 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 17 Jun 2024 22:09:33 +0200 Subject: [PATCH 25/25] fix: fields --- test/test_106_form_from_type.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_106_form_from_type.jl b/test/test_106_form_from_type.jl index f83ca32..8676e1e 100644 --- a/test/test_106_form_from_type.jl +++ b/test/test_106_form_from_type.jl @@ -88,6 +88,6 @@ end end tree = TestTree([1, 2, 3], [4, 5, 6]) - expected_form = """{"class": "RecordArray", "fields": [field1, field2], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}], "parameters": {}, "form_key": "node0"}""" + expected_form = """{"class": "RecordArray", "fields": ["field1", "field2"], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}], "parameters": {}, "form_key": "node0"}""" @test AwkwardArray.tree_branches_type(tree) == expected_form end