Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: json form fields and node counting #107

Merged
merged 26 commits into from
Jun 17, 2024
Merged

fix: json form fields and node counting #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
ff3d2ed
form from type functions
ianna Jun 10, 2024
e712951
add check on iterable type
ianna Jun 10, 2024
a0d377a
add test
ianna Jun 11, 2024
afaaae3
correct numpy type
ianna Jun 11, 2024
fd0f81b
fix offsets type
ianna Jun 11, 2024
11cd249
add test
ianna Jun 11, 2024
9974d57
add module
ianna Jun 11, 2024
965a33e
move function out
ianna Jun 11, 2024
88f92a2
add more functions
ianna Jun 13, 2024
d7b6766
try to fix Symbol issue
ianna Jun 13, 2024
e7967d5
add more tests
ianna Jun 13, 2024
be8e72a
cleanup and add more tests
ianna Jun 13, 2024
25c936d
add tree form test function
ianna Jun 13, 2024
4b890ce
add tests
ianna Jun 14, 2024
6711f70
fix FieldError
ianna Jun 14, 2024
296b6d6
use FieldError for nightly only
ianna Jun 14, 2024
5fb586c
fix typo
ianna Jun 14, 2024
9d4889a
one more
ianna Jun 14, 2024
59e803d
last one
ianna Jun 14, 2024
708120f
fix correct fields and node counting
ianna Jun 17, 2024
5d5fa00
Merge branch 'main' into ianna/fix_json_form
ianna Jun 17, 2024
36a1b89
fix: offsets from int64 to i64
ianna Jun 17, 2024
a575157
fix: more offsets
ianna Jun 17, 2024
8f4287e
fix: one more offset type
ianna Jun 17, 2024
8c1513d
fix: record form node count
ianna Jun 17, 2024
9a26c4e
fix: fields
ianna Jun 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/form_utils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ end
function type_to_form(::Type{Vector{T}}, form_key_id::Int64=0) where {T}
element_type = T
content_form = type_to_form(element_type, form_key_id + 1)
return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " *
return "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", " *
"\"content\": " * content_form * ", " *
"\"form_key\": \"node$(form_key_id)\"}"
end
Expand All @@ -148,7 +148,7 @@ function type_to_form(::Type{Vector{T}}, form_key_id_ref::Base.RefValue{Int64})
form_key = _generate_form_key!(form_key_id_ref)

content_form = type_to_form(element_type, form_key_id_ref)
return "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", " *
return "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", " *
"\"content\": " * content_form * ", " *
"\"form_key\": \"" * form_key * "\"}"
end
Expand Down Expand Up @@ -200,20 +200,21 @@ end

# Helper function for type_to_numpy_like (placeholder implementation)
function type_to_numpy_like(::Type{T}) where {T}
return "int64" # Placeholder implementation
return "i64" # Placeholder implementation
end

# A RecordArray form of all tree brunches
function tree_branches_type(tree, form_key_id::Int64=0)
form = """{"class": "RecordArray", "fields": ["""
form_fields = ""
form_contents = ""

id = form_key_id
id_ref = Ref(id)

form = """{"class": "RecordArray", "fields": ["""
form_key = _generate_form_key!(id_ref)

form_fields = ""
form_contents = ""
for name in propertynames(tree)
form_fields *= """$name, """
form_fields = form_fields * """\"$name\", """
branch = getproperty(tree, name)
branch_type = eltype(branch)
form_contents *= type_to_form(branch_type, id_ref) * ", "
Expand All @@ -225,7 +226,7 @@ function tree_branches_type(tree, form_key_id::Int64=0)

form *= form_fields * """], "contents": [""" * form_contents
form *= """], "parameters": {}, "form_key": \"""" *
_generate_form_key!(id_ref) * "\"}"
form_key * "\"}"

return form
end
Expand Down
50 changes: 25 additions & 25 deletions test/test_106_form_from_type.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,21 @@
@test AwkwardArray.type_to_form(Int32, 0) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Int64, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(Char, 1) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(String, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node2\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(String, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node2\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node1\"}"

@test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Int64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Int}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(Vector{Int32}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Int64}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Float32}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}"""
@test AwkwardArray.type_to_form(Vector{Float64}, 1) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}"""

@test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float32}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float64}}, 0) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node2"}, "form_key": "node1"}, "form_key": "node0"}"""

@test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}"
@test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, 1) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node2\"}, \"form_key\": \"node1\"}"

end

Expand All @@ -35,21 +35,21 @@ end
@test AwkwardArray.type_to_form(Int32, id_ref) == """{"class": "NumpyArray", "primitive": "int32", "form_key": "node4"}"""
@test AwkwardArray.type_to_form(Int64, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node5\"}"
@test AwkwardArray.type_to_form(Char, id_ref) == "{\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node6\"}"
@test AwkwardArray.type_to_form(String, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node8\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node7\"}"
@test AwkwardArray.type_to_form(String, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"uint8\", \"parameters\": { \"__array__\": \"char\" }, \"form_key\": \"node8\"}, \"parameters\": { \"__array__\": \"string\" }, \"form_key\": \"node7\"}"

@test AwkwardArray.type_to_form(Vector{Int}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node10\"}, \"form_key\": \"node9\"}"
@test AwkwardArray.type_to_form(Vector{Int32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node12"}, "form_key": "node11"}"""
@test AwkwardArray.type_to_form(Vector{Int64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node14"}, "form_key": "node13"}"""
@test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}"""
@test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}"""
@test AwkwardArray.type_to_form(Vector{Int}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int64\", \"form_key\": \"node10\"}, \"form_key\": \"node9\"}"
@test AwkwardArray.type_to_form(Vector{Int32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node12"}, "form_key": "node11"}"""
@test AwkwardArray.type_to_form(Vector{Int64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node14"}, "form_key": "node13"}"""
@test AwkwardArray.type_to_form(Vector{Float32}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node16"}, "form_key": "node15"}"""
@test AwkwardArray.type_to_form(Vector{Float64}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node18"}, "form_key": "node17"}"""

@test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "ListOffsetArray", "offsets": "int64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node21"}, "form_key": "node20"}, "form_key": "node19"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int32", "form_key": "node24"}, "form_key": "node23"}, "form_key": "node22"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Int64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "int64", "form_key": "node27"}, "form_key": "node26"}, "form_key": "node25"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float32}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float32", "form_key": "node30"}, "form_key": "node29"}, "form_key": "node28"}"""
@test AwkwardArray.type_to_form(Vector{Vector{Float64}}, id_ref) == """{"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "float64", "form_key": "node33"}, "form_key": "node32"}, "form_key": "node31"}"""

@test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"int64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}"
@test AwkwardArray.type_to_form(SubArray{Int32, 1, nothing}, id_ref) == "{\"class\": \"ListOffsetArray\", \"offsets\": \"i64\", \"content\": {\"class\": \"NumpyArray\", \"primitive\": \"int32\", \"form_key\": \"node35\"}, \"form_key\": \"node34\"}"

end

Expand Down Expand Up @@ -88,6 +88,6 @@ end
end

tree = TestTree([1, 2, 3], [4, 5, 6])
expected_form = """{"class": "RecordArray", "fields": [field1, field2], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node0"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}], "parameters": {}, "form_key": "node2"}"""
expected_form = """{"class": "RecordArray", "fields": ["field1", "field2"], "contents": [{"class": "NumpyArray", "primitive": "int64", "form_key": "node1"}, {"class": "NumpyArray", "primitive": "int64", "form_key": "node2"}], "parameters": {}, "form_key": "node0"}"""
@test AwkwardArray.tree_branches_type(tree) == expected_form
end
Loading