Serialization

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 
 # Nim
 _*
+!_*.py
 spec/nim/bin
 
 # Rust compiled code, cargo and rls data

aggregate/convert_to_json

@@ -38,11 +38,27 @@ def add_fails(first, *seconds)
 end
 
 def add_kst_adoption(tests, log_dir)
-  kst_tests = File.readlines("#{log_dir}/kst_adoption.log")
+  kst_tests = File.readlines("#{log_dir}/kst_adoption.log").map { |name| name.chomp }
+  kst_test_used = {}
+  kst_tests.each { |name| kst_test_used[name] = false }
+  tests.each_pair { |name, test_result|
+    kst_name = nil
+    if kst_test_used.has_key?(name)
+      kst_name = name
+    elsif name =~ /^(.+?)\./ && kst_test_used.has_key?($1)
+      kst_name = $1
+    end
+
+    if kst_name
+      test_result['is_kst'] = true
+      kst_test_used[kst_name] = true
+    end
+  }
   kst_tests.each { |name|
-    name.chomp!
-    tests[name] ||= {}
-    tests[name]['is_kst'] = true
+    if !kst_test_used.fetch(name)
+      tests[name] ||= {}
+      tests[name]['is_kst'] = true
+    end
   }
   tests
 end
@@ -72,10 +88,10 @@ when 'cpp_stl'
     ),
     infile
   )
-when 'java'
+when 'java', 'java-write'
   add_kst_adoption(
     add_fails(
-      JUnitXMLParser.new("#{infile}/junitreports").to_h,
+      JUnitXMLParser.new("#{infile}/junitreports", lang == 'java-write').to_h,
       BuildFailedParser.new("#{infile}/build_failed_tests.txt").to_h
     ),
     infile
@@ -97,9 +113,9 @@ when 'javascript'
     JUnitXMLParser.new("#{infile}/test-output-javascript.xml").to_h,
     infile
   )
-when 'php', 'python', 'construct'
+when 'php', 'python', 'python-write', 'construct'
   add_kst_adoption(
-    JUnitXMLParser.new(infile).to_h,
+    JUnitXMLParser.new(infile, lang == 'python-write').to_h,
     infile
   )
 when 'ruby'

aggregate/junit_xml_parser.rb

@@ -3,7 +3,8 @@
 require 'rexml/document'
 
 class JUnitXMLParser < TestParser
-  def initialize(fn)
+  def initialize(fn, include_classname = false)
+    @include_classname = include_classname
     @docs = if not File.exist?(fn)
       []
     elsif File.directory?(fn)
@@ -35,10 +36,17 @@ def each_test
           name = underscore_to_ucamelcase($1)
         else
           raise "Unable to parse name: \"#{name}\"" unless name =~ /^[Tt]est(.*?)$/
-          if $1[0] == '_'
-            name = underscore_to_ucamelcase($1)
+          name = $1
+          if name[0] == '_'
+            name = underscore_to_ucamelcase(name[1..-1])
+          end
+
+          if @include_classname
+            classname = tc.attribute('classname').value
+            raise "Unable to parse classname: \"#{classname}\"" unless classname =~ /\.Test([^.]*)$/
+            classname = $1
+            name = "#{classname}.#{name}"
           else
-            name = $1
           end
         end
 

build-formats

@@ -1,16 +1,47 @@
-#!/bin/sh
+#!/bin/sh -e
 
 . ./config
 
 rm -rf "$FORMATS_COMPILED_DIR"
 mkdir -p "$FORMATS_COMPILED_DIR"
+
+# 	--java-from-file-class io.kaitai.struct.RandomAccessFileKaitaiStream \
+
 "$COMPILER_DIR/jvm/target/universal/stage/bin/kaitai-struct-compiler" -- \
 	--verbose all -t all -d "$FORMATS_COMPILED_DIR" \
 	--import-path "$FORMATS_REPO_DIR" \
 	--import-path "$FORMATS_KSY_DIR/ks_path" \
 	--java-package io.kaitai.struct.testformats \
 	--php-namespace 'Kaitai\Struct\Tests' \
+	--python-package testformats \
 	--go-package test_formats \
 	--nim-module "kaitai_struct_nim_runtime" \
 	--nim-opaque "../../tests/spec/nim/opaque/" \
 	"$FORMATS_KSY_DIR"/*.ksy || :
+
+mv -n compiled/python compiled/python-tmp
+mkdir compiled/python
+mv -n compiled/python-tmp compiled/python/testformats
+# __init__.py is needed in Python 2
+touch compiled/python/testformats/__init__.py
+
+"$COMPILER_DIR/jvm/target/universal/stage/bin/kaitai-struct-compiler" -- \
+	--verbose file -t java -d "$FORMATS_COMPILED_DIR/java/src" \
+	--no-auto-read \
+	--read-write \
+	--import-path "$FORMATS_REPO_DIR" \
+	--import-path "$FORMATS_KSY_DIR/ks_path" \
+	--java-package io.kaitai.struct.testwrite \
+	"$FORMATS_KSY_DIR"/*.ksy || :
+
+"$COMPILER_DIR/jvm/target/universal/stage/bin/kaitai-struct-compiler" -- \
+	--verbose file -t python -d "$FORMATS_COMPILED_DIR/python/testwrite" \
+	--no-auto-read \
+	--read-write \
+	--import-path "$FORMATS_REPO_DIR" \
+	--import-path "$FORMATS_KSY_DIR/ks_path" \
+	--python-package testwrite \
+	"$FORMATS_KSY_DIR"/*.ksy || :
+
+# __init__.py is needed in Python 2
+touch compiled/python/testwrite/__init__.py

builder/java_builder.rb

@@ -5,8 +5,11 @@
 require_relative 'shellconfig'
 
 class JavaBuilder < PartialBuilder
-  def initialize
-    super
+  def initialize(suite_names, test_out_dir)
+    super()
+
+    @suite_names = suite_names || 'spec,specwrite'
+    @test_out_dir = File.absolute_path(test_out_dir || "#{@config['TEST_OUT_DIR']}/java")
 
     @spec_dir     = File.join('spec', 'java', 'src')
     @compiled_dir = File.join("#{@config['FORMATS_COMPILED_DIR']}", 'java')
@@ -22,9 +25,6 @@ def initialize
     @java_classes_dir = "#{@compiled_dir}/bin"
     @java_classes_dir = File.absolute_path(@java_classes_dir)
     @java_classpath   = [@java_classes_dir, @java_testng_jar].join(File::PATH_SEPARATOR)
-
-    @test_out_dir = "#{@config['TEST_OUT_DIR']}/java"
-    @test_out_dir = File.absolute_path(@test_out_dir)
   end
 
   def list_mandatory_files
@@ -92,7 +92,9 @@ def run_tests
       'java',
       '-cp', @java_classpath,
       'org.testng.TestNG',
-      '-d', @test_out_dir, File.absolute_path(File.join('spec', 'java', 'testng.xml'))
+      '-d', @test_out_dir,
+      '-testnames', @suite_names,
+      File.absolute_path(File.join('spec', 'java', 'testng.xml'))
     ]
     out_log = File.join(@test_out_dir, 'test_run.stdout')
 

ci-java

@@ -2,7 +2,11 @@
 
 . ./config
 
-./run-java
+rm -rf "$TEST_OUT_DIR/java" "$TEST_OUT_DIR/java-write"
+./run-java spec "$TEST_OUT_DIR/java"
+./run-java specwrite "$TEST_OUT_DIR/java-write"
 
 ./kst-adoption-report java
+./kst-adoption-report java-write
 aggregate/convert_to_json java "$TEST_OUT_DIR/java" "$TEST_OUT_DIR/java/ci.json"
+aggregate/convert_to_json java-write "$TEST_OUT_DIR/java-write" "$TEST_OUT_DIR/java-write/ci.json"

ci-python

@@ -2,8 +2,11 @@
 
 . ./config
 
-rm -rf "$TEST_OUT_DIR/python"
-PYTHONPATH="$PYTHON_RUNTIME_DIR:compiled/python:spec/python/extra" PYTHONDONTWRITEBYTECODE=1 PYTHONDEVMODE=1 python ./run-python-xml.py spec/python "$TEST_OUT_DIR/python"
+rm -rf "$TEST_OUT_DIR/python" "$TEST_OUT_DIR/python-write"
+PYTHONPATH="$PYTHON_RUNTIME_DIR:compiled/python:spec/python/extra" PYTHONDONTWRITEBYTECODE=1 PYTHONDEVMODE=1 python ./run-python-xml.py spec/python/spec "$TEST_OUT_DIR/python"
+PYTHONPATH="$PYTHON_RUNTIME_DIR:compiled/python:spec/python:spec/python/extra" PYTHONDONTWRITEBYTECODE=1 PYTHONDEVMODE=1 python ./run-python-xml.py spec/python/specwrite "$TEST_OUT_DIR/python-write"
 
 ./kst-adoption-report python
+./kst-adoption-report python-write
 aggregate/convert_to_json python "$TEST_OUT_DIR/python" "$TEST_OUT_DIR/python/ci.json"
+aggregate/convert_to_json python-write "$TEST_OUT_DIR/python-write" "$TEST_OUT_DIR/python-write/ci.json"

formats/bits_byte_aligned_eof_be.ksy

@@ -0,0 +1,16 @@
+meta:
+  id: bits_byte_aligned_eof_be
+  bit-endian: be
+seq:
+  - id: prebuf
+    size: 8
+  - id: bits
+    type: b31
+  # 1 bit left until the end of file. This means that during serialization, when
+  # the stream is closed / `seek()` is called by the test code, the stream
+  # position must be "aligned" to a byte boundary so that the buffered bits are
+  # written into the final byte.
+  #
+  # We're specifically testing the alignment right before the EOF here to ensure
+  # that the EOF check in the implicitly called "write_align_to_byte()" method
+  # works properly, i.e. that it doesn't wrongly throw an EOF error here.

formats/bits_byte_aligned_eof_le.ksy

@@ -0,0 +1,11 @@
+# Like "bits_byte_aligned_eof_be", but for little-endian order.
+meta:
+  id: bits_byte_aligned_eof_le
+  bit-endian: le
+seq:
+  - id: prebuf
+    size: 8
+  - id: bits
+    type: b31
+    # 1 bit left until the end of file. (... - see "bits_byte_aligned_eof_be"
+    # for more details)

formats/bytes_eos_pad_term.ksy

@@ -0,0 +1,39 @@
+# Tests "pad-right" and "terminator" functionality in fixed-length "size-eos: true" byte arrays
+meta:
+  id: bytes_eos_pad_term
+seq:
+  - id: str_pad
+    size: 20
+    type: str_pad_type
+  - id: str_term
+    size: 20
+    type: str_term_type
+  - id: str_term_and_pad
+    size: 20
+    type: str_term_and_pad_type
+  - id: str_term_include
+    size: 20
+    type: str_term_include_type
+types:
+  str_pad_type:
+    seq:
+      - id: value
+        size-eos: true
+        pad-right: 0x40
+  str_term_type:
+    seq:
+      - id: value
+        size-eos: true
+        terminator: 0x40
+  str_term_and_pad_type:
+    seq:
+      - id: value
+        size-eos: true
+        terminator: 0x40
+        pad-right: 0x2b
+  str_term_include_type:
+    seq:
+      - id: value
+        size-eos: true
+        terminator: 0x40
+        include: true

formats/bytes_pad_term_empty.ksy

@@ -0,0 +1,19 @@
+# Same as "bytes_pad_term", but used with different input file that is
+# meant to test fully empty byte arrays (see "str_pad_term_empty")
+meta:
+  id: bytes_pad_term_empty
+seq:
+  - id: str_pad
+    size: 20
+    pad-right: 0x40
+  - id: str_term
+    size: 20
+    terminator: 0x40
+  - id: str_term_and_pad
+    size: 20
+    terminator: 0x40
+    pad-right: 0x2b
+  - id: str_term_include
+    size: 20
+    terminator: 0x40
+    include: true

formats/bytes_pad_term_equal.ksy

@@ -0,0 +1,23 @@
+# Tests "pad-right" and "terminator" functionality in fixed-length byte arrays
+# with `terminator` == `pad-right`
+meta:
+  id: bytes_pad_term_equal
+seq:
+  - id: s1
+    size: 20
+    terminator: 0x40
+    pad-right: 0x40
+  - id: s2
+    size: 20
+    terminator: 0x40
+    include: true
+    pad-right: 0x2b
+  - id: s3
+    size: 20
+    terminator: 0x2b
+    pad-right: 0x2b
+  - id: s4
+    size: 20
+    terminator: 0x2e
+    include: true
+    pad-right: 0x2e

formats/bytes_pad_term_roundtrip.ksy

@@ -0,0 +1,21 @@
+# Like "bytes_pad_term", but with added `pad-right`s to be able to test for an
+# exact match with the original file after serialization
+meta:
+  id: bytes_pad_term_roundtrip
+seq:
+  - id: str_pad
+    size: 20
+    pad-right: 0x40
+  - id: str_term
+    size: 20
+    terminator: 0x40
+    pad-right: 0x2b
+  - id: str_term_and_pad
+    size: 20
+    terminator: 0x40
+    pad-right: 0x2b
+  - id: str_term_include
+    size: 20
+    terminator: 0x40
+    include: true
+    pad-right: 0x2e

formats/bytes_pad_term_zero_size.ksy

@@ -0,0 +1,19 @@
+# Like "bytes_pad_term", but all `size`s are set to 0 (to make sure we don't
+# fail in this case)
+meta:
+  id: bytes_pad_term_zero_size
+seq:
+  - id: str_pad
+    size: 0
+    pad-right: 0x40
+  - id: str_term
+    size: 0
+    terminator: 0x40
+  - id: str_term_and_pad
+    size: 0
+    terminator: 0x40
+    pad-right: 0x2b
+  - id: str_term_include
+    size: 0
+    terminator: 0x40
+    include: true

formats/eof_exception_bits_be.ksy

@@ -0,0 +1,11 @@
+meta:
+  id: eof_exception_bits_be
+  bit-endian: be
+  ks-debug: true
+seq:
+  - id: pre_bits
+    type: b7
+  # only 2 full bytes (17 bits) available in the stream, should fail with EOF
+  # exception
+  - id: fail_bits
+    type: b18

formats/eof_exception_bits_be2.ksy

@@ -0,0 +1,11 @@
+meta:
+  id: eof_exception_bits_be2
+  bit-endian: be
+  ks-debug: true
+seq:
+  - id: pre_bits
+    type: b8
+  # only 2 full bytes (16 bits) available in the stream, should fail with EOF
+  # exception
+  - id: fail_bits
+    type: b17

formats/eof_exception_bits_le.ksy

@@ -0,0 +1,12 @@
+# Like "eof_exception_bits_be", but for little-endian order.
+meta:
+  id: eof_exception_bits_le
+  bit-endian: le
+  ks-debug: true
+seq:
+  - id: pre_bits
+    type: b7
+  # only 2 full bytes (17 bits) available in the stream, should fail with EOF
+  # exception
+  - id: fail_bits
+    type: b18