diff --git a/lib/zlib.toit b/lib/zlib.toit index bec4aff09..da494d2fb 100644 --- a/lib/zlib.toit +++ b/lib/zlib.toit @@ -136,6 +136,9 @@ class RunLengthDeflateEncoder_ extends ZlibEncoder_: from += read buffer_fullness_ += written + /** + Closes the encoder for writing. + */ close: channel_.send (buffer_.copy 0 buffer_fullness_) try: @@ -164,6 +167,171 @@ class RunLengthGzipEncoder extends RunLengthDeflateEncoder_: constructor: super CrcAndLengthChecksum_ --gzip_header=true +/** +Object that can be read to get output from an $Encoder or a $Decoder. +*/ +class ZlibReader implements reader.Reader: + owner_/Coder_? := null + + constructor.private_: + + /** + Reads output data. + In the default $wait mode this method may block in order to let a + writing task write more data to the compressor or decompressor. + In the non-blocking mode, if the compressor or decompressor has run out of + input data, this method returns a zero length byte array. + If the compressor or decompressor has been closed, and there is no more output + data, this method returns null. + */ + read --wait/bool=true -> ByteArray?: + return owner_.read_ --wait=wait + + close -> none: + owner_.close_read_ + +// An Encoder or Decoder. +abstract class Coder_: + zlib_ ::= ? + closed_write_ := false + closed_read_ := false + signal_ /monitor.Signal := monitor.Signal + state_/int := STATE_READY_TO_READ_ | STATE_READY_TO_WRITE_ + + static STATE_READY_TO_READ_ ::= 1 + static STATE_READY_TO_WRITE_ ::= 2 + + constructor .zlib_: + reader = ZlibReader.private_ + reader.owner_ = this + add_finalizer this:: + this.uninit_ + + /** + A reader that can be used to read the compressed or decompressed data output + by the Encoder or Decoder. + */ + reader/ZlibReader + + read_ --wait/bool -> ByteArray?: + if closed_read_: return null + result := zlib_read_ zlib_ + while result and wait and result.size == 0: + state_ &= ~STATE_READY_TO_READ_ + signal_.wait: state_ & STATE_READY_TO_READ_ != 0 + result = zlib_read_ zlib_ + state_ |= STATE_READY_TO_WRITE_ + signal_.raise + return result + + close_read_ -> none: + state_ |= STATE_READY_TO_WRITE_ + signal_.raise + if not closed_read_: + closed_read_ = true + if closed_write_: + uninit_ + + write --wait/bool=true data -> int: + if closed_read_: throw "READER_CLOSED" + pos := 0 + while pos < data.size: + bytes_written := zlib_write_ zlib_ data[pos..] + if bytes_written == 0: + if wait: + state_ &= ~STATE_READY_TO_WRITE_ + signal_.wait: state_ & STATE_READY_TO_WRITE_ != 0 + else: + state_ |= STATE_READY_TO_READ_ + signal_.raise + if not wait: return bytes_written + pos += bytes_written + return pos + + close -> none: + if not closed_write_: + zlib_close_ zlib_ + closed_write_ = true + state_ |= Coder_.STATE_READY_TO_READ_ + signal_.raise + + /** + Releases memory associated with this compressor. This is called + automatically when this object and the reader have both been closed. + */ + uninit_ -> none: + remove_finalizer this + zlib_close_ zlib_ + +/** +A Zlib compressor/deflater. +Not usually supported on embedded platforms due to high memory use. +*/ +class Encoder extends Coder_: + /** + Creates a new compressor. + The compression level can be -1 for default, 0 for no compression, or 1-9 for + compression levels 1-9. + */ + constructor --level/int=-1: + if not -1 <= level <= 9: throw "ILLEGAL_ARGUMENT" + super (zlib_init_deflate_ resource_freeing_module_ level) + + /** + Writes uncompressed data into the compressor. + In the default $wait mode this method may block and will not return + until all bytes have been written to the compressor. + Returns the number of bytes that were compressed. If zero bytes were + compressed that means that data needs to be read using the reader before + more data can be accepted. + Any bytes that were not compressed need to be resubmitted to this method + later. + */ + write --wait/bool=true data -> int: + return super --wait=wait data + + /** + Closes the encoder. + This tells the encoder that no more uncompressed input is coming. Subsequent + calls to the reader will return the buffered compressed data and then + return null. + */ + close -> none: + super + +/** +A Zlib decompressor/inflater. +Not usually supported on embedded platforms due to high memory use. +*/ +class Decoder extends Coder_: + /** + Creates a new decompressor. + */ + constructor: + super (zlib_init_inflate_ resource_freeing_module_) + + /** + Writes compressed data into the decompressor. + In the default $wait mode this method may block and will not return + until all bytes have been written to the decompressor. + Returns the number of bytes that were decompressed. If zero bytes were + decompressed that means that data needs to be read using the reader before + more data can be accepted. + Any bytes that were not decompressed need to be resubmitted to this method + later. + */ + write --wait/bool=true data -> int: + return super --wait=wait data + + /** + Closes the decoder. + This will tell the decoder that no more compressed input is coming. + Subsequent calls to the reader will return the buffered decompressed data + and then return null. + */ + close -> none: + super + rle_start_ group: #primitive.zlib.rle_start @@ -179,3 +347,21 @@ rle_add_ rle destination index source from to: /// Returns the number of bytes written to terminate the zlib stream. rle_finish_ rle destination index: #primitive.zlib.rle_finish + +zlib_init_deflate_ group level/int: + #primitive.zlib.zlib_init_deflate + +zlib_init_inflate_ group: + #primitive.zlib.zlib_init_inflate + +zlib_read_ zlib -> ByteArray?: + #primitive.zlib.zlib_read + +zlib_write_ zlib data -> int: + #primitive.zlib.zlib_write + +zlib_close_ zlib -> none: + #primitive.zlib.zlib_close + +zlib_uninit_ zlib -> none: + #primitive.zlib.zlib_uninit diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c8195a3a7..fdbed6fdc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -186,6 +186,10 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(TOIT_BLUETOOTH_LIBS "-framework Foundation" "-framework CoreBluetooth" ObjC) endif() +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin" OR (${CMAKE_SYSTEM_NAME} MATCHES "Linux" AND NOT CMAKE_SIZEOF_VOID_P EQUAL 4 AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "ARM")) + set(TOIT_Z_LIB z) +endif() + # Because of the `CACHE INTERNAL ""` at the end of the `set` we can # use this variable outside of the directory. set(TOIT_LINK_LIBS @@ -198,6 +202,7 @@ set(TOIT_LINK_LIBS ${TOIT_LINK_GROUP_END_FLAGS} toit_compiler # TODO(florian): should not be here by default. pthread + ${TOIT_Z_LIB} ${CMAKE_DL_LIBS} ${TOIT_LINK_LIBS_LIBGCC} ${TOIT_LINK_SEGFAULT} diff --git a/src/compiler/propagation/type_primitive_zlib.cc b/src/compiler/propagation/type_primitive_zlib.cc index 59349c6f2..16aa7093d 100644 --- a/src/compiler/propagation/type_primitive_zlib.cc +++ b/src/compiler/propagation/type_primitive_zlib.cc @@ -27,6 +27,12 @@ TYPE_PRIMITIVE_ANY(adler32_clone) TYPE_PRIMITIVE_ANY(rle_start) TYPE_PRIMITIVE_ANY(rle_add) TYPE_PRIMITIVE_ANY(rle_finish) +TYPE_PRIMITIVE_ANY(zlib_init_deflate) +TYPE_PRIMITIVE_ANY(zlib_init_inflate) +TYPE_PRIMITIVE_ANY(zlib_write) +TYPE_PRIMITIVE_ANY(zlib_read) +TYPE_PRIMITIVE_NULL(zlib_close) +TYPE_PRIMITIVE_NULL(zlib_uninit) } // namespace toit::compiler } // namespace toit diff --git a/src/primitive.h b/src/primitive.h index a22b910f1..00f5b1aff 100644 --- a/src/primitive.h +++ b/src/primitive.h @@ -644,6 +644,12 @@ namespace toit { PRIMITIVE(rle_start, 1) \ PRIMITIVE(rle_add, 6) \ PRIMITIVE(rle_finish, 3) \ + PRIMITIVE(zlib_init_deflate, 2) \ + PRIMITIVE(zlib_init_inflate, 1) \ + PRIMITIVE(zlib_write, 2) \ + PRIMITIVE(zlib_read, 1) \ + PRIMITIVE(zlib_close, 1) \ + PRIMITIVE(zlib_uninit, 1) \ #define MODULE_SUBPROCESS(PRIMITIVE) \ PRIMITIVE(init, 0) \ @@ -1007,6 +1013,7 @@ Object* get_absolute_path(Process* process, const wchar_t* pathname, wchar_t* ou #define _A_T_Sha(N, name) MAKE_UNPACKING_MACRO(Sha, N, name) #define _A_T_Adler32(N, name) MAKE_UNPACKING_MACRO(Adler32, N, name) #define _A_T_ZlibRle(N, name) MAKE_UNPACKING_MACRO(ZlibRle, N, name) +#define _A_T_Zlib(N, name) MAKE_UNPACKING_MACRO(Zlib, N, name) #define _A_T_GpioResource(N, name) MAKE_UNPACKING_MACRO(GpioResource, N, name) #define _A_T_UartResource(N, name) MAKE_UNPACKING_MACRO(UartResource, N, name) #define _A_T_UdpSocketResource(N, name) MAKE_UNPACKING_MACRO(UdpSocketResource, N, name) diff --git a/src/primitive_zlib.cc b/src/primitive_zlib.cc index ba8d79fc4..310c95f62 100644 --- a/src/primitive_zlib.cc +++ b/src/primitive_zlib.cc @@ -13,6 +13,12 @@ // The license can be found in the file `LICENSE` in the top level // directory of this repository. +#include "top.h" + +#ifdef CONFIG_TOIT_FULL_ZLIB +#include +#endif + #include "process.h" #include "objects.h" #include "objects_inline.h" @@ -107,4 +113,179 @@ PRIMITIVE(rle_finish) { return Smi::from(written); } +#ifdef CONFIG_TOIT_FULL_ZLIB + +class Zlib : public SimpleResource { + public: + TAG(Zlib); + + Zlib(SimpleResourceGroup* group) : SimpleResource(group) {} + ~Zlib(); + + int init_deflate(int compression_level); + int init_inflate(); + int write(const uint8* data, int length, int* error_return); + int output_available(); + void get_output(uint8* buffer, int length); + void close() { closed_ = true; } + bool closed() const { return closed_; } + + private: + static const int ZLIB_BUFFER_SIZE = 16384; + z_stream stream_; + bool deflate_; + bool closed_ = false; + uint8 output_buffer_[ZLIB_BUFFER_SIZE]; +}; + +int Zlib::init_deflate(int compression_level) { + stream_.zalloc = Z_NULL; + stream_.zfree = Z_NULL; + stream_.opaque = null; + int result = deflateInit(&stream_, compression_level); + stream_.next_out = &output_buffer_[0]; + stream_.avail_out = ZLIB_BUFFER_SIZE; + stream_.data_type = Z_UNKNOWN; + deflate_ = true; + return result; +} + +int Zlib::init_inflate() { + stream_.zalloc = Z_NULL; + stream_.zfree = Z_NULL; + stream_.opaque = null; + int result = inflateInit(&stream_); + stream_.next_out = &output_buffer_[0]; + stream_.avail_out = ZLIB_BUFFER_SIZE; + deflate_ = false; + return result; +} + +Zlib::~Zlib() { + if (deflate_) { + deflateEnd(&stream_); + } else { + inflateEnd(&stream_); + } +} + +int Zlib::write(const uint8* data, int length, int* error_return) { + stream_.next_in = const_cast(data); + stream_.avail_in = length; + int result = deflate_ ? deflate(&stream_, Z_NO_FLUSH) : inflate(&stream_, Z_NO_FLUSH); + *error_return = result; + int written = length - stream_.avail_in; + return written; +} + +int Zlib::output_available() { + if (closed_) { + stream_.avail_in = 0; + if (deflate_) { + deflate(&stream_, Z_FINISH); + } else { + inflate(&stream_, Z_FINISH); + } + } + return ZLIB_BUFFER_SIZE - stream_.avail_out; +} + +void Zlib::get_output(uint8* buffer, int length) { + memcpy(buffer, output_buffer_, length); + stream_.next_out = &output_buffer_[0]; + stream_.avail_out = ZLIB_BUFFER_SIZE; +} + +static Object* zlib_error(Process* process, int error) { + if (error == Z_MEM_ERROR) FAIL(MALLOC_FAILED); + printf("Unknown error message %d\n", error); + FAIL(ERROR); +} + +#endif + +PRIMITIVE(zlib_init_deflate) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(SimpleResourceGroup, group, int, compression_level) + ByteArray* proxy = process->object_heap()->allocate_proxy(); + Zlib* zlib = _new Zlib(group); + if (!zlib) FAIL(MALLOC_FAILED); + int result = zlib->init_deflate(compression_level); + if (result < 0) { + delete zlib; + return zlib_error(process, result); + } + proxy->set_external_address(zlib); + return proxy; +#endif +} + +PRIMITIVE(zlib_init_inflate) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(SimpleResourceGroup, group); + ByteArray* proxy = process->object_heap()->allocate_proxy(); + Zlib* zlib = _new Zlib(group); + if (!zlib) FAIL(MALLOC_FAILED); + int result = zlib->init_inflate(); + if (result < 0) { + delete zlib; + return zlib_error(process, result); + } + proxy->set_external_address(zlib); + return proxy; +#endif +} + +PRIMITIVE(zlib_write) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(Zlib, zlib, Blob, data); + int error; + int bytes_written = zlib->write(data.address(), data.length(), &error); + if (error < 0 && error != Z_BUF_ERROR) return zlib_error(process, error); + return Smi::from(bytes_written); +#endif +} + +PRIMITIVE(zlib_read) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(Zlib, zlib); + int length = zlib->output_available(); + if (length == 0 && zlib->closed()) return process->null_object(); + ByteArray* result = process->allocate_byte_array(length); + if (result == null) FAIL(ALLOCATION_FAILED); + ByteArray::Bytes bytes(result); + zlib->get_output(bytes.address(), length); + return result; +#endif +} + +PRIMITIVE(zlib_close) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(Zlib, zlib); + zlib->close(); + return process->null_object(); +#endif +} + +PRIMITIVE(zlib_uninit) { +#ifndef CONFIG_TOIT_FULL_ZLIB + FAIL(UNIMPLEMENTED); +#else + ARGS(Zlib, zlib); + zlib->resource_group()->unregister_resource(zlib); + zlib_proxy->clear_external_address(); + return process->null_object(); +#endif +} + } diff --git a/src/tags.h b/src/tags.h index b052fb049..a036dcb0c 100644 --- a/src/tags.h +++ b/src/tags.h @@ -39,6 +39,7 @@ namespace toit { fn(Siphash) \ fn(Adler32) \ fn(ZlibRle) \ + fn(Zlib) \ fn(UartResource) \ fn(GpioResource) \ fn(I2sResource) \ diff --git a/src/top.h b/src/top.h index 40bea00b7..36661fdf8 100644 --- a/src/top.h +++ b/src/top.h @@ -104,6 +104,9 @@ #define CONFIG_TOIT_BYTE_DISPLAY 1 #define CONFIG_TOIT_BIT_DISPLAY 1 #define CONFIG_TOIT_FONT 1 +#if !defined(TOIT_WINDOWS) && !defined(BUILD_32) && !defined(__arm__) && !defined(__aarch64__) +#define CONFIG_TOIT_FULL_ZLIB 1 +#endif #endif typedef intptr_t word; diff --git a/tests/zlib_full_test.toit b/tests/zlib_full_test.toit new file mode 100644 index 000000000..1c16890c4 --- /dev/null +++ b/tests/zlib_full_test.toit @@ -0,0 +1,130 @@ +// Copyright (C) 2018 Toitware ApS. +// Use of this source code is governed by a Zero-Clause BSD license that can +// be found in the tests/LICENSE file. + +import bytes +import expect show * +import zlib +import crypto.sha + +main: + // Test whether support is compiled into the VM. + enabled := false + catch: + zlib.Decoder + print "Zlib support is compiled in." + enabled = true + if not enabled: + print "Zlib support is not compiled in." + return + + compressed := simple_encoder + simple_decoder compressed + squashed1 := big_encoder_no_wait + squashed2 := big_encoder_with_wait + + expect_equals squashed1 squashed2 + + big_decoder squashed1 get_sha + + rle_test + +REPEATS ::= 10000 +INPUT ::= "Now is the time for all good men to come to the aid of the party." + +simple_encoder -> ByteArray: + compressor := zlib.Encoder + compressor.write INPUT + compressor.close + reader := compressor.reader + compressed := reader.read + reader.close + return compressed + +simple_decoder compressed/ByteArray -> none: + decompressor := zlib.Decoder + decompressor.write compressed + decompressor.close + reader := decompressor.reader + round_trip := reader.read + expect_equals INPUT round_trip.to_string + +big_encoder_no_wait -> ByteArray: + compressor := zlib.Encoder + task:: + REPEATS.repeat: + for pos := 0; pos < INPUT.size; pos += compressor.write --wait=false INPUT[pos..]: + yield + compressor.close + squashed := #[] + reader := compressor.reader + while data := reader.read --wait=false: + if data.size == 0: + yield + else: + squashed += data + + print "squashed $((REPEATS * INPUT.size) >> 10)k down to $squashed.size bytes" + return squashed + +big_encoder_with_wait -> ByteArray: + compressor := zlib.Encoder + task:: + REPEATS.repeat: + compressor.write INPUT + compressor.close + squashed2 := #[] + reader := compressor.reader + while data := reader.read: + squashed2 += data + reader.close + + print "squashed2 $((REPEATS * INPUT.size) >> 10)k down to $squashed2.size bytes" + + return squashed2 + +get_sha -> ByteArray: + input_sha := sha.Sha256 + REPEATS.repeat: + input_sha.add INPUT + return input_sha.get + +big_decoder squashed/ByteArray input_hash/ByteArray -> none: + decompressor := zlib.Decoder + task:: + decompressor.write squashed + decompressor.close + + sha := sha.Sha256 + while data := decompressor.reader.read: + sha.add data + expect_equals + sha.get + input_hash + +// Encode with the RLE encoder, decode with the full zlib decoder. +rle_test -> none: + encoder := zlib.RunLengthZlibEncoder + + str := "" + 26.repeat: + str += "$(%c 'A' + it)" * it + + task:: + encoder.write str + encoder.close + + encoded := #[] + while data := encoder.read: + encoded += data + + decoder := zlib.Decoder + task:: + decoder.write encoded + decoder.close + + round_trip := #[] + while data := decoder.reader.read: + round_trip += data + expect_equals str round_trip.to_string +