libpressio version 0.67.0

Major Changes

+ introduced log_transform meta compressor to enable point-wise error
  bounds from absolute error bound compressors
+ introduced delta_coding to increase compression in some cases
+ introduced linear_quantizer to increase compression in some cases

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
-project(libpressio VERSION "0.66.3" LANGUAGES CXX C)
+project(libpressio VERSION "0.67.0" LANGUAGES CXX C)
 
 #correct was to set a default build type
 # https://blog.kitware.com/cmake-and-the-default-build-type/
@@ -51,6 +51,9 @@ add_library(libpressio
   #plugins
   ./src/plugins/compressors/compressor_base.cc
   ./src/plugins/compressors/noop.cc
+  ./src/plugins/compressors/delta_encoding.cc
+  ./src/plugins/compressors/log_transform.cc
+  ./src/plugins/compressors/linear_quantizer.cc
   ./src/plugins/compressors/sampling.cc
   ./src/plugins/compressors/resize.cc
   ./src/plugins/compressors/transpose.cc

COPYRIGHT.txt

@@ -1,6 +1,6 @@
 Copyright © 2021 , UChicago Argonne, LLC
 All Rights Reserved
-[libpressio, Version 0.66.3]
+[libpressio, Version 0.67.0]
 Robert Underwood
 Argonne National Laboratory
 

src/plugins/compressors/delta_encoding.cc

@@ -0,0 +1,106 @@
+#include <libpressio_ext/cpp/compressor.h>
+#include <libpressio_ext/cpp/pressio.h>
+#include <std_compat/memory.h>
+#include <sstream>
+
+struct delta_encoder {
+  template <class T>
+  pressio_data operator()(T const* begin, T const* end) {
+    pressio_data d = pressio_data::owning(pressio_dtype_from_type<T>(), {static_cast<size_t>(end - begin)});
+    T* ptr = static_cast<T*>(d.data());
+
+    const size_t len = end-begin;
+    ptr[0] = begin[0];
+    for (size_t i = 1; i < len; ++i) {
+      ptr[i] = begin[i] - begin[i-1];
+    }
+
+    return d;
+  }
+};
+
+struct delta_decoder {
+  template <class T>
+  pressio_data operator()(T const* begin, T const* end) {
+    pressio_data d = pressio_data::owning(pressio_dtype_from_type<T>(), {static_cast<size_t>(end - begin)});
+    T* ptr = static_cast<T*>(d.data());
+
+    const size_t len = end-begin;
+    ptr[0] = begin[0];
+    ptr[1] = begin[0] + begin[1];
+    for (size_t i = 2; i < len; ++i) {
+      ptr[i] = begin[i] + ptr[i-1];
+    }
+
+    return d;
+  }
+};
+
+
+
+class delta_encoding: public libpressio_compressor_plugin {
+  pressio_options get_options_impl() const override {
+    pressio_options opts;
+    set_meta(opts, "delta_encoding:compressor", meta_id, meta);
+    return opts;
+  }
+  pressio_options get_documentation_impl() const override {
+    pressio_options opts;
+    set_meta_docs(opts, "delta_encoding:compressor", "compressor to apply after encoding", meta);
+    set(opts, "pressio:description", R"(delta_encoding
+
+applies delta encoding to prior to compression and reverses it post decompression.
+
+y[0] = x[0];
+y[i] = x[i] - x[i-1];
+
+)");
+
+    return opts;
+  }
+  pressio_options get_configuration_impl() const override {
+    pressio_options opts;
+    set(opts, "pressio:thread_safe", static_cast<int32_t>(get_threadsafe(*meta)));
+    set(opts, "pressio:stability", "experimental");
+    return opts;
+  }
+  int set_options_impl(const pressio_options &options) override {
+    get_meta(options, "delta_encoding:compressor", compressor_plugins(), meta_id, meta);
+    return 0;
+  }
+  int compress_impl(const pressio_data *input, struct pressio_data *output) override {
+    auto delta_encoded = pressio_data_for_each<pressio_data>(*input, delta_encoder{});
+    return meta->compress(&delta_encoded, output);
+  }
+  int decompress_impl(const pressio_data *input, struct pressio_data *output) override {
+    int ret = meta->decompress(input, output);
+    *output = pressio_data_for_each<pressio_data>(*output, delta_decoder{});
+    return ret;
+  }
+  void set_name_impl(std::string const& new_name) override {
+    meta->set_name(new_name + "/" + meta->prefix());
+  }
+  const char* prefix() const override { return "delta_encoding"; }
+  const char* version() const override { 
+    const static std::string version_str = [this]{
+      std::stringstream ss;
+      ss << major_version() << '.' << minor_version() << '.' << patch_version();
+      return ss.str();
+    }();
+    return version_str.c_str();
+  }
+  virtual std::shared_ptr<libpressio_compressor_plugin> clone() override {
+    return compat::make_unique<delta_encoding>(*this);
+  }
+
+  std::string meta_id = "noop";
+  pressio_compressor meta = compressor_plugins().build("noop");
+};
+
+static pressio_register delta_encoding_register(
+    compressor_plugins(),
+    "delta_encoding",
+    []{
+      return compat::make_unique<delta_encoding>();
+    }
+    );

src/plugins/compressors/linear_quantizer.cc

@@ -0,0 +1,123 @@
+#include <libpressio_ext/cpp/compressor.h>
+#include <libpressio_ext/cpp/pressio.h>
+#include <std_compat/memory.h>
+#include <sstream>
+
+struct linear_quantizer_step_finder {
+  template <class T>
+  double operator()(T const* begin, T const* end) {
+    auto ret = std::minmax(begin, end);
+    return std::abs(ret.second-ret.first)/auto_step;
+  }
+  int64_t auto_step;
+};
+
+struct linear_quantizer_encoder {
+  template <class T>
+  pressio_data operator()(T const* begin, T const* end) {
+    pressio_data d = pressio_data::owning(pressio_dtype_from_type<int64_t>(), {static_cast<size_t>(end - begin)});
+    int64_t* ptr = static_cast<int64_t*>(d.data());
+    const size_t len = end-begin;
+    for (size_t i = 0; i < len; ++i) {
+      ptr[i] = begin[i]/step;
+    }
+
+    return d;
+  }
+  double step;
+};
+
+struct linear_quantizer_decoder {
+  template <class T, class V>
+  int operator()(T const* begin, T const* end, V* ptr) {
+    const size_t len = end-begin;
+    for (size_t i = 0; i < len; ++i) {
+      ptr[i] = begin[i]*step;
+    }
+    return 0;
+  }
+  double step;
+};
+
+
+
+class linear_quantizer: public libpressio_compressor_plugin {
+  pressio_options get_options_impl() const override {
+    pressio_options opts;
+    set_meta(opts, "linear_quantizer:compressor", meta_id, meta);
+    set(opts, "linear_quantizer:step", step);
+    set(opts, "linear_quantizer:auto_step", auto_step);
+    return opts;
+  }
+  pressio_options get_documentation_impl() const override {
+    pressio_options opts;
+    set_meta_docs(opts, "linear_quantizer:compressor", "compressor to apply after encoding", meta);
+    set(opts, "pressio:description", R"(linear_quantizer
+
+applies linear_quantizer encoding to prior to compression and reverses it post decompression.
+
+  y[i] = (x[i])/step
+
+)");
+    set(opts, "linear_quantizer:step", "the size of step to use while quantizing");
+    set(opts, "linear_quantizer:auto_step", "the number of steps to assume while automatically determine the step size. 0 means use manual step size");
+
+    return opts;
+  }
+  pressio_options get_configuration_impl() const override {
+    pressio_options opts;
+    set(opts, "pressio:thread_safe", static_cast<int32_t>(get_threadsafe(*meta)));
+    set(opts, "pressio:stability", "experimental");
+    return opts;
+  }
+  int set_options_impl(const pressio_options &options) override {
+    get_meta(options, "linear_quantizer:compressor", compressor_plugins(), meta_id, meta);
+    get(options, "linear_quantizer:step", &step);
+    get(options, "linear_quantizer:auto_step", &auto_step);
+    return 0;
+  }
+  int compress_impl(const pressio_data *input, struct pressio_data *output) override {
+    if(auto_step) {
+      step = pressio_data_for_each<double>(*input, linear_quantizer_step_finder{auto_step});
+    }
+    auto linear_quantizer_encoded = pressio_data_for_each<pressio_data>(*input, linear_quantizer_encoder{step});
+    return meta->compress(&linear_quantizer_encoded, output);
+  }
+  int decompress_impl(const pressio_data *input, struct pressio_data *output) override {
+    pressio_data quantized_output = pressio_data::owning(
+        pressio_int64_dtype,
+        output->dimensions()
+        );
+    int ret = meta->decompress(input, &quantized_output);
+    pressio_data_for_each<int>(quantized_output, *output, linear_quantizer_decoder{step});
+    return ret;
+  }
+  void set_name_impl(std::string const& new_name) override {
+    meta->set_name(new_name + "/" + meta->prefix());
+  }
+  const char* prefix() const override { return "linear_quantizer"; }
+  const char* version() const override { 
+    const static std::string version_str = [this]{
+      std::stringstream ss;
+      ss << major_version() << '.' << minor_version() << '.' << patch_version();
+      return ss.str();
+    }();
+    return version_str.c_str();
+  }
+  virtual std::shared_ptr<libpressio_compressor_plugin> clone() override {
+    return compat::make_unique<linear_quantizer>(*this);
+  }
+
+  int64_t auto_step = 0;
+  double step = 0;
+  std::string meta_id = "noop";
+  pressio_compressor meta = compressor_plugins().build("noop");
+};
+
+static pressio_register linear_quantizer_register(
+    compressor_plugins(),
+    "linear_quantizer",
+    []{
+      return compat::make_unique<linear_quantizer>();
+    }
+    );

src/plugins/compressors/log_transform.cc

@@ -0,0 +1,103 @@
+#include <libpressio_ext/cpp/compressor.h>
+#include <libpressio_ext/cpp/pressio.h>
+#include <std_compat/memory.h>
+#include <sstream>
+#include <cmath>
+
+struct log_encoder {
+  template <class T>
+  pressio_data operator()(T const* begin, T const* end) {
+    pressio_data d = pressio_data::owning(pressio_dtype_from_type<T>(), {static_cast<size_t>(end - begin)});
+    T* ptr = static_cast<T*>(d.data());
+
+    const size_t len = end-begin;
+    for (size_t i = 0; i < len; ++i) {
+      ptr[i] = log(begin[i]);
+    }
+
+    return d;
+  }
+};
+
+struct log_decoder {
+  template <class T>
+  pressio_data operator()(T const* begin, T const* end) {
+    pressio_data d = pressio_data::owning(pressio_dtype_from_type<T>(), {static_cast<size_t>(end - begin)});
+    T* ptr = static_cast<T*>(d.data());
+
+    const size_t len = end-begin;
+    for (size_t i = 0; i < len; ++i) {
+      ptr[i] = exp(begin[i]);
+    }
+
+    return d;
+  }
+};
+
+
+
+class log_transform: public libpressio_compressor_plugin {
+  pressio_options get_options_impl() const override {
+    pressio_options opts;
+    set_meta(opts, "log_transform:compressor", meta_id, meta);
+    return opts;
+  }
+  pressio_options get_documentation_impl() const override {
+    pressio_options opts;
+    set_meta_docs(opts, "log_transform:compressor", "compressor to apply after encoding", meta);
+    set(opts, "pressio:description", R"(log_transform
+
+applies a log transform to prior to compression and experimental transform post decompression.
+
+y[0] = log(x[0]);
+
+)");
+
+    return opts;
+  }
+  pressio_options get_configuration_impl() const override {
+    pressio_options opts;
+    set(opts, "pressio:thread_safe", static_cast<int32_t>(get_threadsafe(*meta)));
+    set(opts, "pressio:stability", "experimental");
+    return opts;
+  }
+  int set_options_impl(const pressio_options &options) override {
+    get_meta(options, "log_transform:compressor", compressor_plugins(), meta_id, meta);
+    return 0;
+  }
+  int compress_impl(const pressio_data *input, struct pressio_data *output) override {
+    auto log_encoded = pressio_data_for_each<pressio_data>(*input, log_encoder{});
+    return meta->compress(&log_encoded, output);
+  }
+  int decompress_impl(const pressio_data *input, struct pressio_data *output) override {
+    int ret = meta->decompress(input, output);
+    *output = pressio_data_for_each<pressio_data>(*output, log_decoder{});
+    return ret;
+  }
+  void set_name_impl(std::string const& new_name) override {
+    meta->set_name(new_name + "/" + meta->prefix());
+  }
+  const char* prefix() const override { return "log_transform"; }
+  const char* version() const override { 
+    const static std::string version_str = [this]{
+      std::stringstream ss;
+      ss << major_version() << '.' << minor_version() << '.' << patch_version();
+      return ss.str();
+    }();
+    return version_str.c_str();
+  }
+  virtual std::shared_ptr<libpressio_compressor_plugin> clone() override {
+    return compat::make_unique<log_transform>(*this);
+  }
+
+  std::string meta_id = "noop";
+  pressio_compressor meta = compressor_plugins().build("noop");
+};
+
+static pressio_register log_transform_register(
+    compressor_plugins(),
+    "log_transform",
+    []{
+      return compat::make_unique<log_transform>();
+    }
+    );