Add support for compressing embedded Lua chunk with BriefLZ

README.adoc

@@ -70,7 +70,6 @@ source .envrc
 
 * Write documentation into README.
 * Write integration tests.
-* Integrate some compression algorithm for compressing embedded Lua sources.
 * Analyse usage of Lua standard modules and exclude unused from the binary.
 
 

Rocksfile

@@ -1,3 +1,4 @@
+brieflz
 depgraph
 ldoc
 lua-cjson

luapak-dev-0.rockspec

@@ -24,6 +24,7 @@ with Lua library and native extensions.]],
 
 dependencies = {
   'lua >= 5.1',
+  'brieflz ~> 0.1.0',
   'depgraph ~> 0.1',
   'lua-glob-pattern ~> 0.2',
   'luafilesystem ~> 1.6',

luapak/cli/make.lua

@@ -73,6 +73,8 @@ Options:
   -o, --output=FILE               Output file name or path. Defaults to base name of the main
                                   script with stripped .lua extension.
 
+  -C, --no-compress               Disable BriefLZ compression of Lua sources.
+
   -M, --no-minify                 Disable minification of Lua sources.
 
   -t, --rocks-tree=DIR            The prefix where to install required modules. Default is
@@ -114,6 +116,7 @@ return function (arg)
   end
 
   local make_opts = {
+    compress = not opts.no_compress,
     debug = opts.debug,
     exclude_modules = split_repeated_option(opts.exclude_modules),
     extra_modules = split_repeated_option(opts.include_modules),

luapak/cli/wrapper.lua

@@ -18,6 +18,7 @@ Arguments:
   MODULE_NAME                 Name of native module to preload (e.g. "cjson").
 
 Options:
+  -C, --no-compress           Do not compress FILE using BriefLZ algorithm.
   -o, --output=FILE           Where to write the generated code; "-" for stdout. Default is "-".
   -v, --verbose               Be verbose, i.e. print debug messages.
   -h, --help                  Display this help message and exit.
@@ -45,7 +46,9 @@ return function (arg)
       and io.stdout
       or assert(io.open(opts.output, 'w'))
 
-  local output = wrapper.generate(lua_chunk, module_names)
+  local output = wrapper.generate(lua_chunk, module_names, {
+      compress = not opts.no_compress,
+    })
 
   assert(out:write(output))
   out:close()

luapak/make.lua

@@ -186,7 +186,7 @@ local function generate_wrapper (output_file, entry_script, lua_modules, native_
     end)
   push(buff, remove_shebang(entry_script))
 
-  assert(fileh:write(wrapper.generate(concat(buff), native_modules)))
+  assert(fileh:write(wrapper.generate(concat(buff), native_modules, opts)))
 
   fileh:flush()
   fileh:close()

luapak/wrapper.lua

@@ -1,6 +1,7 @@
 ---------
 -- Generator of a C "wrapper" for standalone Lua programs.
 ----
+local brieflz = require 'brieflz'
 local wrapper_tmpl = require 'luapak.wrapper_tmpl'
 local utils = require 'luapak.utils'
 
@@ -58,24 +59,53 @@ local function define_preloaded_libs (names)
   return concat(buff, '\n')
 end
 
---- Generates definition of C constant `LUAPAK_LUA_MAIN` with the given chunk of Lua code.
+--- Generates definition of C constant `LUAPAK_SCRIPT` with the given data.
 --
--- @tparam string chunk
+-- @tparam string data
 -- @treturn string
-local function define_lua_main (chunk)
-  return fmt('static const unsigned char LUAPAK_LUA_MAIN[] = %s;\n',
-             encode_c_hex(chunk))
+local function define_script (data)
+  return fmt('static const unsigned char LUAPAK_SCRIPT[] = %s;\n',
+             encode_c_hex(data))
+end
+
+local function define_script_unpacked_size (size)
+  return fmt('static const size_t LUAPAK_SCRIPT_UNPACKED_SIZE = %d;', size)
+end
+
+--- Generates C `#define` directive with the specified constant.
+--
+-- @tparam string name The constant name.
+-- @param value The constant value.
+-- @treturn string
+local function define_macro_const (name, value)
+  local value_t = type(value)
+
+  if value_t == 'number' or value_t == 'boolean' then
+    return fmt('#define %s %s', name, value)
+  else
+    return fmt('#define %s %q', name, tostring(value))
+  end
 end
 
 --- Generates a fragment of C code that should be included in the template.
 --
 -- @tparam string lua_chunk The Lua chunk (source code or byte code) to embed.
+-- @tparam int chunk_size Size of **uncompressed** Lua chunk.
 -- @tparam {string,...} clib_names List of names of native modules to be preload.
+-- @tparam table defs Table of constants to define with `#define` directive.
 -- @treturn string Generated C code.
-local function generate_fragment (lua_chunk, clib_names)
+local function generate_fragment (lua_chunk, chunk_size, clib_names, defs)
   local buffer = {}
 
-  push(buffer, define_lua_main(remove_shebang(lua_chunk)))
+  for name, value in pairs(defs) do
+    push(buffer, define_macro_const(name, value))
+  end
+  push(buffer, '')
+
+  if chunk_size then
+    push(buffer, define_script_unpacked_size(chunk_size))
+  end
+  push(buffer, define_script(lua_chunk))
 
   for _, name in ipairs(clib_names) do
     push(buffer, declare_luaopen_func(name))
@@ -93,12 +123,28 @@ local M = {}
 --
 -- @tparam string lua_chunk The Lua chunk (source code or byte code) to embed.
 -- @tparam ?{string,...} clib_names List of names of native modules to be preload.
+-- @tparam {[string]=bool,...} opts Options: `compress` - enable compression.
 -- @treturn string A source code in C.
-function M.generate (lua_chunk, clib_names)
-  check_args('string, ?table', lua_chunk, clib_names)
+function M.generate (lua_chunk, clib_names, opts)
+  check_args('string, ?table, ?table', lua_chunk, clib_names, opts)
+
+  clib_names = clib_names or {}
+  opts = opts or {}
+
+  lua_chunk = remove_shebang(lua_chunk)
+
+  local defs = {}
+  local chunk_size  -- size of *uncompressed* data
+
+  if opts.compress then
+    lua_chunk, chunk_size = brieflz.pack(lua_chunk)
+    defs['LUAPAK_BRIEFLZ'] = 1
+  else
+    defs['LUAPAK_BRIEFLZ'] = 0
+  end
 
   return (wrapper_tmpl:gsub('//%-%-PLACEHOLDER%-%-//',
-      generate_fragment(lua_chunk, clib_names or {})))
+      generate_fragment(lua_chunk, chunk_size, clib_names, defs)))
 end
 
 return M

luapak/wrapper_tmpl.lua

@@ -18,6 +18,24 @@ return [[
 //--PLACEHOLDER--//
 
 
+/*****************************************************************************
+*                        Compatibility with older Lua                        *
+*****************************************************************************/
+
+#if LUA_VERSION_NUM == 501  // Lua 5.1
+  #define LUA_OK 0
+#endif
+
+/**
+ * Print an error message.
+ * Copied from Lua 5.3 lauxlib.h for compatibility with olders.
+ */
+#if !defined(lua_writestringerror)
+#define lua_writestringerror(s,p) \
+        (fprintf(stderr, (s), (p)), fflush(stderr))
+#endif
+
+
 /*****************************************************************************
 *                                 Stub libdl                                 *
 *****************************************************************************/
@@ -46,27 +64,140 @@ return [[
 
 
 /*****************************************************************************
-*                        Compatibility with older Lua                        *
+*                               BriefLZ depack                               *
 *****************************************************************************/
 
-#if LUA_VERSION_NUM == 501  // Lua 5.1
-  #define LUA_OK 0
-#endif
+// The following code is based on BriefLZ library by Joergen Ibsen,
+// licensed under zlib license.
+// https://github.com/jibsen/brieflz/blob/master/depack.c
+#if LUAPAK_BRIEFLZ == 1
+
+  // Internal data structure.
+  struct blz_State {
+    const unsigned char *src;
+    unsigned char *dst;
+    unsigned int tag;
+    unsigned int bits_left;
+  };
 
-/**
- * Print an error message.
- * Copied from Lua 5.3 lauxlib.h for compatibility with olders.
- */
-#if !defined(lua_writestringerror)
-#define lua_writestringerror(s,p) \
-        (fprintf(stderr, (s), (p)), fflush(stderr))
+  static unsigned int blz_getbit (struct blz_State *bs) {
+    // Check if tag is empty.
+    if (!bs->bits_left--) {
+      // Load next tag
+      bs->tag = (unsigned int) bs->src[0]
+             | ((unsigned int) bs->src[1] << 8);
+      bs->src += 2;
+      bs->bits_left = 15;
+    }
+
+    // Shift bit out of tag.
+    const unsigned int bit = (bs->tag & 0x8000) ? 1 : 0;
+    bs->tag <<= 1;
+
+    return bit;
+  }
+
+  static size_t blz_getgamma (struct blz_State *bs) {
+    size_t result = 1;
+
+    // Input gamma2-encoded bits.
+    do {
+      result = (result << 1) + blz_getbit(bs);
+    } while (blz_getbit(bs));
+
+    return result;
+  }
+
+  /**
+   * Decompress `depacked_size` bytes of data from `src` to `dst`
+   * and return size of decompressed data.
+   */
+  static size_t blz_depack (const void *src, void *dst, size_t depacked_size) {
+    if (depacked_size == 0) {
+      return 0;
+    }
+
+    struct blz_State bs = {
+      .src = (const unsigned char *) src,
+      .dst = (unsigned char *) dst,
+      .bits_left = 0
+    };
+    *bs.dst++ = *bs.src++;  // first byte verbatim
+
+    size_t dst_size = 1;
+
+    // Main decompression loop.
+    while (dst_size < depacked_size) {
+      if (blz_getbit(&bs)) {
+        // Input match length and offset.
+        size_t len = blz_getgamma(&bs) + 2;
+        size_t off = blz_getgamma(&bs) - 2;
+
+        off = (off << 8) + (size_t) *bs.src++ + 1;
+
+        // Copy match.
+        {
+          const unsigned char *p = bs.dst - off;
+          for (size_t i = len; i > 0; --i) {
+            *bs.dst++ = *p++;
+          }
+        }
+        dst_size += len;
+
+      } else {
+        // Copy literal.
+        *bs.dst++ = *bs.src++;
+        dst_size++;
+      }
+    }
+
+    return dst_size;  // decompressed size
+  }
 #endif
 
 
 /*****************************************************************************
 *                                  M a i n                                   *
 *****************************************************************************/
 
+#if LUAPAK_BRIEFLZ == 1
+
+  /**
+   * Decompress and load the embedded Lua script.
+   * If there's no error, the compiled chunk is pushed on top of the stack as
+   * a Lua function. Otherwise an error message is pushed on top of the stack.
+   */
+  static int load_script (lua_State *L) {
+    const size_t unpacked_size = LUAPAK_SCRIPT_UNPACKED_SIZE;
+
+    void *buffer = malloc(unpacked_size);
+    if (buffer == NULL) {
+      lua_pushstring(L, "PANIC: not enough memory for decompression");
+      return LUA_ERRRUN;
+    }
+
+    if (blz_depack(LUAPAK_SCRIPT, buffer, unpacked_size) != unpacked_size) {
+      lua_pushstring(L, "PANIC: decompression failed");
+      return LUA_ERRRUN;
+    }
+
+    const int status = luaL_loadbuffer(L, (const char *) buffer, unpacked_size, "@main");
+    free(buffer);
+
+    return status;
+  }
+#else
+
+  /**
+   * Load the embedded Lua script.
+   * If there's no error, the compiled chunk is pushed on top of the stack as
+   * a Lua function. Otherwise an error message is pushed on top of the stack.
+   */
+  static int load_script (lua_State *L) {
+    return luaL_loadbuffer(L, (const char *) LUAPAK_SCRIPT, sizeof(LUAPAK_SCRIPT), "@main");
+  }
+#endif
+
 #if defined(LUAPAK_WITHOUT_COROUTINE) \
     || defined(LUAPAK_WITHOUT_IO) \
     || defined(LUAPAK_WITHOUT_OS) \
@@ -310,7 +441,7 @@ int main (int argc, char *argv[]) {
 
   preload_bundled_libs(L);
 
-  int status = luaL_loadbuffer(L, (const char*)LUAPAK_LUA_MAIN, sizeof(LUAPAK_LUA_MAIN), "main");
+  int status = load_script(L);
   if (status == LUA_OK) {
     int n = pushargs(L);  // push arguments to script
     status = docall(L, n, LUA_MULTRET);