From dadd9c80f0f7489e3a1dbe778d61da2141532d6f Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Mon, 17 Jul 2017 19:59:02 +0200 Subject: [PATCH] Add support for compressing embedded Lua chunk with BriefLZ --- README.adoc | 1 - Rocksfile | 1 + luapak-dev-0.rockspec | 1 + luapak/cli/make.lua | 3 + luapak/cli/wrapper.lua | 5 +- luapak/make.lua | 2 +- luapak/wrapper.lua | 66 ++++++++++++++--- luapak/wrapper_tmpl.lua | 155 ++++++++++++++++++++++++++++++++++++---- 8 files changed, 209 insertions(+), 25 deletions(-) diff --git a/README.adoc b/README.adoc index 6a9a322..d749d46 100644 --- a/README.adoc +++ b/README.adoc @@ -70,7 +70,6 @@ source .envrc * Write documentation into README. * Write integration tests. -* Integrate some compression algorithm for compressing embedded Lua sources. * Analyse usage of Lua standard modules and exclude unused from the binary. diff --git a/Rocksfile b/Rocksfile index 1367880..f711a54 100644 --- a/Rocksfile +++ b/Rocksfile @@ -1,3 +1,4 @@ +brieflz depgraph ldoc lua-cjson diff --git a/luapak-dev-0.rockspec b/luapak-dev-0.rockspec index 5c1dffe..68fcd1e 100644 --- a/luapak-dev-0.rockspec +++ b/luapak-dev-0.rockspec @@ -24,6 +24,7 @@ with Lua library and native extensions.]], dependencies = { 'lua >= 5.1', + 'brieflz ~> 0.1.0', 'depgraph ~> 0.1', 'lua-glob-pattern ~> 0.2', 'luafilesystem ~> 1.6', diff --git a/luapak/cli/make.lua b/luapak/cli/make.lua index 32e32fb..cfd8c78 100644 --- a/luapak/cli/make.lua +++ b/luapak/cli/make.lua @@ -73,6 +73,8 @@ Options: -o, --output=FILE Output file name or path. Defaults to base name of the main script with stripped .lua extension. + -C, --no-compress Disable BriefLZ compression of Lua sources. + -M, --no-minify Disable minification of Lua sources. -t, --rocks-tree=DIR The prefix where to install required modules. Default is @@ -114,6 +116,7 @@ return function (arg) end local make_opts = { + compress = not opts.no_compress, debug = opts.debug, exclude_modules = split_repeated_option(opts.exclude_modules), extra_modules = split_repeated_option(opts.include_modules), diff --git a/luapak/cli/wrapper.lua b/luapak/cli/wrapper.lua index 78f75ea..6348b9d 100644 --- a/luapak/cli/wrapper.lua +++ b/luapak/cli/wrapper.lua @@ -18,6 +18,7 @@ Arguments: MODULE_NAME Name of native module to preload (e.g. "cjson"). Options: + -C, --no-compress Do not compress FILE using BriefLZ algorithm. -o, --output=FILE Where to write the generated code; "-" for stdout. Default is "-". -v, --verbose Be verbose, i.e. print debug messages. -h, --help Display this help message and exit. @@ -45,7 +46,9 @@ return function (arg) and io.stdout or assert(io.open(opts.output, 'w')) - local output = wrapper.generate(lua_chunk, module_names) + local output = wrapper.generate(lua_chunk, module_names, { + compress = not opts.no_compress, + }) assert(out:write(output)) out:close() diff --git a/luapak/make.lua b/luapak/make.lua index 6cf51fd..b534b46 100644 --- a/luapak/make.lua +++ b/luapak/make.lua @@ -186,7 +186,7 @@ local function generate_wrapper (output_file, entry_script, lua_modules, native_ end) push(buff, remove_shebang(entry_script)) - assert(fileh:write(wrapper.generate(concat(buff), native_modules))) + assert(fileh:write(wrapper.generate(concat(buff), native_modules, opts))) fileh:flush() fileh:close() diff --git a/luapak/wrapper.lua b/luapak/wrapper.lua index 30d59cc..de9153a 100644 --- a/luapak/wrapper.lua +++ b/luapak/wrapper.lua @@ -1,6 +1,7 @@ --------- -- Generator of a C "wrapper" for standalone Lua programs. ---- +local brieflz = require 'brieflz' local wrapper_tmpl = require 'luapak.wrapper_tmpl' local utils = require 'luapak.utils' @@ -58,24 +59,53 @@ local function define_preloaded_libs (names) return concat(buff, '\n') end ---- Generates definition of C constant `LUAPAK_LUA_MAIN` with the given chunk of Lua code. +--- Generates definition of C constant `LUAPAK_SCRIPT` with the given data. -- --- @tparam string chunk +-- @tparam string data -- @treturn string -local function define_lua_main (chunk) - return fmt('static const unsigned char LUAPAK_LUA_MAIN[] = %s;\n', - encode_c_hex(chunk)) +local function define_script (data) + return fmt('static const unsigned char LUAPAK_SCRIPT[] = %s;\n', + encode_c_hex(data)) +end + +local function define_script_unpacked_size (size) + return fmt('static const size_t LUAPAK_SCRIPT_UNPACKED_SIZE = %d;', size) +end + +--- Generates C `#define` directive with the specified constant. +-- +-- @tparam string name The constant name. +-- @param value The constant value. +-- @treturn string +local function define_macro_const (name, value) + local value_t = type(value) + + if value_t == 'number' or value_t == 'boolean' then + return fmt('#define %s %s', name, value) + else + return fmt('#define %s %q', name, tostring(value)) + end end --- Generates a fragment of C code that should be included in the template. -- -- @tparam string lua_chunk The Lua chunk (source code or byte code) to embed. +-- @tparam int chunk_size Size of **uncompressed** Lua chunk. -- @tparam {string,...} clib_names List of names of native modules to be preload. +-- @tparam table defs Table of constants to define with `#define` directive. -- @treturn string Generated C code. -local function generate_fragment (lua_chunk, clib_names) +local function generate_fragment (lua_chunk, chunk_size, clib_names, defs) local buffer = {} - push(buffer, define_lua_main(remove_shebang(lua_chunk))) + for name, value in pairs(defs) do + push(buffer, define_macro_const(name, value)) + end + push(buffer, '') + + if chunk_size then + push(buffer, define_script_unpacked_size(chunk_size)) + end + push(buffer, define_script(lua_chunk)) for _, name in ipairs(clib_names) do push(buffer, declare_luaopen_func(name)) @@ -93,12 +123,28 @@ local M = {} -- -- @tparam string lua_chunk The Lua chunk (source code or byte code) to embed. -- @tparam ?{string,...} clib_names List of names of native modules to be preload. +-- @tparam {[string]=bool,...} opts Options: `compress` - enable compression. -- @treturn string A source code in C. -function M.generate (lua_chunk, clib_names) - check_args('string, ?table', lua_chunk, clib_names) +function M.generate (lua_chunk, clib_names, opts) + check_args('string, ?table, ?table', lua_chunk, clib_names, opts) + + clib_names = clib_names or {} + opts = opts or {} + + lua_chunk = remove_shebang(lua_chunk) + + local defs = {} + local chunk_size -- size of *uncompressed* data + + if opts.compress then + lua_chunk, chunk_size = brieflz.pack(lua_chunk) + defs['LUAPAK_BRIEFLZ'] = 1 + else + defs['LUAPAK_BRIEFLZ'] = 0 + end return (wrapper_tmpl:gsub('//%-%-PLACEHOLDER%-%-//', - generate_fragment(lua_chunk, clib_names or {}))) + generate_fragment(lua_chunk, chunk_size, clib_names, defs))) end return M diff --git a/luapak/wrapper_tmpl.lua b/luapak/wrapper_tmpl.lua index 4f9cb75..a49157e 100644 --- a/luapak/wrapper_tmpl.lua +++ b/luapak/wrapper_tmpl.lua @@ -18,6 +18,24 @@ return [[ //--PLACEHOLDER--// +/***************************************************************************** +* Compatibility with older Lua * +*****************************************************************************/ + +#if LUA_VERSION_NUM == 501 // Lua 5.1 + #define LUA_OK 0 +#endif + +/** + * Print an error message. + * Copied from Lua 5.3 lauxlib.h for compatibility with olders. + */ +#if !defined(lua_writestringerror) +#define lua_writestringerror(s,p) \ + (fprintf(stderr, (s), (p)), fflush(stderr)) +#endif + + /***************************************************************************** * Stub libdl * *****************************************************************************/ @@ -46,20 +64,95 @@ return [[ /***************************************************************************** -* Compatibility with older Lua * +* BriefLZ depack * *****************************************************************************/ -#if LUA_VERSION_NUM == 501 // Lua 5.1 - #define LUA_OK 0 -#endif +// The following code is based on BriefLZ library by Joergen Ibsen, +// licensed under zlib license. +// https://github.com/jibsen/brieflz/blob/master/depack.c +#if LUAPAK_BRIEFLZ == 1 + + // Internal data structure. + struct blz_State { + const unsigned char *src; + unsigned char *dst; + unsigned int tag; + unsigned int bits_left; + }; -/** - * Print an error message. - * Copied from Lua 5.3 lauxlib.h for compatibility with olders. - */ -#if !defined(lua_writestringerror) -#define lua_writestringerror(s,p) \ - (fprintf(stderr, (s), (p)), fflush(stderr)) + static unsigned int blz_getbit (struct blz_State *bs) { + // Check if tag is empty. + if (!bs->bits_left--) { + // Load next tag + bs->tag = (unsigned int) bs->src[0] + | ((unsigned int) bs->src[1] << 8); + bs->src += 2; + bs->bits_left = 15; + } + + // Shift bit out of tag. + const unsigned int bit = (bs->tag & 0x8000) ? 1 : 0; + bs->tag <<= 1; + + return bit; + } + + static size_t blz_getgamma (struct blz_State *bs) { + size_t result = 1; + + // Input gamma2-encoded bits. + do { + result = (result << 1) + blz_getbit(bs); + } while (blz_getbit(bs)); + + return result; + } + + /** + * Decompress `depacked_size` bytes of data from `src` to `dst` + * and return size of decompressed data. + */ + static size_t blz_depack (const void *src, void *dst, size_t depacked_size) { + if (depacked_size == 0) { + return 0; + } + + struct blz_State bs = { + .src = (const unsigned char *) src, + .dst = (unsigned char *) dst, + .bits_left = 0 + }; + *bs.dst++ = *bs.src++; // first byte verbatim + + size_t dst_size = 1; + + // Main decompression loop. + while (dst_size < depacked_size) { + if (blz_getbit(&bs)) { + // Input match length and offset. + size_t len = blz_getgamma(&bs) + 2; + size_t off = blz_getgamma(&bs) - 2; + + off = (off << 8) + (size_t) *bs.src++ + 1; + + // Copy match. + { + const unsigned char *p = bs.dst - off; + for (size_t i = len; i > 0; --i) { + *bs.dst++ = *p++; + } + } + dst_size += len; + + } else { + // Copy literal. + *bs.dst++ = *bs.src++; + dst_size++; + } + } + + return dst_size; // decompressed size + } #endif @@ -67,6 +160,44 @@ return [[ * M a i n * *****************************************************************************/ +#if LUAPAK_BRIEFLZ == 1 + + /** + * Decompress and load the embedded Lua script. + * If there's no error, the compiled chunk is pushed on top of the stack as + * a Lua function. Otherwise an error message is pushed on top of the stack. + */ + static int load_script (lua_State *L) { + const size_t unpacked_size = LUAPAK_SCRIPT_UNPACKED_SIZE; + + void *buffer = malloc(unpacked_size); + if (buffer == NULL) { + lua_pushstring(L, "PANIC: not enough memory for decompression"); + return LUA_ERRRUN; + } + + if (blz_depack(LUAPAK_SCRIPT, buffer, unpacked_size) != unpacked_size) { + lua_pushstring(L, "PANIC: decompression failed"); + return LUA_ERRRUN; + } + + const int status = luaL_loadbuffer(L, (const char *) buffer, unpacked_size, "@main"); + free(buffer); + + return status; + } +#else + + /** + * Load the embedded Lua script. + * If there's no error, the compiled chunk is pushed on top of the stack as + * a Lua function. Otherwise an error message is pushed on top of the stack. + */ + static int load_script (lua_State *L) { + return luaL_loadbuffer(L, (const char *) LUAPAK_SCRIPT, sizeof(LUAPAK_SCRIPT), "@main"); + } +#endif + #if defined(LUAPAK_WITHOUT_COROUTINE) \ || defined(LUAPAK_WITHOUT_IO) \ || defined(LUAPAK_WITHOUT_OS) \ @@ -310,7 +441,7 @@ int main (int argc, char *argv[]) { preload_bundled_libs(L); - int status = luaL_loadbuffer(L, (const char*)LUAPAK_LUA_MAIN, sizeof(LUAPAK_LUA_MAIN), "main"); + int status = load_script(L); if (status == LUA_OK) { int n = pushargs(L); // push arguments to script status = docall(L, n, LUA_MULTRET);