From 1568dadd760d6d1344588edca2a2ca173b17617f Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Sun, 19 Jan 2020 23:43:48 +0200 Subject: [PATCH 01/12] papaw: add Zstandard support --- .gitmodules | 3 +++ README.md | 2 +- azure-pipelines.yml | 4 ++++ meson.build | 31 +++++++++++++++++++++++++++---- meson_options.txt | 2 +- papaw.c | 24 ++++++++++++++++++++++-- papawify.in | 4 ++-- zstd | 1 + 8 files changed, 61 insertions(+), 10 deletions(-) create mode 160000 zstd diff --git a/.gitmodules b/.gitmodules index efacea4..ac8902e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "xz-embedded"] path = xz-embedded url = https://github.com/dimkr/xz-embedded +[submodule "zstd"] + path = zstd + url = https://github.com/facebook/zstd diff --git a/README.md b/README.md index f447550..c2e877f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It is designed to be portable across different devices, therefore it avoids mach ## Implementation -papaw consists of a small executable (~15-20K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html) or the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor. It extracts a LZMA-compressed executable appended to it by the papawify script. +papaw consists of a small executable (~15-40K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html), the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor or the [Zstandard](https://github.com/facebook/zstd) decompressor. It extracts a compressed executable appended to it by the papawify script. The payload executable is extracted to a temporary file. When running as root, this is done by mounting a tmpfs file system and lazily unmounting it before the extraction. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8a08516..d560744 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,6 +37,10 @@ steps: displayName: 'LAMA1 Test' - script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh lzma displayName: 'LAMA1 Build' +- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/test.sh zstd + displayName: 'Zstandard Test' +- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh zstd + displayName: 'Zstandard Build' - task: GitHubRelease@1 inputs: gitHubConnection: 'dimkr' diff --git a/meson.build b/meson.build index 740bb09..6912a07 100644 --- a/meson.build +++ b/meson.build @@ -1,6 +1,6 @@ # This file is part of papaw. # -# Copyright (c) 2019 Dima Krasner +# Copyright (c) 2019, 2020 Dima Krasner # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -61,22 +61,45 @@ endif cfg = configuration_data() +compression_dependencies = [] compression_cflags = ['-DPAPAW_XZ', '-Wno-unused-function'] compression_includes = [include_directories('xz-embedded/userspace'), include_directories('xz-embedded/linux/include/linux')] -cfg.set('XZ_ARGS', '"--check=none", "--lzma2=preset=9e,dict=512KiB"') +cfg.set('CMD', '"xz", "-c", "--check=none", "--lzma2=preset=9e,dict=512KiB"') compression = get_option('compression') if compression == 'lzma' compression_cflags = ['-DPAPAW_LZMA'] compression_includes = [] - cfg.set('XZ_ARGS', '"--format=lzma", "--lzma1=preset=9e,dict=512KiB"') + cfg.set('CMD', '"xz", "-c", "--format=lzma", "--lzma1=preset=9e,dict=512KiB"') +elif compression == 'zstd' + compression_cflags = ['-DPAPAW_ZSTD'] + compression_includes = [] + compression_dependencies += [ + declare_dependency(files: custom_target('zstddeclib', + input: 'zstd/contrib/single_file_decoder/zstddeclib-in.c', + output: 'zstddeclib.c', + command: [ + join_paths(meson.source_root(), 'zstd/contrib/single_file_decoder/combine.sh'), + '-r', + join_paths(meson.source_root(), 'zstd/lib'), + '-r', + join_paths(meson.source_root(), 'zstd/lib/common'), + '-r', + join_paths(meson.source_root(), 'zstd/lib/decompress'), + '-o', + '@OUTPUT@', + '@INPUT@' + ])) + ] + cfg.set('CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') endif papaw = executable('papaw', 'papaw.c', include_directories: compression_includes, c_args: cflags + ['-fvisibility=hidden'] + compression_cflags + config_cflags, - link_args: ldflags) + link_args: ldflags, + dependencies: compression_dependencies) papawu = executable('papawu', 'papaw.c', diff --git a/meson_options.txt b/meson_options.txt index 7cd2d6e..43ce11b 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,4 +2,4 @@ option('dir_prefix', type: 'string', value: '/tmp') option('allow_coredumps', type: 'boolean') option('allow_ptrace', type: 'boolean') option('ci', type: 'boolean', value: false) -option('compression', type: 'combo', choices: ['xz', 'lzma']) +option('compression', type: 'combo', choices: ['xz', 'lzma', 'zstd']) diff --git a/papaw.c b/papaw.c index 0a7979c..3b946f5 100644 --- a/papaw.c +++ b/papaw.c @@ -65,8 +65,10 @@ static void xfree(void *); # include "xz-embedded/linux/lib/xz/xz_dec_lzma2.c" # include "xz-embedded/linux/lib/xz/xz_dec_stream.c" #elif defined(PAPAW_LZMA) -# include "lzma/C/LzmaDec.c" +# include "lzma/C/LzmaDec.c" # define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8 +#elif defined(PAPAW_ZSTD) +# include "zstddeclib.c" #endif #ifdef PAPAW_XZ @@ -134,13 +136,15 @@ static bool extract(const int out, unsigned char *p; ELzmaStatus status; const ISzAlloc alloc = {xalloc, xfree}; +#elif defined(PAPAW_ZSTD) + unsigned char *p; #endif void *map; if (ftruncate(out, (off_t)olen) < 0) return false; -#if defined(PAPAW_XZ) || defined(PAPAW_LZMA) +#if defined(PAPAW_XZ) || defined(PAPAW_LZMA) || defined(PAPAW_ZSTD) if (clen != olen) goto decompress; #endif @@ -205,6 +209,22 @@ static bool extract(const int out, return false; } + munmap(p, (size_t)olen); + return true; +#elif defined(PAPAW_ZSTD) +decompress: + p = mmap(NULL, (size_t)olen, PROT_WRITE, MAP_SHARED, out, 0); + if (p == MAP_FAILED) + return false; + + if (ZSTD_isError(ZSTD_decompress(p, + (size_t)olen, + data, + (size_t)clen))) { + munmap(p, (size_t)olen); + return false; + } + munmap(p, (size_t)olen); return true; #endif diff --git a/papawify.in b/papawify.in index 16e8a7f..6f3b696 100755 --- a/papawify.in +++ b/papawify.in @@ -2,7 +2,7 @@ # This file is part of papaw. # -# Copyright (c) 2019 Dima Krasner +# Copyright (c) 2019, 2020 Dima Krasner # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -34,7 +34,7 @@ parser.add_argument('output') parser.add_argument('-u', '--uncompressed', dest='uncompressed', action='store_true') args = parser.parse_args() -cmd = ["xz", @XZ_ARGS@, "-c"] +cmd = [@CMD@] if args.uncompressed: cmd = ["cat"] diff --git a/zstd b/zstd new file mode 160000 index 0000000..10f0e69 --- /dev/null +++ b/zstd @@ -0,0 +1 @@ +Subproject commit 10f0e6993f9d2f682da6d04aa2385b7d53cbb4ee From dfd2cdd78415a8ed914f6fda5409376a71d779b1 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 08:25:06 +0200 Subject: [PATCH 02/12] meson: fix build failures --- meson.build | 39 +++++++++++++++++++-------------------- papaw.c | 6 +++--- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/meson.build b/meson.build index 6912a07..7ad810f 100644 --- a/meson.build +++ b/meson.build @@ -61,7 +61,7 @@ endif cfg = configuration_data() -compression_dependencies = [] +compression_sources = [] compression_cflags = ['-DPAPAW_XZ', '-Wno-unused-function'] compression_includes = [include_directories('xz-embedded/userspace'), include_directories('xz-embedded/linux/include/linux')] cfg.set('CMD', '"xz", "-c", "--check=none", "--lzma2=preset=9e,dict=512KiB"') @@ -74,32 +74,31 @@ if compression == 'lzma' elif compression == 'zstd' compression_cflags = ['-DPAPAW_ZSTD'] compression_includes = [] - compression_dependencies += [ - declare_dependency(files: custom_target('zstddeclib', - input: 'zstd/contrib/single_file_decoder/zstddeclib-in.c', - output: 'zstddeclib.c', - command: [ - join_paths(meson.source_root(), 'zstd/contrib/single_file_decoder/combine.sh'), - '-r', - join_paths(meson.source_root(), 'zstd/lib'), - '-r', - join_paths(meson.source_root(), 'zstd/lib/common'), - '-r', - join_paths(meson.source_root(), 'zstd/lib/decompress'), - '-o', - '@OUTPUT@', - '@INPUT@' - ])) + compression_sources += [ + custom_target('zstddeclib', + input: 'zstd/contrib/single_file_decoder/zstddeclib-in.c', + output: 'zstddeclib.h', + command: [ + join_paths(meson.source_root(), 'zstd/contrib/single_file_decoder/combine.sh'), + '-r', + join_paths(meson.source_root(), 'zstd/lib'), + '-r', + join_paths(meson.source_root(), 'zstd/lib/common'), + '-r', + join_paths(meson.source_root(), 'zstd/lib/decompress'), + '-o', + '@OUTPUT@', + '@INPUT@' + ]) ] cfg.set('CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') endif papaw = executable('papaw', - 'papaw.c', + ['papaw.c'] + compression_sources, include_directories: compression_includes, c_args: cflags + ['-fvisibility=hidden'] + compression_cflags + config_cflags, - link_args: ldflags, - dependencies: compression_dependencies) + link_args: ldflags) papawu = executable('papawu', 'papaw.c', diff --git a/papaw.c b/papaw.c index 3b946f5..7710732 100644 --- a/papaw.c +++ b/papaw.c @@ -65,10 +65,10 @@ static void xfree(void *); # include "xz-embedded/linux/lib/xz/xz_dec_lzma2.c" # include "xz-embedded/linux/lib/xz/xz_dec_stream.c" #elif defined(PAPAW_LZMA) -# include "lzma/C/LzmaDec.c" -# define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8 +# include "lzma/C/LzmaDec.c" +# define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8 #elif defined(PAPAW_ZSTD) -# include "zstddeclib.c" +# include "zstddeclib.h" #endif #ifdef PAPAW_XZ From ae69b86b83d45bf4288e1bc31ded010363355553 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 09:03:27 +0200 Subject: [PATCH 03/12] papaw: add Deflate support --- README.md | 2 +- azure-pipelines.yml | 4 ++++ meson.build | 4 ++++ meson_options.txt | 2 +- miniz | 2 +- papaw.c | 50 ++++++++++++++++++++++++++++++++++++++------- 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c2e877f..7651463 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It is designed to be portable across different devices, therefore it avoids mach ## Implementation -papaw consists of a small executable (~15-40K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html), the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor or the [Zstandard](https://github.com/facebook/zstd) decompressor. It extracts a compressed executable appended to it by the papawify script. +papaw consists of a small executable (~15-40K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html), the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor, the [Zstandard](https://github.com/facebook/zstd) decompressor or the [miniz](https://github.com/richgel999/miniz) decompressor. It extracts a compressed executable appended to it by the papawify script. The payload executable is extracted to a temporary file. When running as root, this is done by mounting a tmpfs file system and lazily unmounting it before the extraction. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d560744..62a14cf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,6 +41,10 @@ steps: displayName: 'Zstandard Test' - script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh zstd displayName: 'Zstandard Build' +- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/test.sh deflate + displayName: 'Deflate Test' +- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh deflate + displayName: 'Deflate Build' - task: GitHubRelease@1 inputs: gitHubConnection: 'dimkr' diff --git a/meson.build b/meson.build index 7ad810f..c526527 100644 --- a/meson.build +++ b/meson.build @@ -92,6 +92,10 @@ elif compression == 'zstd' ]) ] cfg.set('CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') +elif compression == 'deflate' + compression_cflags = ['-DPAPAW_DEFLATE'] + compression_includes = [] + cfg.set('CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(sys.stdin.buffer.read(), 9))"') endif papaw = executable('papaw', diff --git a/meson_options.txt b/meson_options.txt index 43ce11b..bf9b65c 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,4 +2,4 @@ option('dir_prefix', type: 'string', value: '/tmp') option('allow_coredumps', type: 'boolean') option('allow_ptrace', type: 'boolean') option('ci', type: 'boolean', value: false) -option('compression', type: 'combo', choices: ['xz', 'lzma', 'zstd']) +option('compression', type: 'combo', choices: ['xz', 'lzma', 'zstd', 'deflate']) diff --git a/miniz b/miniz index e71344b..e48d8ba 160000 --- a/miniz +++ b/miniz @@ -1 +1 @@ -Subproject commit e71344bdd7e8e6c18fb67721db7a634ffa202f9e +Subproject commit e48d8bab887deeddb608d713128529c9261bf9e7 diff --git a/papaw.c b/papaw.c index 7710732..ffbad88 100644 --- a/papaw.c +++ b/papaw.c @@ -44,15 +44,31 @@ #endif #ifdef PAPAW_XZ -# define MINIZ_NO_ARCHIVE_APIS # define MINIZ_NO_ZLIB_APIS +#endif + +#if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE) +# define MINIZ_NO_ARCHIVE_APIS + +static void *xalloc(size_t); +static void xfree(void *); +# include "miniz/miniz_common.h" +# undef MZ_MALLOC +# undef MZ_FREE +# define MZ_MALLOC xalloc +# define MZ_FREE xfree + +# ifdef PAPAW_DEFLATE +# include "miniz/miniz_tinfl.c" +# endif + # include "miniz/miniz.c" +#endif +#ifdef PAPAW_XZ # define XZ_EXTERN static # include "xz-embedded/userspace/xz_config.h" -static void *xalloc(size_t); -static void xfree(void *); # undef kmalloc # define kmalloc(size, flags) xalloc(size) # undef kfree @@ -80,9 +96,9 @@ static uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc) #endif -#if defined(PAPAW_LZMA) || defined(PAPAW_XZ) +#if defined(PAPAW_LZMA) || defined(PAPAW_XZ) || defined(PAPAW_DEFLATE) -# ifdef PAPAW_XZ +# if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE) static void *xalloc(size_t size) # else static void *xalloc(const ISzAlloc *p, size_t size) @@ -107,7 +123,7 @@ static void *xalloc(const ISzAlloc *p, size_t size) return ptr + sizeof(size_t); } -# ifdef PAPAW_XZ +# if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE) static void xfree(void *address) # else static void xfree(const ISzAlloc *p, void *address) @@ -138,13 +154,16 @@ static bool extract(const int out, const ISzAlloc alloc = {xalloc, xfree}; #elif defined(PAPAW_ZSTD) unsigned char *p; +#elif defined(PAPAW_DEFLATE) + unsigned char *p; + mz_ulong outlen; #endif void *map; if (ftruncate(out, (off_t)olen) < 0) return false; -#if defined(PAPAW_XZ) || defined(PAPAW_LZMA) || defined(PAPAW_ZSTD) +#if defined(PAPAW_XZ) || defined(PAPAW_LZMA) || defined(PAPAW_ZSTD) || defined(PAPAW_DEFLATE) if (clen != olen) goto decompress; #endif @@ -225,6 +244,23 @@ static bool extract(const int out, return false; } + munmap(p, (size_t)olen); + return true; +#elif defined(PAPAW_DEFLATE) +decompress: + p = mmap(NULL, (size_t)olen, PROT_WRITE, MAP_SHARED, out, 0); + if (p == MAP_FAILED) + return false; + + outlen = (mz_ulong)olen; + if (mz_uncompress(p, + &outlen, + data, + (mz_ulong)clen) != MZ_OK) { + munmap(p, (size_t)olen); + return false; + } + munmap(p, (size_t)olen); return true; #endif From a726fca55084c4207da2f03c87183fc5b075a497 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 16:49:35 +0200 Subject: [PATCH 04/12] ci: add a prebuilt container --- Dockerfile | 25 +++++++++++++++++++++++++ azure-pipelines.yml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..abea351 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# This file is part of papaw. +# +# Copyright (c) 2020 Dima Krasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +FROM dimkr/c-dev:clang + +RUN apt-get -qq update && apt-get -y install zstd diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 62a14cf..2aefb58 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,7 @@ pool: vmImage: 'Ubuntu 16.04' variables: - imageName: dimkr/c-dev:latest + imageName: dimkr/papaw steps: - checkout: self From 1a94f2a1917d7f30a67e762b5c5aed3dedafab66 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 16:52:45 +0200 Subject: [PATCH 05/12] ci: add toolchains --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index abea351..e070497 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,3 +23,4 @@ FROM dimkr/c-dev:clang RUN apt-get -qq update && apt-get -y install zstd +RUN for i in arm-any32-linux-musleabi armeb-any32-linux-musleabi mips-any32-linux-musl mipsel-any32-linux-musl i386-any32-linux-musl; do wget -qO- https://github.com/dimkr/toolchains/releases/latest/download/$i.tar.gz | tar -xzf - -C /; done From f51bf702735ee615261a115dd40fd0a1589edbdb Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 17:11:11 +0200 Subject: [PATCH 06/12] papawify: fix breakage --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index c526527..564f66d 100644 --- a/meson.build +++ b/meson.build @@ -95,7 +95,7 @@ elif compression == 'zstd' elif compression == 'deflate' compression_cflags = ['-DPAPAW_DEFLATE'] compression_includes = [] - cfg.set('CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(sys.stdin.buffer.read(), 9))"') + cfg.set('CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(open(sys.argv[1], \'rb\').read(), 9))"') endif papaw = executable('papaw', From a435377633fcf9162fcbf514b08c69c0007df979 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 17:31:30 +0200 Subject: [PATCH 07/12] papaw: make zstd smaller --- papaw.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/papaw.c b/papaw.c index ffbad88..58fe5ba 100644 --- a/papaw.c +++ b/papaw.c @@ -84,6 +84,12 @@ static void xfree(void *); # include "lzma/C/LzmaDec.c" # define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8 #elif defined(PAPAW_ZSTD) +# define DYNAMIC_BMI2 0 +# define ZSTD_NO_INLINE +# define ZSTDLIB_VISIBILITY +# define ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +# define HUF_FORCE_DECOMPRESS_X1 +# define MEM_FORCE_MEMORY_ACCESS 0 # include "zstddeclib.h" #endif From 7046180322de22fd4a701f763bad36e803e6da28 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 17:44:45 +0200 Subject: [PATCH 08/12] unpapawify: fix breakage --- ci/build.sh | 2 +- meson.build | 15 +++++++++++---- papawify.in | 2 +- unpapawify => unpapawify.in | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) rename unpapawify => unpapawify.in (94%) diff --git a/ci/build.sh b/ci/build.sh index 6a3a7af..8b9f897 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -32,4 +32,4 @@ do done install -m 755 build-$1-arm-any32-linux-musleabi/papawify artifacts/papawify-$1 -install -m 755 unpapawify artifacts/unpapawify-$1 +install -m 755 build-$1-arm-any32-linux-musleabi/unpapawify artifacts/unpapawify-$1 diff --git a/meson.build b/meson.build index 564f66d..27b5f7f 100644 --- a/meson.build +++ b/meson.build @@ -64,13 +64,14 @@ cfg = configuration_data() compression_sources = [] compression_cflags = ['-DPAPAW_XZ', '-Wno-unused-function'] compression_includes = [include_directories('xz-embedded/userspace'), include_directories('xz-embedded/linux/include/linux')] -cfg.set('CMD', '"xz", "-c", "--check=none", "--lzma2=preset=9e,dict=512KiB"') +cfg.set('COMPRESSION_CMD', '"xz", "-c", "--check=none", "--lzma2=preset=9e,dict=512KiB"') +cfg.set('DECOMPRESSION_CMD', '"xz", "-d"') compression = get_option('compression') if compression == 'lzma' compression_cflags = ['-DPAPAW_LZMA'] compression_includes = [] - cfg.set('CMD', '"xz", "-c", "--format=lzma", "--lzma1=preset=9e,dict=512KiB"') + cfg.set('COMPRESSION_CMD', '"xz", "-c", "--format=lzma", "--lzma1=preset=9e,dict=512KiB"') elif compression == 'zstd' compression_cflags = ['-DPAPAW_ZSTD'] compression_includes = [] @@ -91,11 +92,13 @@ elif compression == 'zstd' '@INPUT@' ]) ] - cfg.set('CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') + cfg.set('COMPRESSION_CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') + cfg.set('DECOMPRESSION_CMD', '"zstd", "-d"') elif compression == 'deflate' compression_cflags = ['-DPAPAW_DEFLATE'] compression_includes = [] - cfg.set('CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(open(sys.argv[1], \'rb\').read(), 9))"') + cfg.set('COMPRESSION_CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(open(sys.argv[1], \'rb\').read(), 9))"') + cfg.set('DECOMPRESSION_CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.decompress(sys.stdin.buffer.read()))"') endif papaw = executable('papaw', @@ -123,6 +126,10 @@ papawify = configure_file(input: 'papawify.in', output: 'papawify', configuration: cfg) +unpapawify = configure_file(input: 'unpapawify.in', + output: 'unpapawify', + configuration: cfg) + if get_option('ci') libpapaw_so = shared_library('papaw', 'exe.c', diff --git a/papawify.in b/papawify.in index 6f3b696..0210a03 100755 --- a/papawify.in +++ b/papawify.in @@ -34,7 +34,7 @@ parser.add_argument('output') parser.add_argument('-u', '--uncompressed', dest='uncompressed', action='store_true') args = parser.parse_args() -cmd = [@CMD@] +cmd = [@COMPRESSION_CMD@] if args.uncompressed: cmd = ["cat"] diff --git a/unpapawify b/unpapawify.in similarity index 94% rename from unpapawify rename to unpapawify.in index 19b7b00..95d05b8 100755 --- a/unpapawify +++ b/unpapawify.in @@ -2,7 +2,7 @@ # This file is part of papaw. # -# Copyright (c) 2019 Dima Krasner +# Copyright (c) 2019, 2020 Dima Krasner # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -34,7 +34,7 @@ with open(sys.argv[1], "rb") as infp: with open(sys.argv[2], "wb") as fp: fp.write(data[-(size + 8):-8]) else: - xz = subprocess.Popen(["xz", "-d"], + xz = subprocess.Popen([@DECOMPRESSION_CMD@], stdin=subprocess.PIPE, stdout=open(sys.argv[2], "wb")) xz.stdin.write(data[-(size + 8):-8]) From 31985f97f737c2f51f0446497d5a58ef2432b230 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 17:46:17 +0200 Subject: [PATCH 09/12] ci: fix breakage --- ci/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test.sh b/ci/test.sh index 6d00c30..1151d89 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -73,7 +73,7 @@ test -z "`grep sh-packed-$1 /proc/$$/maps`" test ! -s /proc/$$/exe # make sure unpacking works -./unpapawify sh-packed-$1 sh-unpacked-$1 +./build-$1/unpapawify sh-packed-$1 sh-unpacked-$1 cmp /bin/sh sh-unpacked-$1 # packed executables can be deleted while running @@ -115,5 +115,5 @@ test x`./build-$1/test_putser` = xhello test x`./build-$1/test_uncompressed` = xhello # make sure unpacking of uncompressed binaries works -./unpapawify build-$1/test_uncompressed putser-unpacked +./build-$1/unpapawify build-$1/test_uncompressed putser-unpacked cmp build-$1/putser putser-unpacked From 8614f249e38eb92d590bbc0f549671b793894943 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 18:53:33 +0200 Subject: [PATCH 10/12] meson: make zstd quiet --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 27b5f7f..c47952e 100644 --- a/meson.build +++ b/meson.build @@ -92,7 +92,7 @@ elif compression == 'zstd' '@INPUT@' ]) ] - cfg.set('COMPRESSION_CMD', '"zstd", "-c", "--ultra", "-22", "--no-check"') + cfg.set('COMPRESSION_CMD', '"zstd", "-q", "-c", "--ultra", "-22", "--no-check"') cfg.set('DECOMPRESSION_CMD', '"zstd", "-d"') elif compression == 'deflate' compression_cflags = ['-DPAPAW_DEFLATE'] From fa756130aa57bf171c0a3256ddb1078e56a46b3c Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 18:56:20 +0200 Subject: [PATCH 11/12] README: make the decompressors list clearer --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7651463..9e938d0 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,17 @@ It is designed to be portable across different devices, therefore it avoids mach ## Implementation -papaw consists of a small executable (~15-40K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html), the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor, the [Zstandard](https://github.com/facebook/zstd) decompressor or the [miniz](https://github.com/richgel999/miniz) decompressor. It extracts a compressed executable appended to it by the papawify script. +papaw consists of a small executable (~15-40K) containing a decompressor. It extracts a compressed executable appended to it by the papawify script. The payload executable is extracted to a temporary file. When running as root, this is done by mounting a tmpfs file system and lazily unmounting it before the extraction. +## Supported Decompressors + +* [XZ Embedded](https://tukaani.org/xz/embedded.html) (the default) +* The [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor +* The [Zstandard](https://github.com/facebook/zstd) decompressor +* [miniz](https://github.com/richgel999/miniz) + ## Usage papaw uses [Meson](http://mesonbuild.com/) as its build system. To pack an executable using papaw, build papaw, then use papawify to pack the executable. From 5f13b0d8532d525bfd1e28301a09ecee269c465d Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Mon, 20 Jan 2020 19:00:36 +0200 Subject: [PATCH 12/12] README: document the advantages of LZMA --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9e938d0..da8a7ab 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,14 @@ papaw consists of a small executable (~15-40K) containing a decompressor. It ext The payload executable is extracted to a temporary file. When running as root, this is done by mounting a tmpfs file system and lazily unmounting it before the extraction. -## Supported Decompressors +## Supported Compression Algorithms -* [XZ Embedded](https://tukaani.org/xz/embedded.html) (the default) -* The [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor -* The [Zstandard](https://github.com/facebook/zstd) decompressor -* [miniz](https://github.com/richgel999/miniz) +* LZMA2, using [XZ Embedded](https://tukaani.org/xz/embedded.html) (the default) +* LZMA1, using the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor +* Zstandard, using the [zstd](https://github.com/facebook/zstd) decompressor +* Deflate, using [miniz](https://github.com/richgel999/miniz) + +The first two are extremely similar in compression ratio, code size, memory usage and speed. ## Usage