Merge 5f13b0d into 6a52c7c

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "xz-embedded"]
 	path = xz-embedded
 	url = https://github.com/dimkr/xz-embedded
+[submodule "zstd"]
+	path = zstd
+	url = https://github.com/facebook/zstd

Dockerfile

@@ -0,0 +1,26 @@
+# This file is part of papaw.
+#
+# Copyright (c) 2020 Dima Krasner
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+FROM dimkr/c-dev:clang
+
+RUN apt-get -qq update && apt-get -y install zstd
+RUN for i in arm-any32-linux-musleabi armeb-any32-linux-musleabi mips-any32-linux-musl mipsel-any32-linux-musl i386-any32-linux-musl; do wget -qO- https://github.com/dimkr/toolchains/releases/latest/download/$i.tar.gz | tar -xzf - -C /; done

README.md

@@ -18,10 +18,19 @@ It is designed to be portable across different devices, therefore it avoids mach
 
 ## Implementation
 
-papaw consists of a small executable (~15-20K) containing [XZ Embedded](https://tukaani.org/xz/embedded.html) or the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor. It extracts a LZMA-compressed executable appended to it by the papawify script.
+papaw consists of a small executable (~15-40K) containing a decompressor. It extracts a compressed executable appended to it by the papawify script.
 
 The payload executable is extracted to a temporary file. When running as root, this is done by mounting a tmpfs file system and lazily unmounting it before the extraction.
 
+## Supported Compression Algorithms
+
+* LZMA2, using [XZ Embedded](https://tukaani.org/xz/embedded.html) (the default)
+* LZMA1, using the [LZMA SDK](https://www.7-zip.org/sdk.html) decompressor
+* Zstandard, using the [zstd](https://github.com/facebook/zstd) decompressor
+* Deflate, using [miniz](https://github.com/richgel999/miniz)
+
+The first two are extremely similar in compression ratio, code size, memory usage and speed.
+
 ## Usage
 
 papaw uses [Meson](http://mesonbuild.com/) as its build system. To pack an executable using papaw, build papaw, then use papawify to pack the executable.

azure-pipelines.yml

@@ -24,7 +24,7 @@ pool:
   vmImage: 'Ubuntu 16.04'
 
 variables:
-  imageName: dimkr/c-dev:latest
+  imageName: dimkr/papaw
 
 steps:
 - checkout: self
@@ -37,6 +37,14 @@ steps:
   displayName: 'LAMA1 Test'
 - script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh lzma
   displayName: 'LAMA1 Build'
+- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/test.sh zstd
+  displayName: 'Zstandard Test'
+- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh zstd
+  displayName: 'Zstandard Build'
+- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/test.sh deflate
+  displayName: 'Deflate Test'
+- script: docker run --privileged -w /root/papaw -v `pwd`:/root/papaw $(imageName) ./ci/build.sh deflate
+  displayName: 'Deflate Build'
 - task: GitHubRelease@1
   inputs:
     gitHubConnection: 'dimkr'

ci/build.sh

@@ -32,4 +32,4 @@ do
 done
 
 install -m 755 build-$1-arm-any32-linux-musleabi/papawify artifacts/papawify-$1
-install -m 755 unpapawify artifacts/unpapawify-$1
+install -m 755 build-$1-arm-any32-linux-musleabi/unpapawify artifacts/unpapawify-$1

ci/test.sh

@@ -73,7 +73,7 @@ test -z "`grep sh-packed-$1 /proc/$$/maps`"
 test ! -s /proc/$$/exe
 
 # make sure unpacking works
-./unpapawify sh-packed-$1 sh-unpacked-$1
+./build-$1/unpapawify sh-packed-$1 sh-unpacked-$1
 cmp /bin/sh sh-unpacked-$1
 
 # packed executables can be deleted while running
@@ -115,5 +115,5 @@ test x`./build-$1/test_putser` = xhello
 test x`./build-$1/test_uncompressed` = xhello
 
 # make sure unpacking of uncompressed binaries works
-./unpapawify build-$1/test_uncompressed putser-unpacked
+./build-$1/unpapawify build-$1/test_uncompressed putser-unpacked
 cmp build-$1/putser putser-unpacked

meson.build

@@ -1,6 +1,6 @@
 # This file is part of papaw.
 #
-# Copyright (c) 2019 Dima Krasner
+# Copyright (c) 2019, 2020 Dima Krasner
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -61,19 +61,48 @@ endif
 
 cfg = configuration_data()
 
+compression_sources = []
 compression_cflags = ['-DPAPAW_XZ', '-Wno-unused-function']
 compression_includes = [include_directories('xz-embedded/userspace'), include_directories('xz-embedded/linux/include/linux')]
-cfg.set('XZ_ARGS', '"--check=none", "--lzma2=preset=9e,dict=512KiB"')
+cfg.set('COMPRESSION_CMD', '"xz", "-c", "--check=none", "--lzma2=preset=9e,dict=512KiB"')
+cfg.set('DECOMPRESSION_CMD', '"xz", "-d"')
 
 compression = get_option('compression')
 if compression == 'lzma'
     compression_cflags = ['-DPAPAW_LZMA']
     compression_includes = []
-    cfg.set('XZ_ARGS', '"--format=lzma", "--lzma1=preset=9e,dict=512KiB"')
+    cfg.set('COMPRESSION_CMD', '"xz", "-c", "--format=lzma", "--lzma1=preset=9e,dict=512KiB"')
+elif compression == 'zstd'
+    compression_cflags = ['-DPAPAW_ZSTD']
+    compression_includes = []
+    compression_sources += [
+        custom_target('zstddeclib',
+                      input: 'zstd/contrib/single_file_decoder/zstddeclib-in.c',
+                      output: 'zstddeclib.h',
+                      command: [
+                          join_paths(meson.source_root(), 'zstd/contrib/single_file_decoder/combine.sh'),
+                          '-r',
+                          join_paths(meson.source_root(), 'zstd/lib'),
+                          '-r',
+                          join_paths(meson.source_root(), 'zstd/lib/common'),
+                          '-r',
+                          join_paths(meson.source_root(), 'zstd/lib/decompress'),
+                          '-o',
+                          '@OUTPUT@',
+                          '@INPUT@'
+                      ])
+    ]
+    cfg.set('COMPRESSION_CMD', '"zstd", "-q", "-c", "--ultra", "-22", "--no-check"')
+    cfg.set('DECOMPRESSION_CMD', '"zstd", "-d"')
+elif compression == 'deflate'
+    compression_cflags = ['-DPAPAW_DEFLATE']
+    compression_includes = []
+    cfg.set('COMPRESSION_CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.compress(open(sys.argv[1], \'rb\').read(), 9))"')
+    cfg.set('DECOMPRESSION_CMD', '"python3", "-c", "import sys, zlib; sys.stdout.buffer.write(zlib.decompress(sys.stdin.buffer.read()))"')
 endif
 
 papaw = executable('papaw',
-                   'papaw.c',
+                   ['papaw.c'] + compression_sources,
                    include_directories: compression_includes,
                    c_args: cflags + ['-fvisibility=hidden'] + compression_cflags + config_cflags,
                    link_args: ldflags)
@@ -97,6 +126,10 @@ papawify = configure_file(input: 'papawify.in',
                           output: 'papawify',
                           configuration: cfg)
 
+unpapawify = configure_file(input: 'unpapawify.in',
+                            output: 'unpapawify',
+                            configuration: cfg)
+
 if get_option('ci')
     libpapaw_so = shared_library('papaw',
                                  'exe.c',

meson_options.txt

@@ -2,4 +2,4 @@ option('dir_prefix', type: 'string', value: '/tmp')
 option('allow_coredumps', type: 'boolean')
 option('allow_ptrace', type: 'boolean')
 option('ci', type: 'boolean', value: false)
-option('compression', type: 'combo', choices: ['xz', 'lzma'])
+option('compression', type: 'combo', choices: ['xz', 'lzma', 'zstd', 'deflate'])

papaw.c

@@ -44,15 +44,31 @@
 #endif
 
 #ifdef PAPAW_XZ
-#   define MINIZ_NO_ARCHIVE_APIS
 #   define MINIZ_NO_ZLIB_APIS
+#endif
+
+#if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE)
+#   define MINIZ_NO_ARCHIVE_APIS
+
+static void *xalloc(size_t);
+static void xfree(void *);
+#   include "miniz/miniz_common.h"
+#   undef MZ_MALLOC
+#   undef MZ_FREE
+#   define MZ_MALLOC xalloc
+#   define MZ_FREE xfree
+
+#   ifdef PAPAW_DEFLATE
+#       include "miniz/miniz_tinfl.c"
+#   endif
+
 #   include "miniz/miniz.c"
+#endif
 
+#ifdef PAPAW_XZ
 #   define XZ_EXTERN static
 
 #   include "xz-embedded/userspace/xz_config.h"
-static void *xalloc(size_t);
-static void xfree(void *);
 #   undef kmalloc
 #   define kmalloc(size, flags) xalloc(size)
 #   undef kfree
@@ -66,7 +82,15 @@ static void xfree(void *);
 #   include "xz-embedded/linux/lib/xz/xz_dec_stream.c"
 #elif defined(PAPAW_LZMA)
 #   include "lzma/C/LzmaDec.c"
-#    define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8
+#   define LZMA_HEADER_SIZE LZMA_PROPS_SIZE + 8
+#elif defined(PAPAW_ZSTD)
+#   define DYNAMIC_BMI2 0
+#   define ZSTD_NO_INLINE
+#   define ZSTDLIB_VISIBILITY
+#   define ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+#   define HUF_FORCE_DECOMPRESS_X1
+#   define MEM_FORCE_MEMORY_ACCESS 0
+#   include "zstddeclib.h"
 #endif
 
 #ifdef PAPAW_XZ
@@ -78,9 +102,9 @@ static uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc)
 
 #endif
 
-#if defined(PAPAW_LZMA) || defined(PAPAW_XZ)
+#if defined(PAPAW_LZMA) || defined(PAPAW_XZ) || defined(PAPAW_DEFLATE)
 
-#   ifdef PAPAW_XZ
+#   if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE)
 static void *xalloc(size_t size)
 #   else
 static void *xalloc(const ISzAlloc *p, size_t size)
@@ -105,7 +129,7 @@ static void *xalloc(const ISzAlloc *p, size_t size)
     return ptr + sizeof(size_t);
 }
 
-#   ifdef PAPAW_XZ
+#   if defined(PAPAW_XZ) || defined(PAPAW_DEFLATE)
 static void xfree(void *address)
 #   else
 static void xfree(const ISzAlloc *p, void *address)
@@ -134,13 +158,18 @@ static bool extract(const int out,
     unsigned char *p;
     ELzmaStatus status;
     const ISzAlloc alloc = {xalloc, xfree};
+#elif defined(PAPAW_ZSTD)
+    unsigned char *p;
+#elif defined(PAPAW_DEFLATE)
+    unsigned char *p;
+    mz_ulong outlen;
 #endif
     void *map;
 
     if (ftruncate(out, (off_t)olen) < 0)
         return false;
 
-#if defined(PAPAW_XZ) || defined(PAPAW_LZMA)
+#if defined(PAPAW_XZ) || defined(PAPAW_LZMA) || defined(PAPAW_ZSTD) || defined(PAPAW_DEFLATE)
     if (clen != olen)
         goto decompress;
 #endif
@@ -205,6 +234,39 @@ static bool extract(const int out,
         return false;
     }
 
+    munmap(p, (size_t)olen);
+    return true;
+#elif defined(PAPAW_ZSTD)
+decompress:
+    p = mmap(NULL, (size_t)olen, PROT_WRITE, MAP_SHARED, out, 0);
+    if (p == MAP_FAILED)
+        return false;
+
+    if (ZSTD_isError(ZSTD_decompress(p,
+                                     (size_t)olen,
+                                     data,
+                                     (size_t)clen))) {
+        munmap(p, (size_t)olen);
+        return false;
+    }
+
+    munmap(p, (size_t)olen);
+    return true;
+#elif defined(PAPAW_DEFLATE)
+decompress:
+    p = mmap(NULL, (size_t)olen, PROT_WRITE, MAP_SHARED, out, 0);
+    if (p == MAP_FAILED)
+        return false;
+
+    outlen = (mz_ulong)olen;
+    if (mz_uncompress(p,
+                      &outlen,
+                      data,
+                      (mz_ulong)clen) != MZ_OK) {
+        munmap(p, (size_t)olen);
+        return false;
+    }
+
     munmap(p, (size_t)olen);
     return true;
 #endif

papawify.in

@@ -2,7 +2,7 @@
 
 # This file is part of papaw.
 #
-# Copyright (c) 2019 Dima Krasner
+# Copyright (c) 2019, 2020 Dima Krasner
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -34,7 +34,7 @@ parser.add_argument('output')
 parser.add_argument('-u', '--uncompressed', dest='uncompressed', action='store_true')
 args = parser.parse_args()
 
-cmd = ["xz", @XZ_ARGS@, "-c"]
+cmd = [@COMPRESSION_CMD@]
 if args.uncompressed:
     cmd = ["cat"]
 

unpapawify → unpapawify.in

@@ -2,7 +2,7 @@
 
 # This file is part of papaw.
 #
-# Copyright (c) 2019 Dima Krasner
+# Copyright (c) 2019, 2020 Dima Krasner
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -34,7 +34,7 @@ with open(sys.argv[1], "rb") as infp:
         with open(sys.argv[2], "wb") as fp:
             fp.write(data[-(size + 8):-8])
     else:
-        xz = subprocess.Popen(["xz", "-d"],
+        xz = subprocess.Popen([@DECOMPRESSION_CMD@],
                               stdin=subprocess.PIPE,
                               stdout=open(sys.argv[2], "wb"))
         xz.stdin.write(data[-(size + 8):-8])