giving webi installer the gift of crabz

crabz/README.md

@@ -0,0 +1,116 @@
+---
+title: Crabz
+homepage: https://github.com/sstadick/crabz
+tagline: |
+  crabz: multi-threaded gzip (like pigz, but in Rust)
+---
+
+To update or switch versions, run `webi crabz@stable` (or `@0.8`, `@beta`, etc).
+
+## Cheat Sheet
+
+> `crabz` brings the power of multi-core compression to gzip and deflate. \
+> (and a few other formats + other useful features)
+
+gzip, faster.
+
+```sh
+crabz -I ./example.json
+crabz -d -I ./example.json.gz
+```
+
+```text
+Compressing (gzip) with 8 threads at compression level 6.
+Decompressing (gzip) with 8 threads available.
+```
+
+## Table of Contents
+
+- Files
+- Tar
+- Other Formats
+
+### Files
+
+These are the files/directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/bin/crabz
+```
+
+#### How to Optimize
+
+| Flag                          | Value | Comments                                   |
+| ----------------------------- | ----- | ------------------------------------------ |
+| `-l`, `--compression-level`   | 1-9   | higher is slower                           |
+| `-p`, `--compression-threads` | 8     | set to the number of available cores       |
+|                               |       | (but no more than 4 for decompression)     |
+| `-P`, `--pin-at`              | 0     | pin to physical cores, starting at N       |
+|                               |       | (so 4 threads starting at 0 is 0, 1, 2, 3) |
+
+```sh
+crabz -l 9 -p 8 -I ./example.tar
+
+crabz -d -p 4 -I ./example.tar.gz
+```
+
+#### How to use with Tar
+
+Tar and then compress:
+
+```sh
+tar cv ./example/ | crabz -o ./example.tar.gz
+```
+
+Or decompress and then untar:
+
+```sh
+crabz -d ./example.tar.gz | tar xv
+```
+
+#### How to use with other formats
+
+`crabz` supports most of the _LZ77 with Huffman coding_ compression formats:
+
+| Format           | Extension | Notes                                |
+| ---------------- | --------- | ------------------------------------ |
+| `gzip`           | `.gz`     | of GNU fame                          |
+| [`bgzf`][bgzf]   | `.gz`     | supports random-access decompression |
+| [`mgzip`][mgzip] | `.gz`     | of python fame                       |
+| `zlib`           | `.zz`     | of PNG fame, also `.z`               |
+| [`snap`][snap]   | `.sz`     | of LevelDB and MongoDB fame          |
+| `deflate`        | `.gz`     | the O.G. LZ77                        |
+
+```sh
+crabz --format mgzip -I ./example.tar
+```
+
+```sh
+# DO NOT decompress in-place
+crabz --format mgzip -d ./example.tar.gz -o ./example.tar
+
+# verify before removing the original
+tar tf ./example.tar
+```
+
+⚠️ **Warnings**:
+
+- DO NOT deflate in-place with non-standard formats: \
+   Although `gunzip` will work correctly on files compressed with `mgzip` or
+  `bgzf`, some combinations (ex: decompressing from `mgzip` with `bgzf`) could
+  result in corruption!
+- `tar xvf` and `gzip -l` may report incorrect information, even though `gunzip`
+  will work
+
+See also:
+
+- https://dev.to/biellls/compression-clearing-the-confusion-on-zip-gzip-zlib-and-deflate-15g1
+
+(p.s. `zip` isn't in the list because it's a container format like `tar`, not a
+zip format)
+
+[snap]: https://github.com/google/snappy/blob/main/format_description.txt
+[bgzf]: https://samtools.github.io/hts-specs/SAMv1.pdf
+[mgzip]: https://pypi.org/project/mgzip/

crabz/install.ps1

@@ -0,0 +1,48 @@
+#!/usr/bin/env pwsh
+
+#################
+# Install crabz #
+#################
+
+# Every package should define these variables
+$pkg_cmd_name = "crabz"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\crabz.exe"
+$pkg_dst_bin = "$Env:USERPROFILE\.local\bin"
+$pkg_dst = "$pkg_dst_cmd"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin\crabz.exe"
+$pkg_src_bin = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_cmd"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+# Fetch archive
+IF (!(Test-Path -Path "$pkg_download")) {
+    Write-Output "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
+    Write-Output "Installing crabz"
+
+    # TODO: create package-specific temp directory
+    # Enter tmp
+    Push-Location .local\tmp
+
+    # Settle unpacked archive into place
+    Write-Output "Install Location: $pkg_src_cmd"
+    New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
+    Copy-Item -Path "$pkg_download" -Destination "$pkg_src_cmd"
+
+    # Exit tmp
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
+Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null
+New-Item "$pkg_dst_bin" -ItemType Directory -Force | Out-Null
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

crabz/install.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# The generic functions - version checks, download, extract, etc - are here:
+#   - https://github.com/webinstall/packages/branches/master/_webi/template.sh
+
+set -e
+set -u
+
+pkg_cmd_name="crabz"
+
+# IMPORTANT: this let's other functions know to expect this to be a single file
+WEBI_SINGLE=true
+
+# Every package should define these 6 variables
+pkg_cmd_name="crabz"
+
+pkg_dst_cmd="$HOME/.local/bin/crabz"
+#pkg_dst="$pkg_dst_cmd"
+
+pkg_src_cmd="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin/crabz"
+#pkg_src_dir="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin"
+#pkg_src="$pkg_src_cmd"
+
+pkg_get_current_version() {
+    # 'crabz version' has output in this format:
+    #       crabz git:xxxxxxx
+    # Since that's not sortable, this prints v0.0.0
+    #       v0.0.0
+    echo "v0.0.0"
+}
+
+pkg_install() {
+    # $HOME/.local/opt/crabz-v0.3.5/bin
+    mkdir -p "${pkg_src_bin}"
+
+    # mv ./crabz* "$HOME/.local/opt/crabz-v0.3.5/bin/crabz"
+    mv ./"${pkg_cmd_name}"* "${pkg_src_cmd}"
+
+    # chmod a+x "$HOME/.local/opt/crabz-v0.3.5/bin/crabz"
+    chmod a+x "${pkg_src_cmd}"
+}
+
+pkg_link() {
+    # rm -f "$HOME/.local/bin/crabz"
+    rm -f "${pkg_dst_cmd}"
+
+    # ln -s "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" "$HOME/.local/bin/crabz"
+    ln -s "${pkg_src_cmd}" "${pkg_dst_cmd}"
+}

crabz/releases.js

@@ -0,0 +1,31 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'sstadick';
+var repo = 'crabz';
+
+module.exports = async function (request) {
+  let all = await github(request, owner, repo);
+
+  let releases = [];
+  for (let rel of all.releases) {
+    let isSrc = rel.download.includes('-src.');
+    if (isSrc) {
+      continue;
+    }
+
+    releases.push(rel);
+  }
+  all.releases = releases;
+
+  return all;
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}