diff --git a/crabz/README.md b/crabz/README.md new file mode 100644 index 000000000..352b0170f --- /dev/null +++ b/crabz/README.md @@ -0,0 +1,116 @@ +--- +title: Crabz +homepage: https://github.com/sstadick/crabz +tagline: | + crabz: multi-threaded gzip (like pigz, but in Rust) +--- + +To update or switch versions, run `webi crabz@stable` (or `@0.8`, `@beta`, etc). + +## Cheat Sheet + +> `crabz` brings the power of multi-core compression to gzip and deflate. \ +> (and a few other formats + other useful features) + +gzip, faster. + +```sh +crabz -I ./example.json +crabz -d -I ./example.json.gz +``` + +```text +Compressing (gzip) with 8 threads at compression level 6. +Decompressing (gzip) with 8 threads available. +``` + +## Table of Contents + +- Files +- Tar +- Other Formats + +### Files + +These are the files/directories that are created and/or modified with this +install: + +```text +~/.config/envman/PATH.env +~/.local/bin/crabz +``` + +#### How to Optimize + +| Flag | Value | Comments | +| ----------------------------- | ----- | ------------------------------------------ | +| `-l`, `--compression-level` | 1-9 | higher is slower | +| `-p`, `--compression-threads` | 8 | set to the number of available cores | +| | | (but no more than 4 for decompression) | +| `-P`, `--pin-at` | 0 | pin to physical cores, starting at N | +| | | (so 4 threads starting at 0 is 0, 1, 2, 3) | + +```sh +crabz -l 9 -p 8 -I ./example.tar + +crabz -d -p 4 -I ./example.tar.gz +``` + +#### How to use with Tar + +Tar and then compress: + +```sh +tar cv ./example/ | crabz -o ./example.tar.gz +``` + +Or decompress and then untar: + +```sh +crabz -d ./example.tar.gz | tar xv +``` + +#### How to use with other formats + +`crabz` supports most of the _LZ77 with Huffman coding_ compression formats: + +| Format | Extension | Notes | +| ---------------- | --------- | ------------------------------------ | +| `gzip` | `.gz` | of GNU fame | +| [`bgzf`][bgzf] | `.gz` | supports random-access decompression | +| [`mgzip`][mgzip] | `.gz` | of python fame | +| `zlib` | `.zz` | of PNG fame, also `.z` | +| [`snap`][snap] | `.sz` | of LevelDB and MongoDB fame | +| `deflate` | `.gz` | the O.G. LZ77 | + +```sh +crabz --format mgzip -I ./example.tar +``` + +```sh +# DO NOT decompress in-place +crabz --format mgzip -d ./example.tar.gz -o ./example.tar + +# verify before removing the original +tar tf ./example.tar +``` + +⚠️ **Warnings**: + +- DO NOT deflate in-place with non-standard formats: \ + Although `gunzip` will work correctly on files compressed with `mgzip` or + `bgzf`, some combinations (ex: decompressing from `mgzip` with `bgzf`) could + result in corruption! +- `tar xvf` and `gzip -l` may report incorrect information, even though `gunzip` + will work + +See also: + +- https://dev.to/biellls/compression-clearing-the-confusion-on-zip-gzip-zlib-and-deflate-15g1 + +(p.s. `zip` isn't in the list because it's a container format like `tar`, not a +zip format) + +[snap]: https://github.com/google/snappy/blob/main/format_description.txt +[bgzf]: https://samtools.github.io/hts-specs/SAMv1.pdf +[mgzip]: https://pypi.org/project/mgzip/ diff --git a/crabz/install.ps1 b/crabz/install.ps1 new file mode 100644 index 000000000..6b5cf7c6f --- /dev/null +++ b/crabz/install.ps1 @@ -0,0 +1,48 @@ +#!/usr/bin/env pwsh + +################# +# Install crabz # +################# + +# Every package should define these variables +$pkg_cmd_name = "crabz" + +$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\crabz.exe" +$pkg_dst_bin = "$Env:USERPROFILE\.local\bin" +$pkg_dst = "$pkg_dst_cmd" + +$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin\crabz.exe" +$pkg_src_bin = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin" +$pkg_src_dir = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION" +$pkg_src = "$pkg_src_cmd" + +New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null +$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE" + +# Fetch archive +IF (!(Test-Path -Path "$pkg_download")) { + Write-Output "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download" + & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part" + & Move-Item "$pkg_download.part" "$pkg_download" +} + +IF (!(Test-Path -Path "$pkg_src_cmd")) { + Write-Output "Installing crabz" + + # TODO: create package-specific temp directory + # Enter tmp + Push-Location .local\tmp + + # Settle unpacked archive into place + Write-Output "Install Location: $pkg_src_cmd" + New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null + Copy-Item -Path "$pkg_download" -Destination "$pkg_src_cmd" + + # Exit tmp + Pop-Location +} + +Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" +Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null +New-Item "$pkg_dst_bin" -ItemType Directory -Force | Out-Null +Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse diff --git a/crabz/install.sh b/crabz/install.sh new file mode 100644 index 000000000..6349bd2a5 --- /dev/null +++ b/crabz/install.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# The generic functions - version checks, download, extract, etc - are here: +# - https://github.com/webinstall/packages/branches/master/_webi/template.sh + +set -e +set -u + +pkg_cmd_name="crabz" + +# IMPORTANT: this let's other functions know to expect this to be a single file +WEBI_SINGLE=true + +# Every package should define these 6 variables +pkg_cmd_name="crabz" + +pkg_dst_cmd="$HOME/.local/bin/crabz" +#pkg_dst="$pkg_dst_cmd" + +pkg_src_cmd="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin/crabz" +#pkg_src_dir="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin" +#pkg_src="$pkg_src_cmd" + +pkg_get_current_version() { + # 'crabz version' has output in this format: + # crabz git:xxxxxxx + # Since that's not sortable, this prints v0.0.0 + # v0.0.0 + echo "v0.0.0" +} + +pkg_install() { + # $HOME/.local/opt/crabz-v0.3.5/bin + mkdir -p "${pkg_src_bin}" + + # mv ./crabz* "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" + mv ./"${pkg_cmd_name}"* "${pkg_src_cmd}" + + # chmod a+x "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" + chmod a+x "${pkg_src_cmd}" +} + +pkg_link() { + # rm -f "$HOME/.local/bin/crabz" + rm -f "${pkg_dst_cmd}" + + # ln -s "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" "$HOME/.local/bin/crabz" + ln -s "${pkg_src_cmd}" "${pkg_dst_cmd}" +} diff --git a/crabz/releases.js b/crabz/releases.js new file mode 100644 index 000000000..e93491e52 --- /dev/null +++ b/crabz/releases.js @@ -0,0 +1,31 @@ +'use strict'; + +var github = require('../_common/github.js'); +var owner = 'sstadick'; +var repo = 'crabz'; + +module.exports = async function (request) { + let all = await github(request, owner, repo); + + let releases = []; + for (let rel of all.releases) { + let isSrc = rel.download.includes('-src.'); + if (isSrc) { + continue; + } + + releases.push(rel); + } + all.releases = releases; + + return all; +}; + +if (module === require.main) { + module.exports(require('@root/request')).then(function (all) { + all = require('../_webi/normalize.js')(all); + // just select the first 5 for demonstration + all.releases = all.releases.slice(0, 5); + console.info(JSON.stringify(all, null, 2)); + }); +}