From 253ec214d3c2bbf7e453532dfc4c50dbe416e4f5 Mon Sep 17 00:00:00 2001 From: Benjamin Lupton Date: Wed, 14 Feb 2024 20:53:29 +0800 Subject: [PATCH] - command-working: support --sudo - fs-speed: dd SHOULD NOT be used for filesystem performance, using `fio` instead - get-devices: register btrfs devices if any btrfs device is found - gocryptfs-helper: remove `fs-speed` in migration, remove outdated `sharebox` reference - is-missing: return 1 if no paths were missing - mount-helper: if there are btrfs error and we are unmounting, continue, this worksaround rpi5 usb 3 flakiness, as `usbreset` has no effect: https://github.com/raspberrypi/linux/issues/5753#issuecomment-1943553432 - setup-util: check for apt package installation before uninstalling, support --sudo, properly support --no-xdg - add `setup-util-rpi-update` Signed-off-by: Benjamin Lupton --- commands/command-working | 14 ++- commands/fs-speed | 158 ++++++++++++++++++++++----------- commands/get-devices | 3 +- commands/gocryptfs-helper | 7 -- commands/is-missing | 12 +-- commands/mount-helper | 6 +- commands/setup-util | 98 +++++++++++++------- commands/setup-util-rpi-update | 22 +++++ 8 files changed, 218 insertions(+), 102 deletions(-) create mode 100755 commands/setup-util-rpi-update diff --git a/commands/command-working b/commands/command-working index 7eef5e3d0..0ab64703e 100755 --- a/commands/command-working +++ b/commands/command-working @@ -13,7 +13,11 @@ function command_working() ( Working check is done via [ --help] and [ --version] checks. USAGE: - command-working [--] ... + command-working [...options] [--] ... + + OPTIONS: + --sudo + Test the command with sudo. RETURNS: [0] if all commands are working @@ -26,12 +30,15 @@ function command_working() ( } # process - local item commands=() + local item commands=() option_sudo='no' while test "$#" -ne 0; do item="$1" shift case "$item" in '--help' | '-h') help ;; + '--no-sudo'* | '--sudo'*) + option_sudo="$(get-flag-value --affirmative --fallback="$option_sudo" -- "$item")" + ;; '--') commands+=("$@") shift "$#" @@ -57,11 +64,12 @@ function command_working() ( sshd teip trunk + rpi-update ) function check_status { local cmd=() command_exit_status # ensure sbin commands work - if [[ $* == *sbin* ]]; then + if [[ $* == *sbin* ]] || test "$option_sudo" = 'yes'; then cmd+=( 'sudo-helper' '--reason=Your sudo/root/login password is required to verify this command is available and working:' diff --git a/commands/fs-speed b/commands/fs-speed index 74770c4e7..593b9fcf0 100755 --- a/commands/fs-speed +++ b/commands/fs-speed @@ -16,7 +16,12 @@ function fs_speed() ( Calculate the speed at a directory path. USAGE: - fs-speed [--] ... + fs-speed [...options] [--] ... + + OPTIONS: + --user= + --group= + If specified run the removal commands as this and . EOF if test "$#" -ne 0; then echo-error "$@" @@ -25,13 +30,17 @@ function fs_speed() ( } # process - local item option_paths=() + local item option_quiet option_paths=() option_user='' option_group='' + option_quiet="$(echo-quiet-enabled -- "$@")" while test "$#" -ne 0; do item="$1" shift case "$item" in '--help' | '-h') help ;; + '--no-quiet'* | '--quiet'* | '--no-verbose'* | '--verbose'*) ;; # handled by echo-quiet-enabled '--path='*) option_paths+=("${item#*=}") ;; + '--user='*) option_user="${item#*=}" ;; + '--group='*) option_group="${item#*=}" ;; '--') option_paths+=("$@") shift $# @@ -47,63 +56,33 @@ function fs_speed() ( help 'No s provided.' fi - # ===================================== - # Dependencies - - source "$DOROTHY/sources/ripgrep.bash" - # ===================================== # Action - # prepare - local bs megabytes_in_gigabyte=1024 - if is-mac; then - bs="1m" # mac only - else - bs="1M" # linux only - fi + # is broken: https://github.com/axboe/fio/blob/master/examples/fio-seq-read.fio + # is broken: https://github.com/axboe/fio/blob/master/examples/fio-seq-write.fio + + function eval_wrapper { + while test "$1" = '--'; do + shift + done + if test -n "$option_user" -o -n "$option_group"; then + sudo-helper --inherit --user="$option_user" --group="$option_group" -- "$@" + else + eval-helper --no-quiet -- "$@" + fi + } - local scratchpad log bytes path + local scratchpad log bytes megabytes speed path for path in "${option_paths[@]}"; do # start echo-segment --h1="Speed Test: $path" - # prepare a temporary file in the path we wish to speed test - scratchpad="$(fs-temp --root="$path" --file)" - - # prepare a temporary file in our temp directory, to store the output - # which we use for the speed analysis - log="$(fs-temp --directory='fs-speed' --file="$(basename "$scratchpad").log")" - - # ensure they are empty - fs-rm --quiet --no-confirm -- "$scratchpad" "$log" - - # @todo add some type of DEBUG=yes flag to avoid helpers, so we can still see block and time per block calculations - - # write performance - eval-helper --quiet \ - --pending="$(echo-style --bold="Calculating write performance...")" \ - --success="$(echo-style --success="Calculated write performance.")" \ - --failure="$(echo-style --error="Failed calculate write.")" \ - -- dd if=/dev/random of="$scratchpad" bs="$bs" count="$megabytes_in_gigabyte" |& tee "$log" - bytes="$(rg --only-matching --regexp='(\d+) bytes/sec' --replace='$1' <"$log" || :)" - if test -n "$bytes"; then - echo-style "Writes at " --bold="$((bytes / 1024 / 1024)) megabytes" " a second." - fi - - # read performance - eval-helper --quiet \ - --pending="$(echo-style --bold="Calculating read performance...")" \ - --success="$(echo-style --success="Calculated read performance.")" \ - --failure="$(echo-style --error="Failed calculate read.")" \ - -- dd if="$scratchpad" of=/dev/null bs="$bs" |& tee "$log" - bytes="$(rg --only-matching --regexp='(\d+) bytes/sec' --replace='$1' <"$log" || :)" - if test -n "$bytes"; then - echo-style "Reads at " --bold="$((bytes / 1024 / 1024)) megabytes" " a second." - fi - - # cleanup - fs-rm --quiet --no-confirm -- "$scratchpad" "$log" + # https://raw.githubusercontent.com/axboe/fio/master/examples/fio-seq-RW.fio + cd "$path" + fs-rm --quiet --no-confirm --user="$option_user" --group="$option_group" -- ./fio-seq-RW + eval_wrapper -- fio --name=fio-seq-RW --filename=fio-seq-RW --rw=rw --rwmixread=60 --rwmixwrite=40 --bs=256K --direct=0 --numjobs=2 --time_based=1 --runtime=90 --size=1G --ioengine=libaio --iodepth=16 + fs-rm --quiet --no-confirm --user="$option_user" --group="$option_group" -- ./fio-seq-RW # done echo-segment --g1="Speed Test: $path" @@ -114,3 +93,80 @@ function fs_speed() ( if test "$0" = "${BASH_SOURCE[0]}"; then fs_speed "$@" fi + + +# # prepare +# # https://unix.stackexchange.com/a/121888/50703 <-- DO NOT USE DD +# # https://stackoverflow.com/a/50882704/130638 +# # https://unix.stackexchange.com/a/324210/50703 +# # https://stackoverflow.com/a/29935167/130638 <--- DO NOT USE dd +# local bs size=2 count=5 +# if is-mac; then +# bs="${size}g" # mac only +# else +# bs="${size}G" # linux only +# fi + + +# local scratchpad log bytes megabytes speed path +# for path in "${option_paths[@]}"; do +# # start +# echo-segment --h1="Speed Test: $path" + +# # is broken: https://github.com/axboe/fio/blob/master/examples/fio-seq-read.fio +# # is broken: https://github.com/axboe/fio/blob/master/examples/fio-seq-write.fio +# # works but doesn't allow over-rides: https://raw.githubusercontent.com/axboe/fio/master/examples/fio-seq-RW.fio +# # cd "$path" +# # fio --name=fio-seq-RW --filename=fio-seq-RW --rw=rw --rwmixread=60 --rwmixwrite=40 --bs=256K --direct=0 --numjobs=4 --time_based=1 --runtime=90 --size=10G --ioengine=libaio --iodepth=16 +# # fs-rm --quiet --no-confirm -- './fio-seq-RW' + +# # prepare a temporary file in the path we wish to speed test +# scratchpad="$(fs-temp --root="$path" --file)" + +# # prepare a temporary file in our temp directory, to store the output +# # which we use for the speed analysis +# log="$(fs-temp --directory='fs-speed' --file="$(basename "$scratchpad").log")" + +# # ensure they are empty +# fs-rm --quiet --no-confirm -- "$scratchpad" "$log" + +# # write performance +# eval-helper --quiet="$option_quiet" \ +# --pending="$(echo-style --bold='Calculating write performance...')" \ +# --success="$(echo-style --success='Calculated write performance.')" \ +# --failure="$(echo-style --error='Failed calculate write.')" \ +# -- dd if=/dev/urandom of="$scratchpad" bs="$bs" count="$count" |& tee "$log" +# bytes="$(rg --only-matching --regexp='(\d+) bytes/sec' --replace='$1' <"$log" || :)" +# if test -n "$bytes"; then +# megabytes="$(echo-math --precision=0 -- "$bytes / 1024 / 1024")" +# echo-style 'Writes at ' --bold="$megabytes megabytes" ' a second.' +# else +# speed="$(rg --only-matching --regexp='(\d+ [a-zA-Z]+)/s' --replace='$1' <"$log" || :)" +# if test -n "$speed"; then +# echo-style 'Writes at ' --bold="$speed" ' a second.' +# fi +# fi + +# # read performance +# eval-helper --quiet="$option_quiet" \ +# --pending="$(echo-style --bold='Calculating read performance...')" \ +# --success="$(echo-style --success='Calculated read performance.')" \ +# --failure="$(echo-style --error='Failed calculate read.')" \ +# -- dd if="$scratchpad" of=/dev/zero bs="$bs" |& tee "$log" +# bytes="$(rg --only-matching --regexp='(\d+) bytes/sec' --replace='$1' <"$log" || :)" +# if test -n "$bytes"; then +# megabytes="$(echo-math --precision=0 -- "$bytes / 1024 / 1024")" +# echo-style 'Reads at ' --bold="$megabytes megabytes" ' a second.' +# else +# speed="$(rg --only-matching --regexp='(\d+ [a-zA-Z]+)/s' --replace='$1' <"$log" || :)" +# if test -n "$speed"; then +# echo-style 'Reads at ' --bold="$speed" ' a second.' +# fi +# fi + +# # cleanup +# fs-rm --quiet --no-confirm -- "$scratchpad" "$log" + +# # done +# echo-segment --g1="Speed Test: $path" +# done diff --git a/commands/get-devices b/commands/get-devices index 9245aed03..0ff7b5c85 100755 --- a/commands/get-devices +++ b/commands/get-devices @@ -330,11 +330,12 @@ function get_devices() ( local MOUNT_HAYSTACK='' MOUNT_RAID='' function get_all { MOUNT_HAYSTACK="$(mount)" - if test "$option_filesystem" = 'btrfs'; then + if test "$option_filesystem" = 'btrfs' || grep --quiet --fixed-strings --regexp='btrfs' <<< "$MOUNT_HAYSTACK"; then # if you attach a btrfs cluster to a new machine # it may not be completely discovered until the btrfs agent scans for the devices # and btrfs supports mounting partial filesystems, which cause inumerable errors eval-helper --quiet -- btrfs-helper discover >"$tty_target" + MOUNT_HAYSTACK="$(mount)" fi if is-mac; then MOUNT_RAID="$(sudo-helper -- diskutil appleRAID list)" diff --git a/commands/gocryptfs-helper b/commands/gocryptfs-helper index aedff4a16..ebf2b8b87 100755 --- a/commands/gocryptfs-helper +++ b/commands/gocryptfs-helper @@ -381,11 +381,6 @@ function gocryptfs_helper() ( act_mount "$old_vault" "$old_plain" act_mount "$new_vault" "$new_plain" - # speed - if confirm --positive --ppid=$$ -- "Run a speed comparison?"; then - fs-speed -- "$old_plain" "$new_plain" - fi - # migrate cpr --remove --tool=rsync -- \ "${old_plain}/" \ @@ -419,8 +414,6 @@ function gocryptfs_helper() ( # done cat <<-EOF $(echo-style --success="Migration complete. ✅") - When you are ready to start the sharebox with the migrated vault, run: - $(echo-style --code="sharebox start") EOF } diff --git a/commands/is-missing b/commands/is-missing index ccce93142..c6300f188 100755 --- a/commands/is-missing +++ b/commands/is-missing @@ -9,15 +9,15 @@ function is_missing() ( function help { cat <<-EOF >/dev/stderr ABOUT: - Check if any is missing (not a file/directory/symlink). + Check if all s are missing (not a file/directory/symlink). Opposite of [is-present]. USAGE: is-missing [--] ... RETURNS: - [0] if ANY s were missing - [1] if no s were missing + [0] if all s were missing + [1] if any s were missing EOF if test "$#" -ne 0; then echo-error "$@" @@ -53,11 +53,11 @@ function is_missing() ( local input for input in "${option_inputs[@]}"; do # just -e is faulty, as -e fails on broken symlinks - if test ! -e "$input" -a ! -L "$input"; then - return 0 + if test -e "$input" -o -L "$input"; then + return 1 fi done - return 1 + return 0 ) # fire if invoked standalone diff --git a/commands/mount-helper b/commands/mount-helper index 54f26cd18..6c76f1c2e 100755 --- a/commands/mount-helper +++ b/commands/mount-helper @@ -956,7 +956,11 @@ function mount_helper() ( echo-style --success='Validated' ' ' --invert="$option_share" ' ' --invert="$mount_type" ' ' --invert="$option_label" ' ' --invert="$option_count" else echo-style --error='Failed to validate' ' ' --invert="$option_share" ' ' --invert="$mount_type" ' ' --invert="$option_label" ' ' --invert="$option_count" - return 5 # EIO 5 Input/output error + if [[ $option_actions == *'[unmount]'* ]]; then + echo-style --notice='As the goal is to unmount, we are continuing...' + else + return 5 # EIO 5 Input/output error + fi fi # coerce source diff --git a/commands/setup-util b/commands/setup-util index 3c7428c18..e7ba58eeb 100755 --- a/commands/setup-util +++ b/commands/setup-util @@ -146,6 +146,9 @@ function setup_util() ( --no-xdg If truthy, place the binary in /usr/local/bin and symlink the XDG binary to it. + --sudo + When testing the , use sudo. + --confirm Confirm installation of the utility before installing it. @@ -251,7 +254,7 @@ function setup_util() ( # # process - local item option_quiet check='' action='install' installer='' app='' cli='' name='' xdg='yes' confirm='no' optional='no' fallback='yes' upgrade='no' force='no' order=() + local item option_quiet check='' action='install' installer='' app='' cli='' name='' xdg='yes' sudo='no' confirm='no' optional='no' fallback='yes' upgrade='no' force='no' order=() option_quiet="$(echo-quiet-enabled -- "$@")" while test "$#" -ne 0; do item="$1" @@ -272,6 +275,9 @@ function setup_util() ( '--no-xdg'* | '--xdg'*) xdg="$(get-flag-value --affirmative --fallback="$xdg" -- "$item")" ;; + '--no-sudo'* | '--sudo'*) + sudo="$(get-flag-value --affirmative --fallback="$sudo" -- "$item")" + ;; '--no-confirm'* | '--confirm'*) confirm="$(get-flag-value --affirmative --fallback="$confirm" -- "$item")" ;; @@ -481,6 +487,9 @@ function setup_util() ( # macos $HOME/Applications/$filename.appimage mkdir -p "$APPIMAGE_HOME" print_line "$APPIMAGE_HOME/$filename" + elif test "$xdg" = 'no'; then + sudo mkdir -p '/usr/local/bin' + print_line "/usr/local/bin/$filename" else # macos $HOME/.local/bin/$filename mkdir -p "$XDG_BIN_HOME" @@ -503,34 +512,27 @@ function setup_util() ( # verify if [[ $path == *'.app' ]] && test -d "$path"; then return 0 - elif command-working -- "$path"; then - # adjust - local xdg_path="$path" - if test "$xdg" = 'no' && [[ $path == "$XDG_BIN_HOME"* ]]; then - if test ! -e '/usr/local/bin'; then - sudo-helper -- mkdir -p '/usr/local/bin' - fi - path="/usr/local/bin/$filename" - sudo-helper -- mv "$xdg_path" "$path" - symlink-helper --quiet="$option_quiet" --existing="$path" --symlink="$xdg_path" - command-working -- "$path" - fi + elif command-working --sudo="$sudo" -- "$path"; then return 0 - else - echo-style --error='Failed to verify:' ' ' --code="$path" >/dev/stderr - rm_helper "$path" # remove so it doesn't interfere with other install methods and other processes - return 1 fi + + # failure + echo-style --error='Failed to verify:' ' ' --code="$path" >/dev/stderr + # remove so it doesn't interfere with other install methods and other processes + rm_helper "$path" + # return failure + return 1 } function rm_helper { - local path="$1" - - # prefer moving to trash - # because trash allows invocations to continue - # otherwise active invocations will crash (such as setup-util-bash unisntlling the bash it was invoked with) - if is-present -- "$path"; then - fs-rm --quiet --no-confirm --trash -- "$path" - fi + local path + for path in "$@"; do + # prefer moving to trash + # because trash allows invocations to continue + # otherwise active invocations will crash (such as setup-util-bash unisntlling the bash it was invoked with) + if is-present -- "$path"; then + fs-rm --quiet --no-confirm --trash --sudo="$sudo" -- "$path" + fi + done } # prepare fallbacks, for when the package could be installed, but requires the package system to sshe installed first @@ -549,7 +551,7 @@ function setup_util() ( if test -n "$cli"; then if command-exists -- "$cli"; then cli_exists='yes' - if command-working -- "$cli"; then + if command-working --sudo="$sudo" -- "$cli"; then cli_working='yes' else cli_working='no' @@ -594,11 +596,11 @@ function setup_util() ( fi fi - # for upgrade only, remove xdg bin if it exists, as we don't want conflicts if it is deprecated - # do not do this for uninstall, as we want the actuall methods to handle that + # for upgrade only, remove existing bin if it exists, as we don't want conflicts if it is deprecated + # do not do this for uninstall, as we want the actual methods to handle that if test "$upgrade" = 'yes'; then - if test -n "$cli" -a -e "$XDG_BIN_HOME/$cli"; then - rm_helper "$XDG_BIN_HOME/$cli" + if test -n "$cli"; then + rm_helper "$XDG_BIN_HOME/$cli" "/usr/local/bin/$cli" fi fi @@ -1165,6 +1167,18 @@ function setup_util() ( # supports installing/uninstalling downloaded .deb files # sucessor to aptitude # https://wiki.debian.org/Apt + function __are_all_of_these_apt_packages_installed { + local packages=("$@") package + # returns 1 only if all are missing, returns 0 if any are installed + for package in "${packages[@]}"; do + if apt list --installed 2> /dev/null | grep --quiet --fixed-strings --regexp="$package/" &>/dev/null; then + continue + else + return 1 + fi + done + return 0 + } function do_apt_key_fetch { local key="$1" if [[ $key == *'.asc' ]]; then @@ -1353,6 +1367,10 @@ function setup_util() ( # action args if test "$action" = 'uninstall'; then + if ! __are_all_of_these_apt_packages_installed "${packages[@]}"; then + return 200 # ECUSTOM 200 Not applicable to this uninstallation + fi + # don't use `apt remove --purge` as it is equivalent to `apt purge` # and purge removing python will also remove ufw and samba and a whole bunch of other things that depend on python # as such, only use `--auto-remove` as that is safer @@ -3123,7 +3141,7 @@ function setup_util() ( "${args[@]}" "${packages[@]}" # make alias available - if test -n "$cli" -a "$action" = 'install' && command-missing -- "$cli" && command-working -- "$cli.exe"; then + if test -n "$cli" -a "$action" = 'install' && command-missing -- "$cli" && command-working --sudo="$sudo" -- "$cli.exe"; then mkdir -p "$XDG_BIN_HOME" symlink-helper --quiet="$option_quiet" --existing="$(type -P "$cli.exe")" --symlink="$XDG_BIN_HOME/$cli" fi @@ -3313,15 +3331,21 @@ function setup_util() ( --directory='setup-util' \ --directory )" + elif test "$xdg" = 'no'; then + download_filepath="$( + fs-temp \ + --directory='setup-util' \ + --file="$filename" + )" else download_filepath="$bin_path" fi # uninstall? if test "$action" = 'uninstall'; then - if is-missing -- "$bin_path"; then + if is-missing -- "$bin_path" "$XDG_BIN_HOME/$filename" "/usr/local/bin/$filename"; then return 200 # ECUSTOM 200 Not applicable for this package system fi - rm_helper "$bin_path" + rm_helper "$bin_path" "$XDG_BIN_HOME/$filename" "/usr/local/bin/$filename" return 0 fi @@ -3333,6 +3357,14 @@ function setup_util() ( --directory="$download_directory" \ --filepath="$download_filepath" \ --bearer-token="$bearer_token" + # move to bin path + if test -n "$download_filepath" -a "$download_filepath" != "$bin_path"; then + if test "$xdg" = 'no'; then + sudo-helper -- mv -fv "$download_filepath" "$bin_path" + else + mv -fv "$download_filepath" "$bin_path" + fi + fi # build if test -n "$build_eval"; then diff --git a/commands/setup-util-rpi-update b/commands/setup-util-rpi-update new file mode 100755 index 000000000..b28e8dc8f --- /dev/null +++ b/commands/setup-util-rpi-update @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# https://github.com/raspberrypi/rpi-update + +function setup_util_rpi_update() ( + source "$DOROTHY/sources/bash.bash" + + local options=( + --cli='rpi-update' + --no-xdg + --sudo + "$@" + APT='rpi-update' # RASPBERRY PI OS + DOWNLOAD='https://raw.githubusercontent.com/raspberrypi/rpi-update/HEAD/rpi-update' # OTHER + ) + setup-util "${options[@]}" +) + +# fire if invoked standalone +if test "$0" = "${BASH_SOURCE[0]}"; then + setup_util_rpi_update "$@" +fi