More benchmarks

Author: jpikl

benches/bench.sh

@@ -5,29 +5,53 @@ set -eu
 COMMAND=$1
 NUMBER=0
 
-RESULTS_DIR=target/benches
-RESULTS_FILE=$RESULTS_DIR/$COMMAND.md
+NO_TEST=
+SETUP=
+CLEANUP=
+
+no_test() {
+  NO_TEST=$1
+}
+
+setup() {
+  SETUP=$1
+}
+
+cleanup() {
+  CLEANUP=$1
+}
 
 bench() {
   NUMBER=$((NUMBER + 1))
+
+  [ "$CLEANUP" ] && set -- --cleanup "$CLEANUP" "$@"
+  [ "$SETUP" ] && set -- --setup "$SETUP" "$@"
   set -- hyperfine --warmup 5 "$@"
 
   echo
   echo "## Benchmark #$NUMBER"
+
+  if [ "$NO_TEST" ]; then
+    echo "> **Note:** $NO_TEST"
+  fi
+
   echo
   echo "Command:"
   echo
   echo '```shell'
-  bench_command "$@"
+
+  preview "$@"
+
   echo
   echo '```'
   echo
   echo "Results:"
   echo
-  bench_results "$@"
+
+  execute "$@"
 }
 
-bench_command() {
+preview() {
   printf "%s" "$1"
   shift
 
@@ -39,15 +63,26 @@ bench_command() {
       printf "%s %s" "$1" "$2"
       shift 2
       ;;
+    --setup | --cleanup)
+      printf "%s \"%s\"" "$1" "$(escape "$2")"
+      shift 2
+      ;;
     *)
-      printf "\"%s\"" "$1"
+      printf "\"%s\"" "$(escape "$1")"
       shift
       ;;
     esac
   done
 }
 
-bench_results() {
+escape() {
+  echo "$1" | sed 's/"/\\"/g'
+}
+
+RESULTS_DIR=target/benches
+RESULTS_FILE=$RESULTS_DIR/$COMMAND.md
+
+execute() {
   mkdir -p "$RESULTS_DIR"
   echo >&2
   "$@" --sort mean-time --export-markdown "$RESULTS_FILE" 1>&2

benches/commands/loop.sh

@@ -0,0 +1,5 @@
+# shellcheck shell=sh
+
+bench \
+  "rew loop 10 <$SVEJK_FILE" \
+  "for((i=0;i<10;i++)); do cat $SVEJK_FILE; done"

benches/commands/lower.sh

@@ -0,0 +1,19 @@
+# shellcheck shell=sh
+
+for FILE in "$SVEJK_FILE" "$RUR_FILE"; do
+  bench \
+    "rew lower <$FILE" \
+    "sed 's/\(.*\)/\L\1/' <$FILE" \
+    "while IFS= read -r l;do printf '%s\\n' \"\${l,,}\";done <$FILE"
+done
+
+for FILE in "$SVEJK_FILE" "$RUR_FILE"; do
+  setup "rew ascii <$FILE | rew loop 20 >data.txt"
+  cleanup "rm data.txt"
+  bench \
+    "rew lower <data.txt" \
+    "dd conv=lcase <data.txt" \
+    "tr '[:upper:]' '[:lower:]' <data.txt" \
+    "sed 's/\(.*\)/\L\1/' <data.txt" \
+    "while IFS= read -r l;do printf '%s\\n' \"\${l,,}\";done <data.txt"
+done

benches/commands/skip.sh

@@ -0,0 +1,8 @@
+# shellcheck shell=sh
+
+for FILE in "$SVEJK_FILE" "$RUR_FILE"; do
+  bench \
+    "tail -n +2001 <$FILE" \
+    "coreutils tail -n +2001 <$FILE" \
+    "rew skip 2000 <$FILE"
+done

benches/commands/stream.sh

@@ -0,0 +1,5 @@
+# shellcheck shell=sh
+
+bench \
+  "rew stream {1..100000}" \
+  "printf '%s\n' {1..100000}"

benches/commands/upper.sh

@@ -0,0 +1,20 @@
+# shellcheck shell=sh
+
+for FILE in "$SVEJK_FILE" "$RUR_FILE"; do
+  no_test "\`sed\` and \`bash\` produce slightly different results for characters like \`ß\`"
+  bench \
+    "rew lower <$FILE" \
+    "sed 's/\(.*\)/\U\1/' <$FILE" \
+    "while IFS= read -r l;do printf '%s\\n' \"\${l^^}\";done <$FILE"
+done
+
+for FILE in "$SVEJK_FILE" "$RUR_FILE"; do
+  setup "rew ascii <$FILE | rew loop 20 >data.txt"
+  cleanup "rm data.txt"
+  bench \
+    "rew upper <data.txt" \
+    "dd conv=ucase <data.txt" \
+    "tr '[:lower:]' '[:upper:]' <data.txt" \
+    "sed 's/\(.*\)/\U\1/' <data.txt" \
+    "while IFS= read -r l;do printf '%s\\n' \"\${l^^}\";done <data.txt"
+done

benches/setup.sh

@@ -49,11 +49,8 @@ specs 'tr --version'
 
 BASE_DATA_URL=https://jpikl.github.io/data
 
-SVEJK_REPEAT=100
-SVEJK_INIT="curl $BASE_DATA_URL/$SVEJK_FILE | rew loop $SVEJK_REPEAT >$SVEJK_FILE"
-
-RUR_REPEAT=1000
-RUR_INIT="curl $BASE_DATA_URL/$RUR_FILE | rew loop $RUR_REPEAT >$RUR_FILE"
+SVEJK_INIT="curl $BASE_DATA_URL/$SVEJK_FILE >$SVEJK_FILE"
+RUR_INIT="curl $BASE_DATA_URL/$RUR_FILE >$RUR_FILE"
 
 [ -f "$SVEJK_FILE" ] || sh -c "$SVEJK_INIT"
 [ -f "$RUR_FILE" ] || sh -c "$RUR_INIT"
@@ -68,15 +65,13 @@ echo "$SVEJK_INIT"
 echo '```'
 echo
 echo "- UTF-8, non-ascii (czech diacritics)"
-echo "- 5913 lines (repeated $SVEJK_REPEAT times)"
-echo "- Very long lines (up to 4952 chars)"
+echo "- 5913 very long lines (each up to 4952 chars)"
 echo "- Trimmed whitespaces"
 echo
 echo '```shell'
 echo "$RUR_INIT"
 echo '```'
 echo
 echo "- UTF-8, non-ascii (czech diacritics)"
-echo "- 4477 lines (repeated $RUR_REPEAT times)"
-echo "- Short lines (up to 81 chars)"
+echo "- 4477 short lines (each up to 81 chars)"
 echo "- Untrimmed whitespaces"

benches/test.sh

@@ -10,11 +10,39 @@ RESET="\e[0m"
 COMMAND=$1
 NUMBER=0
 
+SETUP=
+CLEANUP=
+NO_TEST=
+
+no_test() {
+  NO_TEST=true
+}
+
+setup() {
+  SETUP=$1
+}
+
+cleanup() {
+  CLEANUP=$1
+}
+
+reset() {
+  NO_TEST=
+  SETUP=
+  CLEANUP=
+}
+
 bench() {
   NUMBER=$((NUMBER + 1))
 
+  if [ "$NO_TEST" ]; then
+    reset
+    return
+  fi
+
   echo
   printf "%bTesting %s benchmark #%s%b\n" "$BLUE" "$COMMAND" "$NUMBER" "$RESET"
+  sh -c "$SETUP"
 
   FIRST="$1"
   FIRST_RESULT=$(sh -c "$FIRST | cksum")
@@ -34,11 +62,14 @@ bench() {
         "$RED" "$RESET" \
         "$BLUE" "$OTHER" "$RESET" \
         "$BLUE" "$FIRST" "$RESET"
+      sh -c "$CLEANUP"
       exit 1
     fi
   done
 
   printf "%bSUCCESS%b\n" "$GREEN" "$RESET"
+  sh -c "$CLEANUP"
+  reset
 }
 
 # shellcheck disable=SC1090

docs/benchmarks/rew-loop.md

@@ -0,0 +1,23 @@
+# rew loop
+
+This page contains benchmarks for [rew loop](../reference/rew-loop.md) command.
+
+See [benchmark setup](./setup.md) for information about testing environemt and input data files.
+
+## Benchmark #1
+
+Command:
+
+```shell
+hyperfine \
+    --warmup 5 \
+    "rew loop 10 <svejk.txt" \
+    "for((i=0;i<10;i++)); do cat svejk.txt; done"
+```
+
+Results:
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `rew loop 10 <svejk.txt` | 134.5 ± 7.2 | 127.8 | 159.8 | 1.00 |
+| `for((i=0;i<10;i++)); do cat svejk.txt; done` | 209.0 ± 5.6 | 202.0 | 220.9 | 1.55 ± 0.09 |

docs/benchmarks/rew-lower.md

@@ -0,0 +1,93 @@
+# rew lower
+
+This page contains benchmarks for [rew lower](../reference/rew-lower.md) command.
+
+See [benchmark setup](./setup.md) for information about testing environemt and input data files.
+
+## Benchmark #1
+
+Command:
+
+```shell
+hyperfine \
+    --warmup 5 \
+    "rew lower <svejk.txt" \
+    "sed 's/./\l&/g' <svejk.txt" \
+    "sed 's/\(.*\)/\L\1/' <svejk.txt" \
+    "while IFS= read -r l;do printf '%s\n' \"${l,,}\";done <svejk.txt"
+```
+
+Results:
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `rew lower <svejk.txt` | 13.5 ± 0.8 | 12.3 | 15.8 | 1.00 |
+| `sed 's/\(.*\)/\L\1/' <svejk.txt` | 207.8 ± 2.7 | 205.3 | 212.9 | 15.34 ± 0.88 |
+| `while IFS= read -r l;do printf '%s\n' "${l,,}";done <svejk.txt` | 244.7 ± 2.6 | 237.3 | 247.0 | 18.06 ± 1.02 |
+| `sed 's/./\l&/g' <svejk.txt` | 285.8 ± 13.3 | 277.5 | 321.6 | 21.10 ± 1.53 |
+
+## Benchmark #2
+
+Command:
+
+```shell
+hyperfine \
+    --warmup 5 \
+    "rew lower <rur.txt" \
+    "sed 's/./\l&/g' <rur.txt" \
+    "sed 's/\(.*\)/\L\1/' <rur.txt" \
+    "while IFS= read -r l;do printf '%s\n' \"${l,,}\";done <rur.txt"
+```
+
+Results:
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `rew lower <rur.txt` | 2.6 ± 0.4 | 2.1 | 5.3 | 1.00 |
+| `sed 's/\(.*\)/\L\1/' <rur.txt` | 28.5 ± 1.2 | 26.4 | 31.8 | 10.91 ± 1.72 |
+| `sed 's/./\l&/g' <rur.txt` | 32.5 ± 0.8 | 30.6 | 34.9 | 12.43 ± 1.91 |
+| `while IFS= read -r l;do printf '%s\n' "${l,,}";done <rur.txt` | 74.6 ± 2.4 | 69.2 | 80.7 | 28.51 ± 4.42 |
+
+## Benchmark #3
+
+Command:
+
+```shell
+hyperfine \
+    --warmup 5 \
+    --setup "rew ascii <svejk.txt | rew loop 20 >data.txt" \
+    --cleanup "rm data.txt" \
+    "rew lower <data.txt" \
+    "dd conv=lcase <data.txt" \
+    "tr '[:upper:]' '[:lower:]' <data.txt"
+```
+
+Results:
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `rew lower <data.txt` | 10.3 ± 0.8 | 9.3 | 13.0 | 1.00 |
+| `tr '[:upper:]' '[:lower:]' <data.txt` | 28.0 ± 1.1 | 26.0 | 30.5 | 2.71 ± 0.23 |
+| `dd conv=lcase <data.txt` | 146.5 ± 3.5 | 140.3 | 153.8 | 14.19 ± 1.09 |
+
+## Benchmark #4
+
+Command:
+
+```shell
+hyperfine \
+    --warmup 5 \
+    --setup "rew ascii <rur.txt | rew loop 20 >data.txt" \
+    --cleanup "rm data.txt" \
+    "rew lower <data.txt" \
+    "dd conv=lcase <data.txt" \
+    "tr '[:upper:]' '[:lower:]' <data.txt"
+```
+
+Results:
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `rew lower <data.txt` | 2.6 ± 0.3 | 2.1 | 5.0 | 1.00 |
+| `tr '[:upper:]' '[:lower:]' <data.txt` | 5.5 ± 0.5 | 4.8 | 7.3 | 2.11 ± 0.33 |
+| `dd conv=lcase <data.txt` | 18.8 ± 1.1 | 17.0 | 24.9 | 7.23 ± 1.03 |