Add AST comparison test

Author: aibaars

.github/workflows/ci.yml

@@ -20,6 +20,12 @@ jobs:
           node-version: 16
       - run: npm install
       - run: npm test
+      - if: runner.os == 'Linux'
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+      - if: runner.os == 'Linux'
+        run: npm run-script test-compare
   test_windows:
     runs-on: windows-2019
     steps:

package.json

@@ -22,6 +22,7 @@
     "prebuild": "prebuild -r electron -t 3.0.0 -t 4.0.0 -t 4.0.4 -t 5.0.0 --strip && prebuild -t 8.16.0 -t 10.12.0 --strip",
     "prebuild:upload": "prebuild --upload-all",
     "test": "tree-sitter test && script/parse-examples",
+    "test-compare": "script/parse-examples --compare",
     "test-windows": "tree-sitter test"
   },
   "repository": "https://github.com/tree-sitter/tree-sitter-ruby",

script/compare-ast.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+set -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+file="$1"
+"${SCRIPT_DIR}/print-ast.rb" "$file" > "$file.1"
+if [ "$?" != "0" ]; then
+   echo "print-ast failure: $file"
+   rm "$file.1"
+   exit 1
+fi
+"${SCRIPT_DIR}/../node_modules/tree-sitter-cli/cli.js"  parse "$file" | \
+   sed 's/ \[[0-9]\+, [0-9]\+\] - \[[0-9]\+, [0-9]\+\]//' | \
+   tr $'\n' $'\t' | sed 's/\(^\|[[:space:]]*\)(comment)//g' | tr $'\t' $'\n' > "$file.2"
+
+if [ "$?" != "0" ]; then
+   echo "parse failure: $file"
+   rm "$file.1" "$file.2"
+   exit 1
+fi
+diff "$file.1" "$file.2" > "$file.diff"
+if [ "$?" != "0" ]; then
+   echo "diff: $file"
+   exit 1
+else
+   rm "$file".{diff,1,2}
+fi
+

script/known_differences.expected

@@ -0,0 +1,26 @@
+### differences caused by bugs in tree-sitter-ruby ###
+
+# single quoted heredocs do not contain interpolation
+diff: examples/ruby_spec/core/exception/interrupt_spec.rb
+diff: examples/ruby_spec/language/fixtures/squiggly_heredoc.rb
+diff: examples/ruby_spec/language/heredoc_spec.rb
+diff: examples/ruby_spec/library/bigdecimal/BigDecimal_spec.rb
+
+# string array with escape sequences like `%w[\\]`
+diff: examples/ruby_spec/language/array_spec.rb
+diff: examples/ruby_spec/language/fixtures/classes.rb
+
+# bad escape handling for `/\c#{str}/`
+diff: examples/ruby_spec/language/regexp/interpolation_spec.rb
+
+### benign differences ###
+
+# ripper does not distinguish between empty `then` and missing `then`
+diff: examples/ruby_spec/language/case_spec.rb
+diff: examples/ruby_spec/language/if_spec.rb
+diff: examples/ruby_spec/language/pattern_matching_spec.rb
+
+# ripper does not emit events for parentheses in define? and not statements
+diff: examples/ruby_spec/language/defined_spec.rb
+diff: examples/ruby_spec/language/precedence_spec.rb
+

script/parse-examples

@@ -39,3 +39,22 @@ success_percent=$(bc -l <<< "100*${success_count}/${example_count}")
 printf \
   "Successfully parsed %d of %d example files (%.1f%%)\n" \
   $success_count $example_count $success_percent
+
+if [ "$1" == "--compare" ]; then
+
+ find examples -name '*.rb' | grep -v -f script/known_failures.txt \
+   | xargs -P8 -n1 script/compare-ast.sh | sort > script/known_differences.actual
+
+  difference_count=$(wc -l < "script/known_differences.actual")
+
+  (cat script/known_differences.expected | sed -e 's/\s*#.*//' -e '/^$/d' | sort \
+    | diff --label script/known_differences.expected - script/known_differences.actual \
+  ) || (echo "Files script/known_differences.expected and script/known_differences.actual differ"; exit 1)
+  
+  rm -f script/known_differences.actual
+
+  printf \
+    "%d of %d example files were parsed differently compared to Ruby::Ripper\n" \
+    $difference_count $success_count
+
+fi