From 7519160aa891433d746f18c40979c6e5b2c6e4da Mon Sep 17 00:00:00 2001
From: Audrey Dutcher <audrey@rhelmot.io>
Date: Thu, 2 Jan 2025 11:52:39 -0700
Subject: [PATCH] Add typechecking CI (#52)

* Add typechecking CI

* make it work with new files
---
 .github/workflows/angr-ci.yml    | 13 +++++
 ci-image/scripts/ga-typecheck.sh | 20 ++++++++
 ci-image/scripts/typecheck.py    | 81 ++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+)
 create mode 100755 ci-image/scripts/ga-typecheck.sh
 create mode 100755 ci-image/scripts/typecheck.py

diff --git a/.github/workflows/angr-ci.yml b/.github/workflows/angr-ci.yml
index 1582aeb..265b618 100644
--- a/.github/workflows/angr-ci.yml
+++ b/.github/workflows/angr-ci.yml
@@ -49,6 +49,19 @@ jobs:
     - run: /root/scripts/ga-lint.sh
       name: Run linter
 
+  typecheck:
+    name: Typecheck
+    runs-on: ubuntu-22.04
+    container:
+      image: ${{ inputs.container_image }}
+    needs: build
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        name: build_archive
+    - run: /root/scripts/ga-typecheck.sh
+      name: Run type checker
+
   test:
     name: Test
     runs-on: ubuntu-22.04
diff --git a/ci-image/scripts/ga-typecheck.sh b/ci-image/scripts/ga-typecheck.sh
new file mode 100755
index 0000000..e4b792c
--- /dev/null
+++ b/ci-image/scripts/ga-typecheck.sh
@@ -0,0 +1,20 @@
+#!/bin/bash -ex
+
+BASEDIR=$(dirname $(dirname $0))
+SCRIPTS=$BASEDIR/scripts
+
+tar -I zstd -xf build.tar.zst
+cd build
+
+source virtualenv/bin/activate
+pip install pyright
+
+cd src/${GITHUB_REPOSITORY##*/}
+HEAD_REV="$(git rev-parse --abbrev-ref HEAD)"
+if [[ $GITHUB_REF == "master" ]]; then
+    BASE_REV="$(git rev-parse --abbrev-ref HEAD~)"
+else
+    BASE_REV="$(git rev-parse --abbrev-ref master)"
+fi
+
+python $SCRIPTS/typecheck.py $BASE_REV $HEAD_REV
diff --git a/ci-image/scripts/typecheck.py b/ci-image/scripts/typecheck.py
new file mode 100755
index 0000000..f7180cb
--- /dev/null
+++ b/ci-image/scripts/typecheck.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import sys
+import subprocess
+import json
+import dataclasses
+import os
+
+@dataclasses.dataclass
+class FileReport:
+    filename: str
+    diagnostics: list[tuple[int, int, str, str]]
+    errors: int = 0
+    warnings: int = 0
+    lines: int = 0
+    score: float = 0.
+
+def count_lines(filename):
+    with open(filename, 'rb') as fp:
+        return sum(1 for _ in fp)
+
+def typecheck_files(filenames):
+    filenames = [filename for filename in filenames if os.path.exists(filename)]
+    if not filenames:
+        return {}
+    proc = subprocess.run(["pyright", "--outputjson", *filenames], text=False, check=False, stdout=subprocess.PIPE)
+    pyright_report = json.loads(proc.stdout)
+    my_report = {os.path.realpath(filename): FileReport(filename, diagnostics=[], lines=count_lines(filename)) for filename in filenames}
+    for diagnostic in pyright_report["generalDiagnostics"]:
+        severity = diagnostic["severity"]
+        filename = diagnostic["file"]
+        if severity == "error":
+            my_report[filename].errors += 1
+        elif severity == "warning":
+            my_report[filename].warnings += 1
+        start = diagnostic["range"]["start"]
+        my_report[filename].diagnostics.append((start["line"], start["character"], severity, diagnostic["message"]))
+
+    for report in my_report.values():
+        report.score = (report.errors * 10 + report.warnings) / report.lines
+
+    return my_report
+
+def typecheck_diff(rev1, rev2):
+    print(f"Comparing {rev1} --> {rev2}")
+    print()
+    filenames = subprocess.check_output(["git", "diff", "--name-only", f"{rev1}...{rev2}"], text=True).splitlines()
+    subprocess.check_call(["git", "checkout", rev1], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    report1 = typecheck_files(filenames)
+    subprocess.check_call(["git", "checkout", rev2], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    report2 = typecheck_files(filenames)
+
+    status = 0
+
+    for filename in filenames:
+        filename = os.path.realpath(filename)
+        if filename not in report2:
+            continue
+        if filename in report1:
+            base_score = report1[filename].score
+        else:
+            base_score = 0
+        e2 = report2[filename]
+        if e2.score > base_score:
+            status += 1
+            print(f"### {filename} badness increased from {base_score} to {e2.score}. Please fix:")
+            for line, char, severity, text in sorted(e2.diagnostics):
+                print(f"{filename}:{line}:{char}: [{severity}] {text}")
+        elif e2.score < base_score:
+            print(f"### {filename} badness decreased from {base_score} to {e2.score}. Nice!")
+        else:
+            print(f"### {filename} badness remained at {base_score}. Nice!")
+
+    if status > 0:
+        print(f"\n{status} files regressed. Fix them!")
+        return 1
+    else:
+        print("\nYou did it!")
+        return 0
+
+if __name__ == '__main__':
+    sys.exit(typecheck_diff(sys.argv[1], sys.argv[2]))