lint-diff.py

#!/usr/bin/env python3
"""Filter warnings output, to only show output for changed lines."""

# Filter warnings output, to only show output for changed lines.
#
# This is useful when you want to enforce some style guideline, but
# making the changes globally in your project would be too burdensome.
# You can make the requirement only for new and changed lines, so your
# codebase will conform to the new standard gradually, as you edit it.

# Usage:  lint-diff.py [options] diff.txt [warnings.txt]
#         If warnings.txt is omitted, use standard input.
# Output: all lines in warnings.txt that are on a changed line.
#         Output status is 1 if it produced any output, 0 if not, 2 if error.
# Options: --guess-strip means guess values for --strip-diff and --strip-lint.
#          --strip-diff=N means to ignore N leading "/" in diff.txt.
#          --strip-warnings=N means to ignore N leading "/" in warnings.txt.
#              Affects matching, but not output, of lines.
#          --context=N is how many lines adjacent to the changed ones
#              are also considered changed; the default is 2.
#          --debug means to print diagnostic output.

# Here is how you could use this in continuous integration (Azure Pipelines,
# CircleCI, GitHub Actions, and Travis CI are currently supported) to require
# that pull requests satisfy the command `command-that-issues-warnings`:
#
#  if [ -d /tmp/$USER/plume-scripts ] ; then
#   git -C /tmp/$USER/plume-scripts pull -q > /dev/null 2>&1
#  else
#   mkdir -p /tmp/$USER \
#    && git -C /tmp/$USER clone --filter=blob:none -q https://github.com/plume-lib/plume-scripts.git
#  fi
# (command-that-issues-warnings > /tmp/warnings.txt 2>&1) || true
# /tmp/$USER/plume-scripts/ci-lint-diff /tmp/warnings.txt

# Implementation notes:
# 1. It may be possible to achieve a similar result using diff (but not `git diff`):
# https://unix.stackexchange.com/questions/34874/diff-output-line-numbers .
# 2. The documentation for diffFilter (https://github.com/exussum12/coverageChecker)
# suggests it has this same functionality, but my tests indicate it does not.

import argparse
import os
import re
import sys

PROGRAM = os.path.basename(__file__)

DEBUG = False

PLUSPLUSPLUS_RE = re.compile(r"\+\+\+ (\S*).*")

# This cannot be multiline because files are read one line at a time.
FILENAME_LINENO_RE = re.compile("([^:]*):([0-9]+):.*")

INITIAL_WHITESPACE_RE = re.compile("[ \t]")


def eprint(*args, **kwargs):
    """Print to stderr."""
    print(*args, file=sys.stderr, **kwargs)


def strip_dirs(filename, num_dirs):
    """Strip off `num_dirs` leading "/" characters."""
    if num_dirs == 0:
        return filename
    return os.path.join(*(filename.split(os.path.sep)[num_dirs:]))


## Tests:
"""
import os
assert strip_dirs("/a/b/c/d", 0) == '/a/b/c/d'
assert strip_dirs("/a/b/c/d", 1) == 'a/b/c/d'
assert strip_dirs("/a/b/c/d", 2) == 'b/c/d'
assert strip_dirs("/a/b/c/d", 3) == 'c/d'
assert strip_dirs("/a/b/c/d", 4) == 'd'
assert strip_dirs("/a/b/c/", 0) == '/a/b/c/'
assert strip_dirs("/a/b/c/", 1) == 'a/b/c/'
assert strip_dirs("/a/b/c/", 2) == 'b/c/'
assert strip_dirs("/a/b/c/", 3) == 'c/'
assert strip_dirs("/a/b/c/", 4) == ''
"""


def min_strips(filename1, filename2):
    """Returns a 4-tuple of 2 integers and 2 strings.  The integers
    indicate the smallest strip values that make the two filenames equal,
    or a maximal pair if the files have different basenames.  The last two
    elements of the tuple are the argument strings."""
    components1 = filename1.split(os.path.sep)
    components2 = filename2.split(os.path.sep)
    if components1[-1] != components2[-1]:
        ## TODO: is this special case necessary?
        return (1000, 1000, filename1, filename2)
    while components1 and components2 and components1[-1] == components2[-1]:
        del components1[-1]
        del components2[-1]
    return (len(components1), len(components2), filename1, filename2)


## Tests:
"""
import os
assert min_strips("/a/b/c/d", "/a/b/c/d") == (0, 0, "/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "/a/b/c/d") == (2, 1, "e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "/a/b/c/d") == (3, 1, "/e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "a/b/c/d") == (2, 0, "e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "a/b/c/d") == (3, 0, "/e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/a/b/c/d", "/e/f/g/h") == (1000, 1000, "/a/b/c/d", "/e/f/g/h")
"""


def pair_min(pair1, pair2):
    """Given two pairs, returns the one that is pointwise lesser in its first two elements.
    Fails if neither is lesser."""
    if pair1[0] <= pair2[0] and pair1[1] <= pair2[1]:
        return pair1
    if pair1[0] >= pair2[0] and pair1[1] >= pair2[1]:
        return pair2
    raise Exception(f"incomparable pairs: {pair1} {pair2}")


## Tests:
"""
import os
assert pair_min((3,4,"a","b"), (5,6,"c","d")) == (3,4,"a","b")
assert pair_min((4,3,"a","b"), (6,5,"c","d")) == (4,3,"a","b")
assert pair_min((30,40,"a","b"), (5,6,"c","d")) == (5,6,"c","d")
assert pair_min((40,30,"a","b"), (6,5,"c","d")) == (6,5,"c","d")
"""


def diff_filenames(diff_filename):
    """All the filenames in the given diff file."""
    result = set()
    with open(diff_filename, encoding="utf-8") as diff:
        for diff_line in diff:
            match = PLUSPLUSPLUS_RE.match(diff_line)
            if match:
                filename = match.group(1)
                if filename != "/dev/null":
                    result.add(filename)
    return result


def warning_filenames(warning_filename):
    """All the filenames in the given warning file."""
    result = set()
    with open(warning_filename, encoding="utf-8") as warnings:
        for warning_line in warnings:
            match = FILENAME_LINENO_RE.match(warning_line)
            if match:
                result.add(match.group(1))
    return result


def guess_strip_filenames(diff_filenames, warning_filenames):
    """Arguments are two lists of file names.
    Result is a pair of integers."""
    result = (1000, 1000, "no files seen yet", "no files seen yet")
    for diff_filename in diff_filenames:
        for warning_filename in warning_filenames:
            result = pair_min(result, min_strips(diff_filename, warning_filename))
    return result


def guess_strip_files(diff_file, warning_file):
    """Arguments are files produced by diff and a lint tool, respectively.
    Result is a pair of integers."""
    diff_files = diff_filenames(diff_file)
    warning_files = warning_filenames(warning_file)
    result = guess_strip_filenames(diff_files, warning_files)
    diff_prefix = os.path.commonprefix(list(diff_files))
    warnings_prefix = os.path.commonprefix(list(warning_files))
    if result[0] > diff_prefix.count("/") or result[1] > warnings_prefix.count("/"):
        # This is not necessarily a problem.  It is possible that all the
        # diffs, or all the lint output, happens to be in one subdirectory.
        if DEBUG:
            eprint(
                "lint-diff.py: guess_strip_files all in one subdirectory: "
                + "result={result} diff_prefix={diff_prefix} warnings_prefix={warnings_prefix}"
            )
            eprint(f"diff_files={diff_files}")
            eprint(f"warning_files={warning_files}")
    return result


### Main routine


def parse_args():
    """Parse and return the command-line arguments."""
    global DEBUG

    parser = argparse.ArgumentParser(
        description="Filter warnings output, to only show output for changed lines"
    )
    parser.add_argument(
        "--guess-strip",
        dest="guess_strip",
        action="store_true",
        default=False,
        help="guess values for --strip-diff and --strip-warnings",
    )
    parser.add_argument(
        "--strip-diff",
        metavar="NUM_SLASHES",
        dest="strip_diff",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in diff.txt',
    )
    parser.add_argument(
        "--strip-warnings",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in warnings.txt',
    )
    # Deprecated; exists for backwards compatibility
    parser.add_argument(
        "--strip-lint",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='(DEPRECATED) ignore N leading "/" in filenames in warnings.txt',
    )
    parser.add_argument(
        "--context",
        metavar="NUM_LINES",
        dest="context_lines",
        action="store",
        type=int,
        default=2,
        help="how many lines around each changed one are also considered changed",
    )
    parser.add_argument(
        "--debug", dest="DEBUG", action="store_true", help="print diagnostic output"
    )
    parser.add_argument("diff_filename", metavar="diff.txt", default=os.getcwd())
    parser.add_argument("warning_filename", metavar="warnings.txt", default=None)

    args = parser.parse_args()
    DEBUG = args.DEBUG

    if args.guess_strip and args.strip_diff != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-diff")
        sys.exit(2)

    if args.guess_strip and args.strip_warnings != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-warnings")
        sys.exit(2)

    if args.guess_strip and args.warning_filename is None:
        eprint(
            PROGRAM, 'needs "warnings.txt" file argument when --guess-strip is provided'
        )
        sys.exit(2)

    if args.guess_strip:
        guessed_strip = guess_strip_files(args.diff_filename, args.warning_filename)
        if guessed_strip[0] == 1000:
            if DEBUG:
                eprint(
                    "lint-diff.py: --guess-strip failed to guess values (maybe no files in common?)"
                )
        else:
            args.strip_diff = guessed_strip[0]
            args.strip_warnings = guessed_strip[1]
            if DEBUG:
                eprint(
                    "lint-diff.py inferred "
                    + f"--strip-diff={args.strip_diff} --strip-warnings={args.strip_warnings}"
                )

    # A filename if the diff filenames start with "a/" and "b/", otherwise None.
    # Is set by changed_lines().
    args.relative_diff = None

    return args


def changed_lines(args):
    """Returns a dictionary from file names to a set of ints (line numbers for changed lines)."""

    changed = {}

    with open(args.diff_filename, encoding="utf-8") as diff:
        atat_re = re.compile("@@ -([0-9]+)(,[0-9]+)? \\+([0-9]+)(,[0-9]+)? @@.*")
        # content_re = re.compile("[ +-].*")

        filename = ""
        lineno = -1000000
        for diff_line in diff:
            if diff_line.startswith("---"):
                continue
            match = PLUSPLUSPLUS_RE.match(diff_line)
            if match:
                if match.group(1).startswith("b/"):  # heuristic
                    args.relative_diff = diff_line
                try:
                    filename = strip_dirs(match.group(1), args.strip_diff)
                except TypeError:
                    filename = "diff filename above common directory"
                    ## It's not an error; it just means this file doesn't appear in warnings output.
                    # eprint('Bad --strip-diff={0} ; line has fewer "/": {1}'.format(
                    #   strip_diff, match.group(1)))
                    # sys.exit(2)
                if filename not in changed:
                    changed[filename] = set()
                continue
            match = atat_re.match(diff_line)
            if match:
                lineno = int(match.group(3))
                continue
            if diff_line.startswith("+"):
                # Not just the changed line: changed[filename].add(lineno)
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines + 1
                ):
                    changed[filename].add(changed_lineno)
                lineno += 1
            if diff_line.startswith(" "):
                lineno += 1
            if diff_line.startswith("-"):
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines
                ):
                    changed[filename].add(changed_lineno)
                continue

    return changed


def warn_relative_diff(args):
    """Possibly warn about relative directories."""

    result = False
    if args.relative_diff is not None and args.strip_diff == 0:
        # This is usually not an error, so don't warn.
        if DEBUG:
            eprint(
                "warning:",
                args.diff_filename,
                "may use relative paths (e.g.,",
                args.relative_diff.strip(),
                ") but --strip-diff=0",
                ("(guessed)" if args.guess_strip else ""),
            )
            eprint("warning: (Maybe there were no files in common.)")
        result = True
        if DEBUG:
            eprint(f"lint-diff.py: diff file {args.diff_filename}:")
            with open(args.diff_filename, "r", encoding="utf-8") as fin:
                eprint("{}", fin.read())
            eprint(f"lint-diff.py: lint file {args.warning_filename}:")
            with open(args.warning_filename, "r", encoding="utf-8") as fin:
                eprint("{}", fin.read())
            eprint("lint-diff.py: end of input files.")

    return result


def main():
    """The main routine"""

    global DEBUG

    args = parse_args()

    # A dictionary from file names to a set of ints (line numbers for changed lines)
    changed = changed_lines(args)

    if DEBUG:
        for filename in sorted(changed):
            print(filename, sorted(changed[filename]))

    # True if a warning has been issued about relative directories.
    relative_diff_warned = warn_relative_diff(args)

    if args.warning_filename is None:
        args.warning_filename = "stdin"
        warnings = sys.stdin
    else:
        # pylint: disable=consider-using-with
        warnings = open(args.warning_filename, encoding="utf-8")

    # 1 if this produced any output, 0 if not
    status = 0
    # true if we just printed a warning and are looking for continuation lines to print
    print_multiline_warning = False

    for warning_line in warnings:
        if print_multiline_warning and INITIAL_WHITESPACE_RE.match(warning_line):
            print(warning_line, end="")
            continue
        print_multiline_warning = False

        match = FILENAME_LINENO_RE.match(warning_line)
        if match:
            try:
                filename = strip_dirs(match.group(1), args.strip_warnings)
            except TypeError:
                filename = "warnings filename above common directory"
                ## It's not an error; it just means this file doesn't appear in warnings output.
                # eprint('Bad --strip-warnings={0} ; line has fewer "/": {1}'.format(
                #   strip_warnings, match.group(1)))
                # sys.exit(2)
            if (
                filename.startswith("/")
                and args.relative_diff is not None
                and args.strip_warnings == 0
            ):
                if not relative_diff_warned:
                    eprint(
                        # No spaces around string literals becaues this is `eprint`.
                        "warning:",
                        args.diff_filename,
                        "uses relative paths but",
                        args.warning_filename,
                        "uses absolute paths",
                    )
                    relative_diff_warned = True
            lineno = int(match.group(2))
            if filename in changed and lineno in changed[filename]:
                print(warning_line, end="")
                status = 1
                print_multiline_warning = True

    if warnings is not sys.stdin:
        warnings.close

    sys.exit(status)


if __name__ == "__main__":
    main()