django_i18n_lint.py

#! /usr/bin/env python
"""
Prints out all
"""

import os
import re
from optparse import OptionParser


def location(str, pos):
    """Given a string str and an integer pos, find the line number and character in that line that correspond to pos"""
    lineno, charpos = 1, 1
    counter = 0
    for char in str:
        if counter == pos:
            return lineno, charpos
        elif char == "\n":
            lineno += 1
            charpos = 1
            counter += 1
        else:
            charpos += 1
            counter += 1

    return lineno, charpos


# Things that are OK:
GOOD_STRINGS = re.compile(
    r"""
        (
        # django comment
        {%\ comment\ %}.*?{%\ endcomment\ %}

        # already translated text (until Django 3.0)
        |{%\ ?blocktrans.*?{%\ ?endblocktrans\ ?%}

        # already translated text (Django 3.1+)
        |{%\ ?blocktranslate.*?{%\ ?endblocktranslate\ ?%}

         # any django template function (catches {% trans ..) aswell
        |{%.*?%}

         # CSS
        |<style.*?</style>

         # JS
        |<script.*?</script>

         # A html title or value attribute that's been translated
        |(?:value|title|summary|alt)="{%\ ?trans.*?%}"

         # A html title or value attribute that's just a template var
        |(?:value|title|summary|alt)="{{.*?}}"

         # An <option> value tag
        |<option[^<>]+?value="[^"]*?"

         # Any html attribute that's not value or title (single quote, double quote and html5 quoteless)
         # NB at the start we want to grab any trailing quote from the previous attribute
         # FIXME This will fail for some quoteless attr values.
        |(?:['"]\W+)?[a-z:-]+?(?<!alt)(?<!value)(?<!title)(?<!summary)=(?:'(?:{{.*?}}|{%.*?%}|[^']*)'|"(?:{{.*?}}|{%.*?%}|[^"]*)+"|[a-zA-Z\.]+)

         # The actual alt/value/title tag itself cannot be translated, but the value should be
         # Treat data-title/data-original-title etc as equivalanets. Think this is some bootstrap thing & HTML5
        |(?:['"]\W+)?(?:data-|data-original-)?(?:alt|value|title|summary)=['"]?

         # Boolean attributes
        |<[^<>]+?(?:checked|selected|disabled|readonly|multiple|ismap|defer|async|declare|noresize|nowrap|noshade|compact|hidden|itemscope|autofocus|autoplay|controls|download)[^<>]*?>

         # HTML opening tag
        |<[\w:]+

         # End of a html opening tag
         # NB: catching end of quote from last attribute
        |(?:['"]\W*)?/?>

         # closing html tag
        |</.*?>

         # any django template variable
        |{{.*?}}

         # any django template tag
        |{%.*?%}

         # any angular.js template
        |\[\[.*?\]\]

         # HTML doctype
        |<!DOCTYPE.*?>

         # IE specific HTML
        |<!--\[if.*?<!\[endif\]-->

         # HTML comment
        |<!--.*?-->

         # HTML entities
        |&[a-z]{1,10};

        # HTML entities
        |&\#x[0-9]{1,10};

         # CSS style
        |<style.*?</style>

         # another common template comment
        |{\#.*?\#}
        )""",
    # MULTILINE to match across lines and DOTALL to make . include the newline
    re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE,
)

# Stops us matching non-letter parts, e.g. just hypens, full stops etc.
LETTERS = re.compile(r"[^\W\d_]")

LEADING_TRAILING_WHITESPACE = re.compile(r"(^\W+|\W+$)")


def split_into_good_and_bad(template):
    for index, match in enumerate(GOOD_STRINGS.split(template)):
        yield (index, match)


def split_trailing_space(string):
    """Given a string, returns a tuple of 3 string, the leading whitespace, middle, and trailing whitespace"""
    results = LEADING_TRAILING_WHITESPACE.split(string)
    if len(results) == 1:
        # no spaces
        return ("", string, "")
    elif len(results) == 3 and results[0] == "" and results[2] != "":
        # only leading whitespace
        return (results[1], results[2], "")
    elif len(results) == 3 and results[0] != "" and results[2] == "":
        # only trailing
        return ("", results[0], results[1])
    elif len(results) == 5:
        # leading and trailing whitespace
        return (results[1], results[2], results[3])
    else:
        raise NotImplementedError("Unknown case: %r %r" % (string, results))


def replace_strings(filename, overwrite=False, force=False, accept=[]):
    full_text_lines = []
    with open(filename) as fp:
        content = fp.read()

    offset = 0
    ignore_lines = find_ignored_lines(content)

    for index, string in split_into_good_and_bad(content):
        if index % 2 == 1:
            full_text_lines.append(string)
        elif index % 2 == 0:
            # Ignore it if it doesn't have letters
            m = LETTERS.search(string)
            if not m:
                full_text_lines.append(string)
            else:
                # split out the leading whitespace and trailing
                leading_whitespace, message, trailing_whitespace = split_trailing_space(
                    string
                )
                full_text_lines.append(leading_whitespace)

                # Find location of first letter
                lineno, charpos = location(string, offset + m.span()[0])

                if any(r.match(message) for r in accept):
                    full_text_lines.append(message)
                elif lineno in ignore_lines:
                    full_text_lines.append(message)
                elif force:
                    full_text_lines.append(
                        '{% translate "' + message.replace('"', '\\"') + '" %}'
                    )

                else:
                    change = input("Make %r translatable? [Y/n] " % message)
                    if change == "y" or change == "":
                        full_text_lines.append(
                            '{% translate "' + message.replace('"', '\\"') + '" %}'
                        )
                    else:
                        full_text_lines.append(message)

                full_text_lines.append(trailing_whitespace)
        offset += len(string)

    full_text = "".join(full_text_lines)
    if overwrite:
        save_filename = filename
    else:
        save_filename = filename.split(".")[0] + "_translated.html"
    open(save_filename, "w").write(full_text)
    print("Fully translated! Saved as: %s" % save_filename)


def find_ignored_lines(template):
    lines = set()
    for m in re.finditer(r"{#\s*notrans\s*#}", template):
        offset = m.span()[0]
        lineno, charpos = location(template, offset)
        lines.add(lineno)
    return lines


def non_translated_text(template):
    offset = 0

    ignore_lines = find_ignored_lines(template)

    # Find the parts of the template that don't match this regex
    # taken from http://www.technomancy.org/python/strings-that-dont-match-regex/
    for index, match in split_into_good_and_bad(template):
        if index % 2 == 0:
            # Ignore it if it doesn't have letters
            m = LETTERS.search(match)
            if m:
                # Get location of first letter
                lineno, charpos = location(template, offset + m.span()[0])
                if lineno in ignore_lines:
                    offset += len(match)
                    continue
                yield (
                    lineno,
                    charpos,
                    match.strip().replace("\n", "").replace("\r", "")[:120],
                )

        offset += len(match)


def print_strings(filename, accept=[]):
    with open(filename) as fp:
        file_contents = fp.read()

    for lineno, charpos, message in non_translated_text(file_contents):
        if any(r.match(message) for r in accept):
            continue

        print("%s:%s:%s:%s" % (filename, lineno, charpos, message))


def filenames_to_work_on(directory, exclude_filenames):
    """Return list of files in directory that we should look at"""
    files = []
    for dirpath, dirs, filenames in os.walk(directory):
        files.extend(
            os.path.join(dirpath, fname)
            for fname in filenames
            if (fname.endswith(".html") or fname.endswith(".txt"))
            and fname not in exclude_filenames
        )
    return files


def main():
    parser = OptionParser(usage="usage: %prog [options] <filenames>")
    parser.add_option(
        "-r",
        "--replace",
        action="store_true",
        dest="replace",
        help="Ask to replace the strings in the file.",
        default=False,
    )
    parser.add_option(
        "-o",
        "--overwrite",
        action="store_true",
        dest="overwrite",
        help=(
            "When replacing the strings, overwrite the original file. If not specified, "
            "the file will be saved in a separate file named X_translated.html"
        ),
        default=False,
    )
    parser.add_option(
        "-f",
        "--force",
        action="store_true",
        dest="force",
        help="Force to replace string with no questions",
        default=False,
    )
    parser.add_option(
        "-e",
        "--exclude",
        action="append",
        dest="exclude_filename",
        help="Exclude these filenames from being linted",
        default=[],
    )
    parser.add_option(
        "-x",
        "--accept",
        action="append",
        dest="accept",
        help="Exclude these regexes from results",
        default=[],
    )
    (options, args) = parser.parse_args()

    # Create a list of files to check
    if len(args) == 0:
        args = [os.getcwd()]
    files = []
    for arg in args:
        if os.path.isdir(arg):
            files = filenames_to_work_on(arg, options.exclude_filename)
        elif arg not in options.exclude_filename:
            files.append(arg)

    accept_regexes = [re.compile(r) for r in options.accept]

    for filename in files:
        if options.replace:
            replace_strings(
                filename, overwrite=True, force=options.force, accept=accept_regexes
            )
        else:
            print_strings(filename, accept=accept_regexes)


if __name__ == "__main__":
    main()