Armor AutoCompleter against unicode errors #305

aider/io.py

@@ -44,7 +44,7 @@ def __init__(self, root, rel_fnames, addable_rel_fnames, commands, encoding):
             try:
                 with open(fname, "r", encoding=self.encoding) as f:
                     content = f.read()
-            except FileNotFoundError:
+            except (FileNotFoundError, UnicodeDecodeError):
                 continue
             try:
                 lexer = guess_lexer_for_filename(fname, content)

tests/test_commands.py

@@ -468,3 +468,20 @@ def test_cmd_add_existing_with_dirty_repo(self):
             del coder
             del commands
             del repo
+
+    def test_cmd_add_unicode_error(self):
+        # Initialize the Commands and InputOutput objects
+        io = InputOutput(pretty=False, yes=True)
+        from aider.coders import Coder
+
+        coder = Coder.create(models.GPT35, None, io)
+        commands = Commands(io, coder)
+
+        fname = "file.txt"
+        encoding = "utf-16"
+        some_content_which_will_error_if_read_with_encoding_utf8 = "ÅÍÎÏ".encode(encoding)
+        with open(fname, "wb") as f:
+            f.write(some_content_which_will_error_if_read_with_encoding_utf8)
+
+        commands.cmd_add("file.txt")
+        self.assertEqual(coder.abs_fnames, set())

tests/test_io.py

@@ -1,8 +1,10 @@
 import os
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
 from aider.io import AutoCompleter, InputOutput
+from tests.utils import ChdirTemporaryDirectory
 
 
 class TestInputOutput(unittest.TestCase):
@@ -19,6 +21,28 @@ def test_autocompleter_with_non_existent_file(self):
         autocompleter = AutoCompleter(root, rel_fnames, addable_rel_fnames, commands, "utf-8")
         self.assertEqual(autocompleter.words, set(rel_fnames))
 
+    def test_autocompleter_with_unicode_file(self):
+        with ChdirTemporaryDirectory():
+            root = ""
+            fname = "file.py"
+            rel_fnames = [fname]
+            addable_rel_fnames = []
+            commands = None
+            autocompleter = AutoCompleter(root, rel_fnames, addable_rel_fnames, commands, "utf-8")
+            self.assertEqual(autocompleter.words, set(rel_fnames))
+
+            Path(fname).write_text("def hello(): pass\n")
+            autocompleter = AutoCompleter(root, rel_fnames, addable_rel_fnames, commands, "utf-8")
+            self.assertEqual(autocompleter.words, set(rel_fnames + ["hello"]))
+
+            encoding = "utf-16"
+            some_content_which_will_error_if_read_with_encoding_utf8 = "ÅÍÎÏ".encode(encoding)
+            with open(fname, "wb") as f:
+                f.write(some_content_which_will_error_if_read_with_encoding_utf8)
+
+            autocompleter = AutoCompleter(root, rel_fnames, addable_rel_fnames, commands, "utf-8")
+            self.assertEqual(autocompleter.words, set(rel_fnames))
+
 
 if __name__ == "__main__":
     unittest.main()