Implement compatibility with Python 3.12

There are  few changes in bytecode, including a few documented changes
that are not actually present.

lib/python/pyflyby/_autoimp.py

@@ -36,10 +36,6 @@
 else:
     LOAD_SHIFT = 0
 
-if sys.version_info > (3, 11):
-    LOAD_SHIFT = 1
-else:
-    LOAD_SHIFT = 0
 
 if sys.version_info >= (3,10):
     from types import NoneType, EllipsisType
@@ -1164,10 +1160,13 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
             "_find_loads_without_stores_in_code(): expected a CodeType; got a %s"
             % (type(co).__name__,))
     # Initialize local constants for fast access.
-    from opcode import HAVE_ARGUMENT, EXTENDED_ARG, opmap
+    from opcode import EXTENDED_ARG, opmap
 
     LOAD_ATTR    = opmap['LOAD_ATTR']
+    # LOAD_METHOD is _supposed_ to be removed in 3.12 but still present in opmap
+    # if sys.version_info < (3, 12):
     LOAD_METHOD  = opmap['LOAD_METHOD']
+    # endif
     LOAD_GLOBAL  = opmap['LOAD_GLOBAL']
     LOAD_NAME    = opmap['LOAD_NAME']
     STORE_ATTR   = opmap['STORE_ATTR']
@@ -1267,12 +1266,11 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
     earliest_backjump_label = _find_earliest_backjump_label(bytecode)
     # Loop through bytecode.
     while i < n:
-        c = bytecode[i]
-        op = _op(c)
+        op = bytecode[i]
         i += 1
         if op == CACHE:
             continue
-        if op >= HAVE_ARGUMENT:
+        if take_arg(op):
             oparg = bytecode[i] | extended_arg
             extended_arg = 0
             if op == EXTENDED_ARG:
@@ -1289,9 +1287,40 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
                 stores.add(fullname)
                 continue
             if op in [LOAD_ATTR, LOAD_METHOD]:
-                # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* so far;
-                # possibly more LOAD_ATTR/STORE_ATTR will follow
-                pending.append(co.co_names[oparg])
+                if sys.version_info >= (3,12):
+                    # from the docs:
+                    #
+                    # If the low bit of namei is not set, this replaces
+                    # STACK[-1] with getattr(STACK[-1], co_names[namei>>1]).
+                    #
+                    # If the low bit of namei is set, this will attempt to load
+                    # a method named co_names[namei>>1] from the STACK[-1]
+                    # object. STACK[-1] is popped. This bytecode distinguishes
+                    # two cases: if STACK[-1] has a method with the correct
+                    # name, the bytecode pushes the unbound method and
+                    # STACK[-1]. STACK[-1] will be used as the first argument
+                    # (self) by CALL when calling the unbound method. Otherwise,
+                    # NULL and the object returned by the attribute lookup are
+                    # pushed.
+                    #
+                    # Changed in version 3.12: If the low bit of namei is set,
+                    # then a NULL or self is pushed to the stack before the
+                    # attribute or unbound method respectively.
+                    #
+                    # Implication for Pyflyby
+                    #
+                    # In our case I think it means we are always looking at
+                    # oparg>>1 as the name of the names we need to load,
+                    # Though we don't keep track of the stack, and so we may get
+                    # wrong results ?
+                    #
+                    # In any case this seem to match what load_method was doing
+                    # before.
+                    pending.append(co.co_names[oparg>>1])
+                else:
+                    # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* so far;
+                    # possibly more LOAD_ATTR/STORE_ATTR will follow
+                    pending.append(co.co_names[oparg])
                 continue
             # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* (and no more
             # LOAD_ATTR/STORE_ATTR)
@@ -1378,10 +1407,14 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
         if isinstance(arg, types.CodeType):
             _find_loads_without_stores_in_code(arg, loads_without_stores)
 
-
-def _op(c):
-    return c
-
+if sys.version_info >= (3,12):
+    from dis import hasarg
+    def take_arg(op):
+        return op in hasarg
+else:
+    def take_arg(op):
+        from opcode import HAVE_ARGUMENT
+        return op >= HAVE_ARGUMENT
 
 def _find_earliest_backjump_label(bytecode):
     """
@@ -1423,19 +1456,19 @@ def _find_earliest_backjump_label(bytecode):
       The earliest target of a backward jump, as an offset into the bytecode.
     """
     # Code based on dis.findlabels().
-    from opcode import HAVE_ARGUMENT, hasjrel, hasjabs
+    from opcode import hasjrel, hasjabs
     if not isinstance(bytecode, bytes):
         raise TypeError
     n = len(bytecode)
     earliest_backjump_label = n
     i = 0
     while i < n:
-        c = bytecode[i]
-        op = _op(c)
+        op = bytecode[i]
         i += 1
-        if op < HAVE_ARGUMENT:
+        if not take_arg(op):
             continue
-        oparg = _op(bytecode[i]) + _op(bytecode[i+1])*256
+        assert i+1 < len(bytecode)
+        oparg = bytecode[i] + bytecode[i+1]*256
         i += 2
         label = None
         if op in hasjrel:

lib/python/pyflyby/_importclns.py

@@ -18,7 +18,8 @@
 from   pyflyby._util            import (cached_attribute, cmp, partition,
                                         stable_unique)
 
-from typing import Dict, ClassVar, Sequence, Union, List
+from   typing                   import (ClassVar, Dict, FrozenSet, List,
+                                        Sequence, Union)
 
 
 class NoSuchImportError(ValueError):
@@ -49,7 +50,7 @@ class ImportSet(object):
     """
 
     _EMPTY : ClassVar[ImportSet]
-    _importset: frozenset[Import]
+    _importset : FrozenSet[Import]
 
     def __new__(cls, arg, ignore_nonimports=False, ignore_shadowed=False):
         """
@@ -480,17 +481,17 @@ def count_lines(import_column):
                 % (type(params.align_imports).__name__,))
         return ''.join(pp(statement, import_column) for statement in statements)
 
-    def __contains__(self, x):
+    def __contains__(self, x) -> bool:
         return x in self._importset
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         if self is other:
             return True
         if not isinstance(other, ImportSet):
             return NotImplemented
         return self._importset == other._importset
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         return not (self == other)
 
     # The rest are defined by total_ordering
@@ -509,7 +510,7 @@ def __cmp__(self, other):
     def __hash__(self):
         return hash(self._importset)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.imports)
 
     def __iter__(self):

lib/python/pyflyby/_imports2s.py

@@ -15,6 +15,8 @@
 
 from typing import Union
 
+from textwrap import indent
+
 
 class SourceToSourceTransformationBase(object):
 
@@ -60,6 +62,9 @@ def output(self, params=None) -> PythonBlock:
         result = PythonBlock(result, filename=self.input.filename)
         return result
 
+    def __repr__(self):
+        return f"<{self.__class__.__name__}\n{indent(str(self.pretty_print()),'    ')}\n at 0x{hex(id(self))}>"
+
 
 class SourceToSourceTransformation(SourceToSourceTransformationBase):
 

lib/python/pyflyby/_importstmt.py

@@ -16,7 +16,7 @@
                                         longest_common_prefix)
 
 
-from   typing                   import Dict, Tuple, Optional
+from   typing                   import Dict, Tuple, Optional, Union
 
 
 
@@ -51,7 +51,7 @@ def read_black_config() -> Dict:
 
 
 class ImportFormatParams(FormatParams):
-    align_imports:bool = True
+    align_imports:Union[bool, set, list, tuple, str] = True
     """
     Whether and how to align 'from modulename import aliases...'.  If ``True``,
     then the 'import' keywords will be aligned within a block.  If an integer,