pythongh-107538: [Enum] fix handling of inverted/negative values (pythonGH-132273)

* Fix flag mask inversion when unnamed flags exist.

For example:

    class Flag(enum.Flag):
        A = 0x01
        B = 0x02
        MASK = 0xff

    ~Flag.MASK is Flag(0)

* EJECT and KEEP flags (IntEnum is KEEP) use direct value.

* correct Flag inversion to only flip flag bits

IntFlag will flip all bits -- this only makes a difference in flag sets with
missing values.

* correct negative assigned values in flags

negative values are no longer used as-is, but become inverted; i.e.

    class Y(self.enum_type):
        A = auto()
        B = auto()
        C = ~A        # aka ~1 aka 0b1 110 (from enum.bin()) aka 6
        D = auto()

    assert Y.C. is Y.B|Y.D

Author: ethanfurman

Lib/enum.py

@@ -535,7 +535,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
         # now set the __repr__ for the value
         classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases)
         #
-        # Flag structures (will be removed if final class is not a Flag
+        # Flag structures (will be removed if final class is not a Flag)
         classdict['_boundary_'] = (
                 boundary
                 or getattr(first_enum, '_boundary_', None)
@@ -544,6 +544,29 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
         classdict['_singles_mask_'] = 0
         classdict['_all_bits_'] = 0
         classdict['_inverted_'] = None
+        # check for negative flag values and invert if found (using _proto_members)
+        if Flag is not None and bases and issubclass(bases[-1], Flag):
+            bits = 0
+            inverted = []
+            for n in member_names:
+                p = classdict[n]
+                if isinstance(p.value, int):
+                    if p.value < 0:
+                        inverted.append(p)
+                    else:
+                        bits |= p.value
+                elif p.value is None:
+                    pass
+                elif isinstance(p.value, tuple) and p.value and isinstance(p.value[0], int):
+                    if p.value[0] < 0:
+                        inverted.append(p)
+                    else:
+                        bits |= p.value[0]
+            for p in inverted:
+                if isinstance(p.value, int):
+                    p.value = bits & p.value
+                else:
+                    p.value = (bits & p.value[0], ) + p.value[1:]
         try:
             classdict['_%s__in_progress' % cls] = True
             enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
@@ -1487,7 +1510,10 @@ def _missing_(cls, value):
                         )
         if value < 0:
             neg_value = value
-            value = all_bits + 1 + value
+            if cls._boundary_ in (EJECT, KEEP):
+                value = all_bits + 1 + value
+            else:
+                value = singles_mask & value
         # get members and unknown
         unknown = value & ~flag_mask
         aliases = value & ~singles_mask

Lib/test/test_enum.py

@@ -1002,12 +1002,18 @@ class OpenAB(self.enum_type):
             self.assertIs(~(A|B), OpenAB(252))
             self.assertIs(~AB_MASK, OpenAB(0))
             self.assertIs(~OpenAB(0), AB_MASK)
+            self.assertIs(OpenAB(~4), OpenAB(251))
         else:
             self.assertIs(~A, B)
             self.assertIs(~B, A)
+            self.assertIs(OpenAB(~1), B)
+            self.assertIs(OpenAB(~2), A)
             self.assertIs(~(A|B), OpenAB(0))
             self.assertIs(~AB_MASK, OpenAB(0))
             self.assertIs(~OpenAB(0), (A|B))
+            self.assertIs(OpenAB(~3), OpenAB(0))
+            self.assertIs(OpenAB(~4), OpenAB(3))
+            self.assertIs(OpenAB(~33), B)
         #
         class OpenXYZ(self.enum_type):
             X = 4
@@ -1031,13 +1037,38 @@ class OpenXYZ(self.enum_type):
             self.assertIs(~X, Y|Z)
             self.assertIs(~Y, X|Z)
             self.assertIs(~Z, X|Y)
+            self.assertIs(OpenXYZ(~4), Y|Z)
+            self.assertIs(OpenXYZ(~2), X|Z)
+            self.assertIs(OpenXYZ(~1), X|Y)
             self.assertIs(~(X|Y), Z)
             self.assertIs(~(X|Z), Y)
             self.assertIs(~(Y|Z), X)
             self.assertIs(~(X|Y|Z), OpenXYZ(0))
             self.assertIs(~XYZ_MASK, OpenXYZ(0))
             self.assertTrue(~OpenXYZ(0), (X|Y|Z))
 
+    def test_assigned_negative_value(self):
+        class X(self.enum_type):
+            A = auto()
+            B = auto()
+            C = A | B
+            D = ~A
+        self.assertEqual(list(X), [X.A, X.B])
+        self.assertIs(~X.A, X.B)
+        self.assertIs(X.D, X.B)
+        self.assertEqual(X.D.value, 2)
+        #
+        class Y(self.enum_type):
+            A = auto()
+            B = auto()
+            C = A | B
+            D = ~A
+            E = auto()
+        self.assertEqual(list(Y), [Y.A, Y.B, Y.E])
+        self.assertIs(~Y.A, Y.B|Y.E)
+        self.assertIs(Y.D, Y.B|Y.E)
+        self.assertEqual(Y.D.value, 6)
+
 
 class TestPlainEnumClass(_EnumTests, _PlainOutputTests, unittest.TestCase):
     enum_type = Enum
@@ -3680,6 +3711,8 @@ class SkipFlag(enum.Flag):
             C = 4 | B
         #
         self.assertTrue(SkipFlag.C in (SkipFlag.A|SkipFlag.C))
+        self.assertTrue(SkipFlag.B in SkipFlag.C)
+        self.assertIs(SkipFlag(~1), SkipFlag.B)
         self.assertRaisesRegex(ValueError, 'SkipFlag.. invalid value 42', SkipFlag, 42)
         #
         class SkipIntFlag(enum.IntFlag):
@@ -3688,6 +3721,8 @@ class SkipIntFlag(enum.IntFlag):
             C = 4 | B
         #
         self.assertTrue(SkipIntFlag.C in (SkipIntFlag.A|SkipIntFlag.C))
+        self.assertTrue(SkipIntFlag.B in SkipIntFlag.C)
+        self.assertIs(SkipIntFlag(~1), SkipIntFlag.B|SkipIntFlag.C)
         self.assertEqual(SkipIntFlag(42).value, 42)
         #
         class MethodHint(Flag):
@@ -4727,6 +4762,8 @@ class Color(Flag):
             BLUE = 4
             WHITE = -1
         # no error means success
+        self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE])
+        self.assertEqual(Color.WHITE.value, 7)
 
 
 class TestInternals(unittest.TestCase):

Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst

@@ -0,0 +1 @@
+Fix flag mask inversion when unnamed flags exist.

Misc/NEWS.d/next/Library/2025-04-08-07-25-10.gh-issue-107583.JGfbhq.rst

@@ -0,0 +1,4 @@
+Fix :class:`!Flag` inversion when flag set has missing values
+(:class:`!IntFlag` still flips all bits); fix negative assigned values
+during flag creation (both :class:`!Flag` and :class:`!IntFlag` ignore
+missing values).