Fix rendering of Literal annotations with enum values (#11517)

Co-authored-by: Adam Turner <[email protected]>

CHANGES

@@ -77,6 +77,9 @@ Bugs fixed
   Patch by Bénédikt Tran.
 * #10930: Highlight all search terms on the search results page.
   Patch by Dmitry Shachnev.
+* #11473: Type annotations containing :py:data:`~typing.Literal` enumeration
+  values now render correctly.
+  Patch by Bénédikt Tran.
 
 Testing
 -------

sphinx/util/typing.py

@@ -7,11 +7,14 @@
 from collections.abc import Sequence
 from struct import Struct
 from types import TracebackType
-from typing import Any, Callable, ForwardRef, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, ForwardRef, TypeVar, Union
 
 from docutils import nodes
 from docutils.parsers.rst.states import Inliner
 
+if TYPE_CHECKING:
+    import enum
+
 try:
     from types import UnionType  # type: ignore[attr-defined] # python 3.10 or above
 except ImportError:
@@ -186,7 +189,14 @@ def restify(cls: type | None, mode: str = 'fully-qualified-except-typing') -> st
                 args = ', '.join(restify(a, mode) for a in cls.__args__[:-1])
                 text += fr"\ [[{args}], {restify(cls.__args__[-1], mode)}]"
             elif cls.__module__ == 'typing' and getattr(origin, '_name', None) == 'Literal':
-                text += r"\ [%s]" % ', '.join(repr(a) for a in cls.__args__)
+                literal_args = []
+                for a in cls.__args__:
+                    if inspect.isenumattribute(a):
+                        literal_args.append(_format_literal_enum_arg(a, mode=mode))
+                    else:
+                        literal_args.append(repr(a))
+                text += r"\ [%s]" % ', '.join(literal_args)
+                del literal_args
             elif cls.__args__:
                 text += r"\ [%s]" % ", ".join(restify(a, mode) for a in cls.__args__)
 
@@ -338,7 +348,21 @@ def stringify_annotation(
             returns = stringify_annotation(annotation_args[-1], mode)
             return f'{module_prefix}Callable[[{args}], {returns}]'
         elif qualname == 'Literal':
-            args = ', '.join(repr(a) for a in annotation_args)
+            from sphinx.util.inspect import isenumattribute  # lazy loading
+
+            def format_literal_arg(arg):
+                if isenumattribute(arg):
+                    enumcls = arg.__class__
+
+                    if mode == 'smart':
+                        # MyEnum.member
+                        return f'{enumcls.__qualname__}.{arg.name}'
+
+                    # module.MyEnum.member
+                    return f'{enumcls.__module__}.{enumcls.__qualname__}.{arg.name}'
+                return repr(arg)
+
+            args = ', '.join(map(format_literal_arg, annotation_args))
             return f'{module_prefix}Literal[{args}]'
         elif str(annotation).startswith('typing.Annotated'):  # for py39+
             return stringify_annotation(annotation_args[0], mode)
@@ -352,6 +376,14 @@ def stringify_annotation(
     return module_prefix + qualname
 
 
+def _format_literal_enum_arg(arg: enum.Enum, /, *, mode: str) -> str:
+    enum_cls = arg.__class__
+    if mode == 'smart' or enum_cls.__module__ == 'typing':
+        return f':py:attr:`~{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+    else:
+        return f':py:attr:`{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+
+
 # deprecated name -> (object to return, canonical path or empty string)
 _DEPRECATED_OBJECTS = {
     'stringify': (stringify_annotation, 'sphinx.util.typing.stringify_annotation'),

tests/roots/test-ext-autodoc/target/literal.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Literal, TypeVar
+
+
+class MyEnum(Enum):
+    a = 1
+
+
+T = TypeVar('T', bound=Literal[1234])
+"""docstring"""
+
+
+U = TypeVar('U', bound=Literal[MyEnum.a])
+"""docstring"""
+
+
+def bar(x: Literal[1234]):
+    """docstring"""
+
+
+def foo(x: Literal[MyEnum.a]):
+    """docstring"""

tests/test_ext_autodoc.py

@@ -2467,3 +2467,59 @@ def test_canonical(app):
         '      docstring',
         '',
     ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_literal_render(app):
+    def bounded_typevar_rst(name, bound):
+        return [
+            '',
+            f'.. py:class:: {name}',
+            '   :module: target.literal',
+            '',
+            '   docstring',
+            '',
+            f'   alias of TypeVar({name!r}, bound={bound})',
+            '',
+        ]
+
+    def function_rst(name, sig):
+        return [
+            '',
+            f'.. py:function:: {name}({sig})',
+            '   :module: target.literal',
+            '',
+            '   docstring',
+            '',
+        ]
+
+    # autodoc_typehints_format can take 'short' or 'fully-qualified' values
+    # and this will be interpreted as 'smart' or 'fully-qualified-except-typing' by restify()
+    # and 'smart' or 'fully-qualified' by stringify_annotation().
+
+    options = {'members': None, 'exclude-members': 'MyEnum'}
+    app.config.autodoc_typehints_format = 'short'
+    actual = do_autodoc(app, 'module', 'target.literal', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.literal',
+        '',
+        *bounded_typevar_rst('T', r'\ :py:obj:`~typing.Literal`\ [1234]'),
+        *bounded_typevar_rst('U', r'\ :py:obj:`~typing.Literal`\ [:py:attr:`~target.literal.MyEnum.a`]'),
+        *function_rst('bar', 'x: ~typing.Literal[1234]'),
+        *function_rst('foo', 'x: ~typing.Literal[MyEnum.a]'),
+    ]
+
+    # restify() assumes that 'fully-qualified' is 'fully-qualified-except-typing'
+    # because it is more likely that a user wants to suppress 'typing.*'
+    app.config.autodoc_typehints_format = 'fully-qualified'
+    actual = do_autodoc(app, 'module', 'target.literal', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.literal',
+        '',
+        *bounded_typevar_rst('T', r'\ :py:obj:`~typing.Literal`\ [1234]'),
+        *bounded_typevar_rst('U', r'\ :py:obj:`~typing.Literal`\ [:py:attr:`target.literal.MyEnum.a`]'),
+        *function_rst('bar', 'x: typing.Literal[1234]'),
+        *function_rst('foo', 'x: typing.Literal[target.literal.MyEnum.a]'),
+    ]

tests/test_util_typing.py

@@ -1,6 +1,7 @@
 """Tests util.typing functions."""
 
 import sys
+from enum import Enum
 from numbers import Integral
 from struct import Struct
 from types import TracebackType
@@ -31,6 +32,10 @@ class MyClass2(MyClass1):
     __qualname__ = '<MyClass2>'
 
 
+class MyEnum(Enum):
+    a = 1
+
+
 T = TypeVar('T')
 MyInt = NewType('MyInt', int)
 
@@ -194,6 +199,9 @@ def test_restify_type_Literal():
     from typing import Literal  # type: ignore[attr-defined]
     assert restify(Literal[1, "2", "\r"]) == ":py:obj:`~typing.Literal`\\ [1, '2', '\\r']"
 
+    assert restify(Literal[MyEnum.a], 'fully-qualified-except-typing') == ':py:obj:`~typing.Literal`\\ [:py:attr:`tests.test_util_typing.MyEnum.a`]'
+    assert restify(Literal[MyEnum.a], 'smart') == ':py:obj:`~typing.Literal`\\ [:py:attr:`~tests.test_util_typing.MyEnum.a`]'
+
 
 def test_restify_pep_585():
     assert restify(list[str]) == ":py:class:`list`\\ [:py:class:`str`]"  # type: ignore[attr-defined]
@@ -478,6 +486,10 @@ def test_stringify_type_Literal():
     assert stringify_annotation(Literal[1, "2", "\r"], "fully-qualified") == "typing.Literal[1, '2', '\\r']"
     assert stringify_annotation(Literal[1, "2", "\r"], "smart") == "~typing.Literal[1, '2', '\\r']"
 
+    assert stringify_annotation(Literal[MyEnum.a], 'fully-qualified-except-typing') == 'Literal[tests.test_util_typing.MyEnum.a]'
+    assert stringify_annotation(Literal[MyEnum.a], 'fully-qualified') == 'typing.Literal[tests.test_util_typing.MyEnum.a]'
+    assert stringify_annotation(Literal[MyEnum.a], 'smart') == '~typing.Literal[MyEnum.a]'
+
 
 @pytest.mark.skipif(sys.version_info[:2] <= (3, 9), reason='python 3.10+ is required.')
 def test_stringify_type_union_operator():