Fix and refactors for docparams extension (#7398)

* Fix and refactors for ``docparams`` extension

The ``re_only_desc`` regex did not match for white and characters after
``\n``, so some description-only lines weren't getting matched. In
addition, lookaheads were added to ``re_param_line`` (i.e. make sure the
type group is not followed by a new line (``\n``)). Lastly, named groups
(ala Perl regular expressions) were added for slightly improved clarity.

Co-authored-by: Hendry, Adam <[email protected]>

Author: 2 people

doc/whatsnew/fragments/7398.other

@@ -0,0 +1,4 @@
+The ``docparams`` extension now considers typing in Numpy style docstrings
+as "documentation" for the ``missing-param-doc`` message.
+
+Refs #7398

pylint/extensions/_check_docs_utils.py

@@ -247,7 +247,7 @@ class SphinxDocstring(Docstring):
 
     re_multiple_simple_type = r"""
         (?:{container_type}|{type})
-        (?:(?:\s+(?:of|or)\s+|\s*,\s*)(?:{container_type}|{type}))*
+        (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}))*
     """.format(
         type=re_type, container_type=re_simple_container_type
     )
@@ -449,7 +449,7 @@ class GoogleDocstring(Docstring):
 
     re_multiple_type = r"""
         (?:{container_type}|{type}|{xref})
-        (?:(?:\s+(?:of|or)\s+|\s*,\s*)(?:{container_type}|{type}|{xref}))*
+        (?:(?:\s+(?:of|or)\s+|\s*,\s*|\s+\|\s+)(?:{container_type}|{type}|{xref}))*
     """.format(
         type=re_type, xref=re_xref, container_type=re_container_type
     )
@@ -728,22 +728,25 @@ class NumpyDocstring(GoogleDocstring):
         re.X | re.S | re.M,
     )
 
-    re_default_value = r"""((['"]\w+\s*['"])|(True)|(False)|(None))"""
+    re_default_value = r"""((['"]\w+\s*['"])|(\d+)|(True)|(False)|(None))"""
 
     re_param_line = re.compile(
         rf"""
-        \s*  (\*{{0,2}}\w+)(\s?(:|\n))              # identifier with potential asterisks
+        \s*  (?P<param_name>\*{{0,2}}\w+)(\s?(:|\n)) # identifier with potential asterisks
         \s*
-        (
+        (?P<param_type>
          (
           ({GoogleDocstring.re_multiple_type})      # default type declaration
           (,\s+optional)?                           # optional 'optional' indication
          )?
          (
           {{({re_default_value},?\s*)+}}            # set of default values
          )?
-        \n)?
-        \s* (.*)                                    # optional description
+         (?:$|\n)
+        )?
+        (
+         \s* (?P<param_desc>.*)                     # optional description
+        )?
     """,
         re.X | re.S,
     )
@@ -794,15 +797,26 @@ def match_param_docs(self) -> tuple[set[str], set[str]]:
                 continue
 
             # check if parameter has description only
-            re_only_desc = re.match(r"\s*  (\*{0,2}\w+)\s*:?\n", entry)
+            re_only_desc = re.match(r"\s*(\*{0,2}\w+)\s*:?\n\s*\w*$", entry)
             if re_only_desc:
-                param_name = match.group(1)
-                param_desc = match.group(2)
+                param_name = match.group("param_name")
+                param_desc = match.group("param_type")
                 param_type = None
             else:
-                param_name = match.group(1)
-                param_type = match.group(3)
-                param_desc = match.group(4)
+                param_name = match.group("param_name")
+                param_type = match.group("param_type")
+                param_desc = match.group("param_desc")
+                # The re_param_line pattern needs to match multi-line which removes the ability
+                # to match a single line description like 'arg : a number type.'
+                # We are not trying to determine whether 'a number type' is correct typing
+                # but we do accept it as typing as it is in the place where typing
+                # should be
+                if param_type is None and re.match(r"\s*(\*{0,2}\w+)\s*:.+$", entry):
+                    param_type = param_desc
+                # If the description is "" but we have a type description
+                # we consider the description to be the type
+                if not param_desc and param_type:
+                    param_desc = param_type
 
             if param_type:
                 params_with_type.add(param_name)

tests/functional/ext/docparams/missing_param_doc.py

@@ -30,7 +30,7 @@ def foobar4(arg1, arg2): #[missing-param-doc, missing-type-doc]
     """
     print(arg1, arg2)
 
-def foobar5(arg1, arg2): #[missing-param-doc, missing-type-doc]
+def foobar5(arg1, arg2): #[missing-type-doc]
     """function foobar ...
     Parameters
     ----------
@@ -63,7 +63,7 @@ def foobar8(arg1): #[missing-any-param-doc]
 
     print(arg1)
 
-def foobar9(arg1, arg2, arg3): #[missing-param-doc]
+def foobar9(arg1, arg2, arg3):
     """function foobar ...
     Parameters
     ----------
@@ -73,7 +73,7 @@ def foobar9(arg1, arg2, arg3): #[missing-param-doc]
     """
     print(arg1, arg2, arg3)
 
-def foobar10(arg1, arg2, arg3): #[missing-param-doc, missing-type-doc]
+def foobar10(arg1, arg2, arg3): #[missing-type-doc]
     """function foobar ...
     Parameters
     ----------

tests/functional/ext/docparams/missing_param_doc.txt

@@ -1,18 +1,15 @@
 missing-any-param-doc:3:0:3:11:foobar1:"Missing any documentation in ""foobar1""":HIGH
 missing-any-param-doc:8:0:8:11:foobar2:"Missing any documentation in ""foobar2""":HIGH
-missing-param-doc:15:0:15:11:foobar3:"""arg1, arg2, arg3"" missing in parameter documentation":HIGH
+missing-param-doc:15:0:15:11:foobar3:"""arg2"" missing in parameter documentation":HIGH
 missing-type-doc:15:0:15:11:foobar3:"""arg2"" missing in parameter type documentation":HIGH
 missing-param-doc:24:0:24:11:foobar4:"""arg2"" missing in parameter documentation":HIGH
 missing-type-doc:24:0:24:11:foobar4:"""arg2"" missing in parameter type documentation":HIGH
-missing-param-doc:33:0:33:11:foobar5:"""arg2"" missing in parameter documentation":HIGH
 missing-type-doc:33:0:33:11:foobar5:"""arg1"" missing in parameter type documentation":HIGH
-missing-param-doc:43:0:43:11:foobar6:"""arg2, arg3"" missing in parameter documentation":HIGH
+missing-param-doc:43:0:43:11:foobar6:"""arg3"" missing in parameter documentation":HIGH
 missing-type-doc:43:0:43:11:foobar6:"""arg3"" missing in parameter type documentation":HIGH
 missing-any-param-doc:53:0:53:11:foobar7:"Missing any documentation in ""foobar7""":HIGH
 missing-any-param-doc:61:0:61:11:foobar8:"Missing any documentation in ""foobar8""":HIGH
-missing-param-doc:66:0:66:11:foobar9:"""arg1, arg2, arg3"" missing in parameter documentation":HIGH
-missing-param-doc:76:0:76:12:foobar10:"""arg2"" missing in parameter documentation":HIGH
 missing-type-doc:76:0:76:12:foobar10:"""arg1, arg3"" missing in parameter type documentation":HIGH
 missing-any-param-doc:88:0:88:12:foobar11:"Missing any documentation in ""foobar11""":HIGH
-missing-param-doc:97:0:97:12:foobar12:"""arg1, arg3"" missing in parameter documentation":HIGH
+missing-param-doc:97:0:97:12:foobar12:"""arg3"" missing in parameter documentation":HIGH
 missing-type-doc:97:0:97:12:foobar12:"""arg2, arg3"" missing in parameter type documentation":HIGH

tests/functional/ext/docparams/parameter/missing_param_doc_required_Numpy.py

@@ -392,6 +392,7 @@ def test_ignores_optional_specifier_numpy(param, param2="all"):
     """
     return param, param2
 
+
 def test_with_list_of_default_values(arg, option, option2):
     """Reported in https://github.com/PyCQA/pylint/issues/4035.
 
@@ -406,3 +407,18 @@ def test_with_list_of_default_values(arg, option, option2):
 
     """
     return arg, option, option2
+
+
+def test_with_descriptions_instead_of_typing(arg, axis, option):
+    """We choose to accept description in place of typing as well.
+
+    See: https://github.com/PyCQA/pylint/pull/7398.
+
+    Parameters
+    ----------
+    arg : a number type.
+    axis : int or None
+    option : {"y", "n"}
+        Do I do it?
+    """
+    return arg, option