Sphinx Doc-comments support

LICENSE.txt

@@ -117,3 +117,67 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+
+Support for token processing and doc-comments has been adapted from the Sphinx project - 
+as well as many other docstring parsing related helpers and features. 
+Sphinx is licensed as follows: 
+
+
+Copyright (c) 2007-2024 by the Sphinx team (see AUTHORS file).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+The implementation of numpydoc docstring preprocessor 
+was derived from Sphinx's which itselft is partially derived 
+from code under the following license:
+
+-------------------------------------------------------------------------------
+
+Copyright (C) 2008 Stefan van der Walt <[email protected]>, Pauli Virtanen <[email protected]>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

README.rst

@@ -77,6 +77,8 @@ in development
 ^^^^^^^^^^^^^^
 
 * Drop Python 3.7 and support Python 3.13.
+* Add support for doc-comments as found in Sphinx. Use the special comment formatting ``#:`` to start the comment instead of just ``#``.
+  Comments need to be either on a line of their own before the definition, or immediately after the assignment on the same line. 
 * Trigger a warning when several docstrings are detected for the same object.
 * Improve typing of docutils related code.
 * Run unit tests on all supported combinations of Python versions and platforms, including PyPy for Windows. Previously, tests where ran on all supported Python version for Linux, but not for MacOS and Windows.

docs/source/codedoc.rst

@@ -83,6 +83,28 @@ Assignments to ``__doc__`` inside functions are ignored by pydoctor. This can be
 
 Augmented assignments like ``+=`` are currently ignored as well, but that is an implementation limitation rather than a design decision, so this might change in the future.
 
+Doc-comments
+------------
+
+Documentation can also be put into a comment with special formatting, using a ``#:`` to start the comment instead of just ``#``.
+Comments need to be either on their own before the definition, OR immediately after the assignment on the same line. 
+The latter form is restricted to one line only.::
+
+    var = True #: Doc comment for module attribute.
+
+    class Foo:
+
+        #: Doc comment for class attribute Foo.bar.
+        #: It can have multiple lines.
+        #: @type: int
+        bar = 1
+
+        flox = 1.5   #: Doc comment for Foo.flox. One line only.
+
+        def __init__(self):
+            #: Doc comment for instance attribute qux.
+            self.qux = 3
+
 Constants
 ---------
 

pydoctor/astbuilder.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import ast
+import tokenize
 import sys
 
 from functools import partial
@@ -17,14 +18,17 @@
 from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval
 from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, 
                                is__name__equals__main__, unstring_annotation, upgrade_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents,
-                               get_docstring_node, get_assign_docstring_node, unparse, NodeVisitor, Parentage, Str)
+                               get_docstring_node, get_assign_docstring_node, extract_doc_comment_before, extract_doc_comment_after, unparse, NodeVisitor, Parentage, Str)
 
+def parseFile(path: Path) -> tuple[ast.Module, Sequence[str]]:
+    """
+    Parse the contents of a Python source file.
 
-def parseFile(path: Path) -> ast.Module:
-    """Parse the contents of a Python source file."""
-    with open(path, 'rb') as f:
-        src = f.read() + b'\n'
-    return _parse(src, filename=str(path))
+    @returns: Tuple: ast module, sequence of source code lines.
+    """
+    with tokenize.open(path) as f: 
+        src = f.read() + '\n'
+    return _parse(src, filename=str(path)), src.splitlines(keepends=True)
 
 if sys.version_info >= (3,8):
     _parse = partial(ast.parse, type_comments=True)
@@ -747,6 +751,25 @@ def _handleAssignment(self,
         else:
             raise IgnoreAssignment()
 
+    def _handleDocComment(self, node: ast.Assign | ast.AnnAssign, target: ast.expr) -> None:
+        # Process the doc-comments, this is very similiar to the inline docstrings.
+        try:
+            parent, name = self._contextualizeTarget(target)
+        except ValueError:
+            return
+
+        # fetch the target of the doc-comment
+        if (attr:=parent.contents.get(name)) is None:
+            return
+
+        lines = self.builder.lines_collection[self.module]
+        if lines:
+            for doc_comment in [extract_doc_comment_before(node, lines), 
+                                extract_doc_comment_after(node, lines)]:
+                if doc_comment:
+                    attr._setDocstringValue(doc_comment[1], doc_comment[0])
+
+
     def visit_Assign(self, node: ast.Assign) -> None:
         lineno = node.lineno
         expr = node.value
@@ -774,9 +797,11 @@ def visit_Assign(self, node: ast.Assign) -> None:
                 continue
             else:
                 if not isTupleAssignment:
+                    self._handleDocComment(node, target)
                     self._handleInlineDocstrings(node, target)
                 else:
                     for elem in cast(ast.Tuple, target).elts: # mypy is not as smart as pyright yet.
+                        self._handleDocComment(node, elem)
                         self._handleInlineDocstrings(node, elem)
 
     def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
@@ -787,6 +812,7 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
         except IgnoreAssignment:
             return
         else:
+            self._handleDocComment(node, node.target)
             self._handleInlineDocstrings(node, node.target)
 
     def _getClassFromMethodContext(self) -> Optional[model.Class]:
@@ -1133,11 +1159,12 @@ class ASTBuilder:
     def __init__(self, system: model.System):
         self.system = system
 
-        self.current = cast(model.Documentable, None) # current visited object.
-        self.currentMod: Optional[model.Module] = None # current module, set when visiting ast.Module.
+        self.current = cast(model.Documentable, None) #: current visited object
+        self.currentMod: Optional[model.Module] = None #: module, set when visiting ast.Module
 
         self._stack: List[model.Documentable] = []
-        self.ast_cache: Dict[Path, Optional[ast.Module]] = {}
+        self.ast_cache: Dict[Path, Optional[ast.Module]] = {} #: avoids calling parse() twice for the same path
+        self.lines_collection: dict[model.Module, Sequence[str] | None] = {} #: mapping from modules to source code lines 
 
     def _push(self, 
               cls: Type[DocumentableT], 
@@ -1248,20 +1275,24 @@ def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]:
             return self.ast_cache[path]
         except KeyError:
             mod: Optional[ast.Module] = None
+            lines: Sequence[str] | None = None
             try:
-                mod = parseFile(path)
+                mod, lines = parseFile(path)
             except (SyntaxError, ValueError) as e:
                 ctx.report(f"cannot parse file, {e}")
 
             self.ast_cache[path] = mod
+            self.lines_collection[ctx] = lines
             return mod
 
     def parseString(self, py_string:str, ctx: model.Module) -> Optional[ast.Module]:
-        mod = None
+        mod: Optional[ast.Module] = None
+        lines: Sequence[str] | None = None
         try:
-            mod = _parse(py_string)
+            mod, lines = _parse(py_string), py_string.splitlines(keepends=True)
         except (SyntaxError, ValueError):
             ctx.report("cannot parse string")
+        self.lines_collection[ctx] = lines
         return mod
 
 model.System.defaultBuilder = ASTBuilder