From 2d61d540065c67e0958b3aed491f36e5e4b545a9 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 4 Oct 2024 10:18:26 -0400 Subject: [PATCH 01/13] format markdown --- .../src/main/server-jetty/requirements.txt | 1 + .../src/main/server-netty/requirements.txt | 1 + .../auto_completer/_completer.py | 13 +- .../auto_completer/_signature_help.py | 179 ++++++++++++++++++ py/server/setup.py | 2 +- .../completer/PythonAutoCompleteObserver.java | 4 +- 6 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 py/server/deephaven_internal/auto_completer/_signature_help.py diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 72c871eb253..125023cbc72 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -2,6 +2,7 @@ adbc-driver-manager==1.1.0 adbc-driver-postgresql==1.1.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 +docstring_parser==0.16 importlib_resources==6.4.3 java-utilities==0.3.0 jedi==0.19.1 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 72c871eb253..125023cbc72 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -2,6 +2,7 @@ adbc-driver-manager==1.1.0 adbc-driver-postgresql==1.1.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 +docstring_parser==0.16 importlib_resources==6.4.3 java-utilities==0.3.0 jedi==0.19.1 diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index 39ea08cc25a..7be6dd55338 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -3,10 +3,12 @@ # from __future__ import annotations from enum import Enum +from docstring_parser import parse from typing import Any, Union, List from jedi import Interpreter, Script from jedi.api.classes import Completion, Signature from importlib.metadata import version +from ._signature_help import _get_signature_result import sys import warnings @@ -78,6 +80,8 @@ class Completer: def __init__(self): self._docs = {} self._versions = {} + # Cache for signature markdown + self.signature_cache = {} # we will replace this w/ top-level globals() when we open the document self.__scope = globals() # might want to make this a {uri: []} instead of [] @@ -214,14 +218,7 @@ def do_signature_help( # keep checking the latest version as we run, so updated doc can cancel us if not self._versions[uri] == version: return [] - - result: list = [ - signature.to_string(), - signature.docstring(raw=True), - [[param.to_string().strip(), param.docstring(raw=True).strip()] for param in signature.params], - signature.index if signature.index is not None else -1 - ] - results.append(result) + results.append(_get_signature_result(signature)) return results diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py new file mode 100644 index 00000000000..403ef617154 --- /dev/null +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -0,0 +1,179 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# +from __future__ import annotations +from inspect import Parameter +from typing import Any, List +from docstring_parser import parse, Docstring +from jedi.api.classes import Signature + +from pprint import pprint + +result_cache = {} + + +def _hash(signature: Signature) -> str: + """A simple way to identify signatures""" + return f"{signature.to_string()}\n{signature.docstring(raw=True)}" + + +def _generate_param_markdown(param: dict) -> List[Any]: + description = f"##### **{param['name']}**" + if param['type'] is not None: + description += f": *{param['type']}*" + description += "\n\n" + + if param['default'] is not None: + description += f"Default: {param['default']}\n\n" + + if param['description'] is not None: + description += f"{param['description']}\n\n" + + return description + "---" + + +def _get_params(signature: Signature, docs: Docstring) -> List[Any]: + """ + Combines all available parameter information from the signature and docstring. + + Args: + signature: The signature from `jedi` + docs: The parsed docstring from `docstring_parser` + + Returns: + A list of dictionaries that contain the parameter name, description, type, and default value. + """ + + params = [] + params_info = {} + + # Take information from docs first + for param in docs.params: + params_info[param.arg_name.replace("*", "")] = { + "description": param.description.strip(), + "type": param.type_name, + } + + for param in signature.params: + param_str = param.to_string().strip() + + # Add back * or ** for display purposes only + if param.kind == Parameter.VAR_POSITIONAL: + name = f"*{param.name}" + elif param.kind == Parameter.VAR_KEYWORD: + name = f"**{param.name}" + else: + name = param.name + + # Use type in signature first, then type in docs, then None + if ":" in param_str: + type_ = param_str.split(":")[1].split("=")[0].strip() + elif param.name in params_info: + type_ = params_info[param.name]["type"] + else: + type_ = None + + params.append({ + "name": name, + "description": params_info.get(param.name, {}).get("description"), + "type": type_, + "default": param_str.split("=")[1] if "=" in param_str else None, + }) + + return params + + +def _get_raises(docs: Docstring) -> List[Any]: + raises = [] + for raise_ in docs.raises: + raises.append({ + "name": raise_.type_name, + "description": raise_.description + }) + + return raises + + +def _get_returns(docs: Docstring) -> List[Any]: + returns = [] + for return_ in docs.many_returns: + returns.append({ + "name": return_.type_name, + "description": return_.description + }) + + return returns + + +def _get_signature_result(signature: Signature) -> List[Any]: + """ Gets the result of a signature to be used by `do_signature_help` + + Returns: + A list that contains [signature name, docstring, param docstrings, index] + """ + + docstring = signature.docstring(raw=True) + cache_key = _hash(signature) + + if cache_key in result_cache: + result = result_cache[cache_key].copy() # deep copy not needed since only index is different + result.append(signature.index if signature.index is not None else -1) + return result + + docs = parse(docstring) + + # Nothing parsed, revert to plaintext + if docstring == docs.description: + return [ + signature.to_string(), + signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), + [[param.to_string().strip(), ""] for param in signature.params], + signature.index if signature.index is not None else -1, + ] + + params = _get_params(signature, docs) + raises = _get_raises(docs) + returns = _get_returns(docs) + description = docs.description.strip().replace("\n", " \n") + "\n\n" + + if len(params) > 0: + description += "#### **Parameters**\n\n" + for param in params: + description += f"> **{param['name']}**" + if param['type'] is not None: + description += f": *{param['type']}*" + if param['default'] is not None: + description += f" ⋅ (default: *{param['default']}*)" + description += " \n" + + if param['description'] is not None: + description += f"> {param['description']}" + description += "\n\n" + + if len(returns) > 0: + description += "#### **Returns**\n\n" + for return_ in returns: + if return_["name"] is not None: + description += f"> **{return_['name']}** \n" + if return_["description"] is not None: + description += f"> {return_['description']}" + description += "\n\n" + + if len(raises) > 0: + description += "#### **Raises**\n\n" + for raises_ in raises: + if raises_["name"] is not None: + description += f"> **{raises_['name']}** \n" + if raises_["description"] is not None: + description += f"> {raises_['description']}" + description += "\n\n" + + result = [ + f"{signature.to_string().split('(')[0]}(...)" if len(signature.to_string()) > 20 else signature.to_string(), + description.strip(), + [[signature.params[i].to_string().strip(), _generate_param_markdown(params[i])] for i in range(len(signature.params))], + ] + result_cache[cache_key] = result.copy() # deep copy not needed since only index is different + result.append(signature.index if signature.index is not None else -1) + + return result diff --git a/py/server/setup.py b/py/server/setup.py index 874787746d8..76c6858f52f 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -68,7 +68,7 @@ def _compute_version(): 'numba; python_version < "3.13"', ], extras_require={ - "autocomplete": ["jedi==0.19.1"], + "autocomplete": ["jedi==0.19.1", "docstring_parser==0.16"], }, entry_points={ 'deephaven.plugin': ['registration_cls = deephaven.pandasplugin:PandasPluginRegistration'] diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java index 22c46a78797..0dfd4709f98 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -257,14 +257,14 @@ private GetSignatureHelpResponse getSignatureHelp(GetSignatureHelpRequest reques final SignatureInformation.Builder item = SignatureInformation.newBuilder(); item.setLabel(label); - item.setDocumentation(MarkupContent.newBuilder().setValue(docstring).setKind("plaintext").build()); + item.setDocumentation(MarkupContent.newBuilder().setValue(docstring).setKind("markdown").build()); item.setActiveParameter(activeParam); signature.get(2).asList().forEach(obj -> { final List param = obj.asList(); item.addParameters(ParameterInformation.newBuilder().setLabel(param.get(0).getStringValue()) .setDocumentation(MarkupContent.newBuilder().setValue(param.get(1).getStringValue()) - .setKind("plaintext").build())); + .setKind("markdown").build())); }); finalItems.add(item.build()); From 775a14306b56de266ba9aeda12dd573f98633af0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 7 Oct 2024 14:45:01 -0400 Subject: [PATCH 02/13] update docker server --- docker/registry/server-base/gradle.properties | 2 +- docker/registry/slim-base/gradle.properties | 2 +- .../src/main/server-jetty/requirements.txt | 16 ++++++++-------- .../src/main/server-netty/requirements.txt | 16 ++++++++-------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties index a9314887d1b..0e864d1e778 100644 --- a/docker/registry/server-base/gradle.properties +++ b/docker/registry/server-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge -deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:66f8cecdac170dfb8bf284e41684480359a15443dc5a5b8a1efc95987f3ddcb5 +deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:952cdb2da5da301711d3ebd8a324ecb85dc47bde94d4305bf9b34a8cec3bf0db diff --git a/docker/registry/slim-base/gradle.properties b/docker/registry/slim-base/gradle.properties index 61745e6630c..ed38bd628c9 100644 --- a/docker/registry/slim-base/gradle.properties +++ b/docker/registry/slim-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-slim-base:edge -deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:0605678d4961227fa0a8e83c0c8b030f56e581c070504dcca49a44a9ab43d8f0 +deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:1cad36879e4d2161d7dc3658a9aee261c58d3a1ae005f412bad58fc6636e0d4c diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 125023cbc72..56fbb07863e 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -1,20 +1,20 @@ -adbc-driver-manager==1.1.0 -adbc-driver-postgresql==1.1.0 -connectorx==0.3.3; platform.machine == 'x86_64' +adbc-driver-manager==1.2.0 +adbc-driver-postgresql==1.2.0 +connectorx==0.3.3 deephaven-plugin==0.6.0 docstring_parser==0.16 -importlib_resources==6.4.3 +importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 jpy==0.18.0 llvmlite==0.43.0 numba==0.60.0 -numpy==2.0.1 -pandas==2.2.2 +numpy==2.0.2 +pandas==2.2.3 parso==0.8.4 pyarrow==17.0.0 python-dateutil==2.9.0.post0 -pytz==2024.1 +pytz==2024.2 six==1.16.0 typing_extensions==4.12.2 -tzdata==2024.1 +tzdata==2024.2 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 125023cbc72..56fbb07863e 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -1,20 +1,20 @@ -adbc-driver-manager==1.1.0 -adbc-driver-postgresql==1.1.0 -connectorx==0.3.3; platform.machine == 'x86_64' +adbc-driver-manager==1.2.0 +adbc-driver-postgresql==1.2.0 +connectorx==0.3.3 deephaven-plugin==0.6.0 docstring_parser==0.16 -importlib_resources==6.4.3 +importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 jpy==0.18.0 llvmlite==0.43.0 numba==0.60.0 -numpy==2.0.1 -pandas==2.2.2 +numpy==2.0.2 +pandas==2.2.3 parso==0.8.4 pyarrow==17.0.0 python-dateutil==2.9.0.post0 -pytz==2024.1 +pytz==2024.2 six==1.16.0 typing_extensions==4.12.2 -tzdata==2024.1 +tzdata==2024.2 From d22070fb96d653abf6d5f5ffc3fa09181f373f91 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 8 Oct 2024 09:22:58 -0400 Subject: [PATCH 03/13] fix requirements --- docker/server-jetty/src/main/server-jetty/requirements.txt | 4 ++-- docker/server/src/main/server-netty/requirements.txt | 4 ++-- py/server/setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 56fbb07863e..9ca0a2784d7 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -1,8 +1,8 @@ adbc-driver-manager==1.2.0 adbc-driver-postgresql==1.2.0 -connectorx==0.3.3 +connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -docstring_parser==0.16 +docstring_parser>=0.16 importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 56fbb07863e..9ca0a2784d7 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -1,8 +1,8 @@ adbc-driver-manager==1.2.0 adbc-driver-postgresql==1.2.0 -connectorx==0.3.3 +connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -docstring_parser==0.16 +docstring_parser>=0.16 importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 diff --git a/py/server/setup.py b/py/server/setup.py index 76c6858f52f..a69f36814c7 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -68,7 +68,7 @@ def _compute_version(): 'numba; python_version < "3.13"', ], extras_require={ - "autocomplete": ["jedi==0.19.1", "docstring_parser==0.16"], + "autocomplete": ["jedi==0.19.1", "docstring_parser>=0.16"], }, entry_points={ 'deephaven.plugin': ['registration_cls = deephaven.pandasplugin:PandasPluginRegistration'] From 0ef6462122a9de87f63b5efa4855b38898a3bea7 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 11 Oct 2024 10:47:25 -0400 Subject: [PATCH 04/13] add dynamic signature, clean --- .../auto_completer/_completer.py | 2 - .../auto_completer/_signature_help.py | 179 ++++++++++++------ 2 files changed, 122 insertions(+), 59 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index 7be6dd55338..9a284c7bfd3 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -3,11 +3,9 @@ # from __future__ import annotations from enum import Enum -from docstring_parser import parse from typing import Any, Union, List from jedi import Interpreter, Script from jedi.api.classes import Completion, Signature -from importlib.metadata import version from ._signature_help import _get_signature_result import sys import warnings diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 403ef617154..61aff88658f 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -3,12 +3,19 @@ # from __future__ import annotations from inspect import Parameter -from typing import Any, List +from typing import Any from docstring_parser import parse, Docstring from jedi.api.classes import Signature -from pprint import pprint +IGNORE_PARAM_NAMES = ("", "/", "*") +MAX_DISPLAY_SIG_LEN = 128 # 3 lines is 150, but there could be overflow so 150 could result in 4 lines +POSITIONAL_KINDS = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD, Parameter.VAR_POSITIONAL) + +# key: result from _hash +# value: another dictionary that has the following keys: +# description: The markdown description (result from _generate_description_markdown) +# param_docs: A list of param markdown descriptions (result from _generate_param_markdowns) result_cache = {} @@ -17,22 +24,7 @@ def _hash(signature: Signature) -> str: return f"{signature.to_string()}\n{signature.docstring(raw=True)}" -def _generate_param_markdown(param: dict) -> List[Any]: - description = f"##### **{param['name']}**" - if param['type'] is not None: - description += f": *{param['type']}*" - description += "\n\n" - - if param['default'] is not None: - description += f"Default: {param['default']}\n\n" - - if param['description'] is not None: - description += f"{param['description']}\n\n" - - return description + "---" - - -def _get_params(signature: Signature, docs: Docstring) -> List[Any]: +def _get_params(signature: Signature, docs: Docstring) -> list[Any]: """ Combines all available parameter information from the signature and docstring. @@ -83,7 +75,7 @@ def _get_params(signature: Signature, docs: Docstring) -> List[Any]: return params -def _get_raises(docs: Docstring) -> List[Any]: +def _get_raises(docs: Docstring) -> list[Any]: raises = [] for raise_ in docs.raises: raises.append({ @@ -94,7 +86,7 @@ def _get_raises(docs: Docstring) -> List[Any]: return raises -def _get_returns(docs: Docstring) -> List[Any]: +def _get_returns(docs: Docstring) -> list[Any]: returns = [] for return_ in docs.many_returns: returns.append({ @@ -105,40 +97,21 @@ def _get_returns(docs: Docstring) -> List[Any]: return returns -def _get_signature_result(signature: Signature) -> List[Any]: - """ Gets the result of a signature to be used by `do_signature_help` - - Returns: - A list that contains [signature name, docstring, param docstrings, index] - """ - - docstring = signature.docstring(raw=True) - cache_key = _hash(signature) - - if cache_key in result_cache: - result = result_cache[cache_key].copy() # deep copy not needed since only index is different - result.append(signature.index if signature.index is not None else -1) - return result - - docs = parse(docstring) - - # Nothing parsed, revert to plaintext - if docstring == docs.description: - return [ - signature.to_string(), - signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), - [[param.to_string().strip(), ""] for param in signature.params], - signature.index if signature.index is not None else -1, - ] - - params = _get_params(signature, docs) +def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: raises = _get_raises(docs) returns = _get_returns(docs) - description = docs.description.strip().replace("\n", " \n") + "\n\n" + + if docs.description is None: + description = "" + else: + description = docs.description.strip().replace("\n", " \n") + "\n\n" if len(params) > 0: description += "#### **Parameters**\n\n" for param in params: + if param['name'] in IGNORE_PARAM_NAMES: + continue + description += f"> **{param['name']}**" if param['type'] is not None: description += f": *{param['type']}*" @@ -147,7 +120,7 @@ def _get_signature_result(signature: Signature) -> List[Any]: description += " \n" if param['description'] is not None: - description += f"> {param['description']}" + description += f"> {param['description']}".replace('\n\n', '\n\n> ') description += "\n\n" if len(returns) > 0: @@ -168,12 +141,104 @@ def _get_signature_result(signature: Signature) -> List[Any]: description += f"> {raises_['description']}" description += "\n\n" - result = [ - f"{signature.to_string().split('(')[0]}(...)" if len(signature.to_string()) > 20 else signature.to_string(), - description.strip(), - [[signature.params[i].to_string().strip(), _generate_param_markdown(params[i])] for i in range(len(signature.params))], - ] - result_cache[cache_key] = result.copy() # deep copy not needed since only index is different - result.append(signature.index if signature.index is not None else -1) + return description.strip() + + +def _generate_display_sig(signature: Signature) -> str: + if len(signature.to_string()) <= MAX_DISPLAY_SIG_LEN: + return signature.to_string() + + # Use 0 as default to display start of signature + index = signature.index if signature.index is not None else 0 + display_sig = f"{signature.name}(" + + if index > 0: + display_sig += "..., " + + # If current arg is positional, display next 2 args + # If current arg is keyword, only display current args + if signature.params[index].kind in POSITIONAL_KINDS: + # Clamp index so that 3 args are shown, even at last index + index = max(min(index, len(signature.params) - 3), 0) + end_index = index + 3 + # If the next arg is not positional, do not show the one after it + # Otherwise, this arg will show 2 ahead, and then next arg will show 0 ahead + if signature.params[index + 1].kind not in POSITIONAL_KINDS: + end_index -= 1 + display_sig += ", ".join([param.to_string() for param in signature.params[index:end_index]]) + if index + 3 < len(signature.params): + display_sig += ", ..." + else: + display_sig += signature.params[index].to_string() + if index + 1 < len(signature.params): + display_sig += ", ..." + + return display_sig + ")" + + +def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[Any]: + param_docs = [] + for i in range(len(signature.params)): + if signature.params[i].to_string().strip() in IGNORE_PARAM_NAMES: + continue + + param = params[i] + description = f"##### **{param['name']}**" + if param['type'] is not None: + description += f": *{param['type']}*" + description += "\n\n" + if param['description'] is not None: + description += f"{param['description']}\n\n" + description += "---" + + param_docs.append([signature.params[i].to_string().strip(), description]) + + return param_docs + + +def _get_signature_result(signature: Signature) -> list[Any]: + """ Gets the result of a signature to be used by `do_signature_help` + + Returns: + A list that contains [signature name, docstring, param docstrings, index] + """ + + docstring = signature.docstring(raw=True) + cache_key = _hash(signature) + + # Check cache + if cache_key in result_cache: + result = result_cache[cache_key] + return [ + _generate_display_sig(signature), + result["description"], + result["param_docs"], + signature.index if signature.index is not None else -1, + ] - return result + # Parse the docstring to extract information + docs = parse(docstring) + # Nothing parsed, revert to plaintext + if docstring == docs.description: + return [ + signature.to_string(), + signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), + [[param.to_string().strip(), ""] for param in signature.params], + signature.index if signature.index is not None else -1, + ] + + # Get params in this scope because it'll be used multiple times + params = _get_params(signature, docs) + description = _generate_description_markdown(docs, params) + param_docs = _generate_param_markdowns(signature, params) + result_cache[cache_key] = { + "description": description, + "param_docs": param_docs, + } + + return [ + _generate_display_sig(signature), + description, + param_docs, + signature.index if signature.index is not None else -1, + ] From a9186958825e6d9bce6fcf26944e84cf7d4487c5 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 11 Oct 2024 13:38:18 -0400 Subject: [PATCH 05/13] fix requirements --- docker/server-jetty/src/main/server-jetty/requirements.txt | 2 +- docker/server/src/main/server-netty/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 9ca0a2784d7..e12ec196b30 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -2,7 +2,7 @@ adbc-driver-manager==1.2.0 adbc-driver-postgresql==1.2.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -docstring_parser>=0.16 +docstring_parser==0.16 importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 9ca0a2784d7..e12ec196b30 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -2,7 +2,7 @@ adbc-driver-manager==1.2.0 adbc-driver-postgresql==1.2.0 connectorx==0.3.3; platform.machine == 'x86_64' deephaven-plugin==0.6.0 -docstring_parser>=0.16 +docstring_parser==0.16 importlib_resources==6.4.5 java-utilities==0.3.0 jedi==0.19.1 From b9df3f55483c7a42e41f6b07672fd76bfaa1c992 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 18 Oct 2024 10:52:23 -0400 Subject: [PATCH 06/13] add tests --- .../auto_completer/_signature_help.py | 25 +-- py/server/tests/data/signatures.py | 188 ++++++++++++++++++ py/server/tests/test_docstring_parser.py | 115 +++++++++++ 3 files changed, 310 insertions(+), 18 deletions(-) create mode 100644 py/server/tests/data/signatures.py create mode 100644 py/server/tests/test_docstring_parser.py diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 61aff88658f..982982837c2 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -76,25 +76,11 @@ def _get_params(signature: Signature, docs: Docstring) -> list[Any]: def _get_raises(docs: Docstring) -> list[Any]: - raises = [] - for raise_ in docs.raises: - raises.append({ - "name": raise_.type_name, - "description": raise_.description - }) - - return raises + return [dict(name=raise_.type_name, description=raise_.description) for raise_ in docs.raises] def _get_returns(docs: Docstring) -> list[Any]: - returns = [] - for return_ in docs.many_returns: - returns.append({ - "name": return_.type_name, - "description": return_.description - }) - - return returns + return [dict(name=return_.type_name, description=return_.description) for return_ in docs.many_returns] def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: @@ -185,7 +171,7 @@ def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[A param = params[i] description = f"##### **{param['name']}**" if param['type'] is not None: - description += f": *{param['type']}*" + description += f" : *{param['type']}*" description += "\n\n" if param['description'] is not None: description += f"{param['description']}\n\n" @@ -219,7 +205,10 @@ def _get_signature_result(signature: Signature) -> list[Any]: # Parse the docstring to extract information docs = parse(docstring) # Nothing parsed, revert to plaintext - if docstring == docs.description: + # Based on code, the meta attribute seems to be a list of parsed items. Then, the parse function returns the + # style with the most amount of items in meta. If there are no items, that should be mean nothing was parsed. + # https://github.com/rr-/docstring_parser/blob/4951137875e79b438d52a18ac971ec0c28ef269c/docstring_parser/parser.py#L46 + if len(docs.meta) == 0: return [ signature.to_string(), signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), diff --git a/py/server/tests/data/signatures.py b/py/server/tests/data/signatures.py new file mode 100644 index 00000000000..320613603f4 --- /dev/null +++ b/py/server/tests/data/signatures.py @@ -0,0 +1,188 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# +def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): + """ + Description + + Args: + has_docs: Arg has docs + has_type: Arg has type + not_real: Arg does not exist in signature + *positional: Positional arg has docs + has_default: Arg has default + has_type_default: Arg has type and default + **keyword: Keyword arg has docs + """ + +args_str = """ +def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): + \"\"\" + Description + + Args: + has_docs: Arg has docs + has_type: Arg has type + not_real: Arg does not exist in signature + *positional: Positional arg has docs + has_default: Arg has default + has_type_default: Arg has type and default + **keyword: Keyword arg has docs + \"\"\" +""" + +args_str_result = """\ +Description + +#### **Parameters** + +> **has_docs** +> Arg has docs + +> **has_type**: *str | int* +> Arg has type + +> ***positional** +> Positional arg has docs + +> **has_default** ⋅ (default: *1*) +> Arg has default + +> **has_type_default**: *str | int* ⋅ (default: *1*) +> Arg has type and default + +> ****keyword** +> Keyword arg has docs""" + +def args_no_docs(no_docs, /, *, keyword_no_docs=None): + """ + Description + + Args: + not_real: Arg does not exist in signature + /: Should not show + *: Should not show + """ + +args_no_docs_str = """ +def args_no_docs(no_docs, /, *, keyword_no_docs=None): + \"\"\" + Description + + Args: + not_real: Arg does not exist in signature + /: Should not show + *: Should not show + \"\"\" +""" + +args_no_docs_result = """\ +Description + +#### **Parameters** + +> **no_docs** + + +> **keyword_no_docs** ⋅ (default: *None*)""" + +def raises_various(): + """ + Description + + Raises: + Exception: Exception description + ValueError: ValueError description. + This is a continuation of ValueError + """ + +raises_various_str = """ +def raises_various(): + \"\"\" + Description + + Raises: + Exception: Exception description + ValueError: ValueError description. + This is a continuation of ValueError + \"\"\" +""" + +raises_various_result = """\ +Description + +#### **Raises** + +> **Exception** +> Exception description + +> **ValueError** +> ValueError description. +This is a continuation of ValueError""" + +def returns_various(): + """ + :returns: Return has docs + :returns foo: foo description + :returns bar: bar description + """ + +returns_various_str = """ +def returns_various(): + \"\"\" + :returns: Return has docs + :returns foo: foo description + :returns bar: bar description + \"\"\" +""" + +returns_various_result = """\ +#### **Returns** + +> Return has docs + +> **foo** +> foo description + +> **bar** +> bar description""" + +def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): + """ + :returns a: b + """ + +original_signature_str = """ +def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): + \"\"\" + :returns a: b + \"\"\" +""" + +def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, + aaaaaa10, aaaaaa11, aaaaaa12): + """ + :returns a: b + """ + +truncate_positional_str = """ +def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, + aaaaaa10, aaaaaa11, aaaaaa12): + \"\"\" + :returns a: b + \"\"\" +""" + +def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, + aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): + """ + :returns a: b + """ + +truncate_keyword_str = """ +def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, + aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): + \"\"\" + :returns a: b + \"\"\" +""" diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py new file mode 100644 index 00000000000..edb8c9bb620 --- /dev/null +++ b/py/server/tests/test_docstring_parser.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# +from functools import wraps +from typing import Callable + +from docstring_parser import parse, Docstring +from jedi import Script, Interpreter +from jedi.api.classes import Signature + +from deephaven_internal.auto_completer._signature_help import _get_params, _generate_description_markdown, _generate_display_sig +from tests.testbase import BaseTestCase + +from .data.signatures import * + + +class DocstringParser(BaseTestCase): + + @staticmethod + def create_test(name: str, code: str, func: Callable, func_call_append: str = ""): + def decorator(f): + @wraps(f) + def wrapper(self): + s = Script(f"{code}\n{name}({func_call_append}").get_signatures() + self.assertIsInstance(s, list) + self.assertEqual(len(s), 1) + f(self, s[0], parse(s[0].docstring(raw=True))) + + i = Interpreter(f"{name}({func_call_append}", [{name: func}]).get_signatures() + self.assertIsInstance(s, list) + self.assertEqual(len(s), 1) + f(self, i[0], parse(i[0].docstring(raw=True))) + + return wrapper + return decorator + + @create_test("args", args_str, args) + def test_args(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + args_str_result + ) + + @create_test("args_no_docs", args_no_docs_str, args_no_docs) + def test_args_no_docs(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + args_no_docs_result + ) + + @create_test("raises_various", raises_various_str, raises_various) + def test_raises_various(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + raises_various_result + ) + + @create_test("returns_various", returns_various_str, returns_various) + def test_returns_various(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + returns_various_result + ) + + @create_test("original_signature", original_signature_str, original_signature) + def test_original_signature(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_display_sig(signature), + "original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09)" + ) + + @create_test("truncate_positional", truncate_positional_str, truncate_positional) + def test_truncate_positional_0(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, ...)") + + @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, ") + def test_truncate_positional_1(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa01, aaaaaa02, aaaaaa03, ...)") + + @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") + def test_truncate_positional_10(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") + + @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") + def test_truncate_positional_11(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") + + @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") + def test_truncate_positional_12(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") + + @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword) + def test_truncate_keyword_0(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_keyword(aaaaaa00, aaaaaa01=1, ...)") + + @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword, "1, ") + def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa01=1, ...)") + + @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword, "1, aaaaaa12=") + def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa012=1)") From e810344b89cf52b504e8cf6643825b623c583d2e Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 18 Oct 2024 12:05:46 -0400 Subject: [PATCH 07/13] fix test --- py/server/tests/test_docstring_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py index edb8c9bb620..e112539de54 100644 --- a/py/server/tests/test_docstring_parser.py +++ b/py/server/tests/test_docstring_parser.py @@ -112,4 +112,4 @@ def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword, "1, aaaaaa12=") def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa012=1)") + self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa12=1)") From 96f675332612c9f8f89157a46271d58a9a07c2af Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 18 Oct 2024 14:36:53 -0400 Subject: [PATCH 08/13] parse examples --- .../auto_completer/_signature_help.py | 55 +++++++++++-------- py/server/tests/data/signatures.py | 55 +++++++++++++++++++ py/server/tests/test_docstring_parser.py | 16 ++++++ 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 982982837c2..1ab52c8298b 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -75,18 +75,7 @@ def _get_params(signature: Signature, docs: Docstring) -> list[Any]: return params -def _get_raises(docs: Docstring) -> list[Any]: - return [dict(name=raise_.type_name, description=raise_.description) for raise_ in docs.raises] - - -def _get_returns(docs: Docstring) -> list[Any]: - return [dict(name=return_.type_name, description=return_.description) for return_ in docs.many_returns] - - def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: - raises = _get_raises(docs) - returns = _get_returns(docs) - if docs.description is None: description = "" else: @@ -109,22 +98,31 @@ def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: description += f"> {param['description']}".replace('\n\n', '\n\n> ') description += "\n\n" - if len(returns) > 0: + if len(docs.many_returns) > 0: description += "#### **Returns**\n\n" - for return_ in returns: - if return_["name"] is not None: - description += f"> **{return_['name']}** \n" - if return_["description"] is not None: - description += f"> {return_['description']}" + for return_ in docs.many_returns: + if return_.type_name is not None: + description += f"> **{return_.type_name}** \n" + if return_.description is not None: + description += f"> {return_.description}" description += "\n\n" - if len(raises) > 0: + if len(docs.raises) > 0: description += "#### **Raises**\n\n" - for raises_ in raises: - if raises_["name"] is not None: - description += f"> **{raises_['name']}** \n" - if raises_["description"] is not None: - description += f"> {raises_['description']}" + for raises_ in docs.raises: + if raises_.type_name is not None: + description += f"> **{raises_.type_name}** \n" + if raises_.description is not None: + description += f"> {raises_.description}" + description += "\n\n" + + if len(docs.examples) > 0: + description += "#### **Examples**\n\n" + for example in docs.examples: + if example.description is not None and example.description.startswith(">>> "): + description += f"```\n{example.description}\n```" + else: + description += example.description description += "\n\n" return description.strip() @@ -231,3 +229,14 @@ def _get_signature_result(signature: Signature) -> list[Any]: param_docs, signature.index if signature.index is not None else -1, ] + +def test(): + """ + dtest + + Returns: + a: asdas + + Examples: + >>> 123 + """ diff --git a/py/server/tests/data/signatures.py b/py/server/tests/data/signatures.py index 320613603f4..67bf5342814 100644 --- a/py/server/tests/data/signatures.py +++ b/py/server/tests/data/signatures.py @@ -147,6 +147,61 @@ def returns_various(): > **bar** > bar description""" +def example_string(): + """ + Description + + Examples: + Plain text + """ + +example_string_str = """ +def example_string(): + \"\"\" + Description + + Examples: + Plain text + \"\"\" +""" + +example_string_result = """\ +Description + +#### **Examples** + +Plain text""" + +def example_code(): + """ + Description + + Examples: + >>> Code + Still code + """ + +example_code_str = """ +def example_code(): + \"\"\" + Description + + Examples: + >>> Code + Still code + \"\"\" +""" + +example_code_result = """\ +Description + +#### **Examples** + +``` +>>> Code +Still code +```""" + def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): """ :returns a: b diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py index e112539de54..1e399403c82 100644 --- a/py/server/tests/test_docstring_parser.py +++ b/py/server/tests/test_docstring_parser.py @@ -66,6 +66,22 @@ def test_returns_various(self, signature: Signature, docs: Docstring): returns_various_result ) + @create_test("example_string", example_string_str, example_string) + def test_example_string(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + example_string_result + ) + + @create_test("example_code", example_code_str, example_code) + def test_example_code(self, signature: Signature, docs: Docstring): + self.assertNotEqual(len(docs.meta), 0) + self.assertEqual( + _generate_description_markdown(docs, _get_params(signature, docs)), + example_code_result + ) + @create_test("original_signature", original_signature_str, original_signature) def test_original_signature(self, signature: Signature, docs: Docstring): self.assertNotEqual(len(docs.meta), 0) From 3654825e4d0f55a8ec84c95968d4c5de868a6ed8 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 18 Oct 2024 14:53:41 -0400 Subject: [PATCH 09/13] clean and add comments --- .../auto_completer/_completer.py | 4 +-- .../auto_completer/_signature_help.py | 27 ++++++++++--------- py/server/tests/test_docstring_parser.py | 9 +++++++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index 9a284c7bfd3..e6a5d5f292b 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -6,7 +6,7 @@ from typing import Any, Union, List from jedi import Interpreter, Script from jedi.api.classes import Completion, Signature -from ._signature_help import _get_signature_result +from ._signature_help import _get_signature_help import sys import warnings @@ -216,7 +216,7 @@ def do_signature_help( # keep checking the latest version as we run, so updated doc can cancel us if not self._versions[uri] == version: return [] - results.append(_get_signature_result(signature)) + results.append(_get_signature_help(signature)) return results diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 1ab52c8298b..d5fb76a9121 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -129,6 +129,12 @@ def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: def _generate_display_sig(signature: Signature) -> str: + """ + Generate the signature text for the signature help. Truncates the signature if it is too long. If the current + argument is positional, it will display the next 2 arguments. If the current argument is keyword, it will only + display the current argument. + """ + if len(signature.to_string()) <= MAX_DISPLAY_SIG_LEN: return signature.to_string() @@ -161,6 +167,10 @@ def _generate_display_sig(signature: Signature) -> str: def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[Any]: + """ + Generate markdown for each parameter in the signature. This will be shown on top of the description markdown. + """ + param_docs = [] for i in range(len(signature.params)): if signature.params[i].to_string().strip() in IGNORE_PARAM_NAMES: @@ -180,9 +190,12 @@ def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[A return param_docs -def _get_signature_result(signature: Signature) -> list[Any]: +def _get_signature_help(signature: Signature) -> list[Any]: """ Gets the result of a signature to be used by `do_signature_help` + If no docstring information is parsed, then the signature will be displayed in Markdown but with plaintext style + whitespace. Any cached docstring must have some docstring information. + Returns: A list that contains [signature name, docstring, param docstrings, index] """ @@ -209,6 +222,7 @@ def _get_signature_result(signature: Signature) -> list[Any]: if len(docs.meta) == 0: return [ signature.to_string(), + # Since signature is a markdown, replace whitespace in a way to preserve how it originally looks signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), [[param.to_string().strip(), ""] for param in signature.params], signature.index if signature.index is not None else -1, @@ -229,14 +243,3 @@ def _get_signature_result(signature: Signature) -> list[Any]: param_docs, signature.index if signature.index is not None else -1, ] - -def test(): - """ - dtest - - Returns: - a: asdas - - Examples: - >>> 123 - """ diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py index 1e399403c82..d9ef30e63be 100644 --- a/py/server/tests/test_docstring_parser.py +++ b/py/server/tests/test_docstring_parser.py @@ -18,6 +18,15 @@ class DocstringParser(BaseTestCase): @staticmethod def create_test(name: str, code: str, func: Callable, func_call_append: str = ""): + """ + Wraps an autocomplete test to run it in a both Jedi Script and Interpreter. + + Args: + name: the name of the function being autocompleted + code: the string version of the function, for Jedi Script + func: the function object, for Jedi Interpreter + func_call_append: the string to append at the end of the function call + """ def decorator(f): @wraps(f) def wrapper(self): From 54fb6b7d61d9fd1432ba3298b1cf80843dd7c9ce Mon Sep 17 00:00:00 2001 From: mikebender Date: Fri, 7 Feb 2025 14:01:09 -0500 Subject: [PATCH 10/13] Clean up the tests for docstrings - Rather than duplicating the code, using inspect.get_source you can get the source code. Makes the tests much easier - Moved everything into one file, a little bit easier to follow. --- .../auto_completer/_signature_help.py | 2 +- py/server/tests/data/signatures.py | 243 ------------- py/server/tests/test_docstring_parser.py | 323 ++++++++++++------ 3 files changed, 213 insertions(+), 355 deletions(-) delete mode 100644 py/server/tests/data/signatures.py diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index d5fb76a9121..941d1314951 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending # from __future__ import annotations from inspect import Parameter diff --git a/py/server/tests/data/signatures.py b/py/server/tests/data/signatures.py deleted file mode 100644 index 67bf5342814..00000000000 --- a/py/server/tests/data/signatures.py +++ /dev/null @@ -1,243 +0,0 @@ -# -# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -# -def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): - """ - Description - - Args: - has_docs: Arg has docs - has_type: Arg has type - not_real: Arg does not exist in signature - *positional: Positional arg has docs - has_default: Arg has default - has_type_default: Arg has type and default - **keyword: Keyword arg has docs - """ - -args_str = """ -def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): - \"\"\" - Description - - Args: - has_docs: Arg has docs - has_type: Arg has type - not_real: Arg does not exist in signature - *positional: Positional arg has docs - has_default: Arg has default - has_type_default: Arg has type and default - **keyword: Keyword arg has docs - \"\"\" -""" - -args_str_result = """\ -Description - -#### **Parameters** - -> **has_docs** -> Arg has docs - -> **has_type**: *str | int* -> Arg has type - -> ***positional** -> Positional arg has docs - -> **has_default** ⋅ (default: *1*) -> Arg has default - -> **has_type_default**: *str | int* ⋅ (default: *1*) -> Arg has type and default - -> ****keyword** -> Keyword arg has docs""" - -def args_no_docs(no_docs, /, *, keyword_no_docs=None): - """ - Description - - Args: - not_real: Arg does not exist in signature - /: Should not show - *: Should not show - """ - -args_no_docs_str = """ -def args_no_docs(no_docs, /, *, keyword_no_docs=None): - \"\"\" - Description - - Args: - not_real: Arg does not exist in signature - /: Should not show - *: Should not show - \"\"\" -""" - -args_no_docs_result = """\ -Description - -#### **Parameters** - -> **no_docs** - - -> **keyword_no_docs** ⋅ (default: *None*)""" - -def raises_various(): - """ - Description - - Raises: - Exception: Exception description - ValueError: ValueError description. - This is a continuation of ValueError - """ - -raises_various_str = """ -def raises_various(): - \"\"\" - Description - - Raises: - Exception: Exception description - ValueError: ValueError description. - This is a continuation of ValueError - \"\"\" -""" - -raises_various_result = """\ -Description - -#### **Raises** - -> **Exception** -> Exception description - -> **ValueError** -> ValueError description. -This is a continuation of ValueError""" - -def returns_various(): - """ - :returns: Return has docs - :returns foo: foo description - :returns bar: bar description - """ - -returns_various_str = """ -def returns_various(): - \"\"\" - :returns: Return has docs - :returns foo: foo description - :returns bar: bar description - \"\"\" -""" - -returns_various_result = """\ -#### **Returns** - -> Return has docs - -> **foo** -> foo description - -> **bar** -> bar description""" - -def example_string(): - """ - Description - - Examples: - Plain text - """ - -example_string_str = """ -def example_string(): - \"\"\" - Description - - Examples: - Plain text - \"\"\" -""" - -example_string_result = """\ -Description - -#### **Examples** - -Plain text""" - -def example_code(): - """ - Description - - Examples: - >>> Code - Still code - """ - -example_code_str = """ -def example_code(): - \"\"\" - Description - - Examples: - >>> Code - Still code - \"\"\" -""" - -example_code_result = """\ -Description - -#### **Examples** - -``` ->>> Code -Still code -```""" - -def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): - """ - :returns a: b - """ - -original_signature_str = """ -def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): - \"\"\" - :returns a: b - \"\"\" -""" - -def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, - aaaaaa10, aaaaaa11, aaaaaa12): - """ - :returns a: b - """ - -truncate_positional_str = """ -def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, - aaaaaa10, aaaaaa11, aaaaaa12): - \"\"\" - :returns a: b - \"\"\" -""" - -def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, - aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): - """ - :returns a: b - """ - -truncate_keyword_str = """ -def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, - aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): - \"\"\" - :returns a: b - \"\"\" -""" diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py index d9ef30e63be..84b0e6f90c0 100644 --- a/py/server/tests/test_docstring_parser.py +++ b/py/server/tests/test_docstring_parser.py @@ -1,140 +1,241 @@ # -# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending # -from functools import wraps +import inspect from typing import Callable from docstring_parser import parse, Docstring from jedi import Script, Interpreter -from jedi.api.classes import Signature from deephaven_internal.auto_completer._signature_help import _get_params, _generate_description_markdown, _generate_display_sig from tests.testbase import BaseTestCase -from .data.signatures import * - class DocstringParser(BaseTestCase): - @staticmethod - def create_test(name: str, code: str, func: Callable, func_call_append: str = ""): + def get_script_signature(self, func: Callable, func_call_append=""): """ - Wraps an autocomplete test to run it in a both Jedi Script and Interpreter. + Get the signature of the function passed in. using Jedi Script. Args: - name: the name of the function being autocompleted - code: the string version of the function, for Jedi Script - func: the function object, for Jedi Interpreter + func: the function object. Will be used with Jedi Interpreter, and source used for Jedi Script func_call_append: the string to append at the end of the function call """ - def decorator(f): - @wraps(f) - def wrapper(self): - s = Script(f"{code}\n{name}({func_call_append}").get_signatures() - self.assertIsInstance(s, list) - self.assertEqual(len(s), 1) - f(self, s[0], parse(s[0].docstring(raw=True))) - - i = Interpreter(f"{name}({func_call_append}", [{name: func}]).get_signatures() - self.assertIsInstance(s, list) - self.assertEqual(len(s), 1) - f(self, i[0], parse(i[0].docstring(raw=True))) - - return wrapper - return decorator - - @create_test("args", args_str, args) - def test_args(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - args_str_result - ) + code = inspect.getsource(func) + s = Script(f"{code}\n{func.__name__}({func_call_append}").get_signatures() + self.assertIsInstance(s, list) + self.assertEqual(len(s), 1) + return s[0] + + def get_interpreter_signature(self, func: Callable, func_call_append=""): + i = Interpreter(f"{func.__name__}({func_call_append}", [{func.__name__: func}]).get_signatures() + self.assertIsInstance(i, list) + self.assertEqual(len(i), 1) + return i[0] + + def expect_description(self, func: Callable, expected_result: str, func_call_append =""): + """ + Test whether the function passed in results in the expected markdown docs. Tests both interpreter and script. - @create_test("args_no_docs", args_no_docs_str, args_no_docs) - def test_args_no_docs(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) + Args: + func: the function object. Will be used with Jedi Interpreter, and source used for Jedi Script + expected_result: the expected markdown result + func_call_append: the string to append at the end of the function call + """ + script_signature = self.get_script_signature(func, func_call_append) + script_docstring = script_signature.docstring(raw=True) self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - args_no_docs_result + _generate_description_markdown(parse(script_docstring), _get_params(script_signature, parse(script_docstring))), + expected_result ) - @create_test("raises_various", raises_various_str, raises_various) - def test_raises_various(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) + interpreter_signature = self.get_interpreter_signature(func, func_call_append) + interpreter_docstring = interpreter_signature.docstring(raw=True) self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - raises_various_result + # Need to use _generate_display_sig for the original_signature ones, not this method... grr. + _generate_description_markdown(parse(interpreter_docstring), _get_params(interpreter_signature, parse(interpreter_docstring))), + expected_result ) - @create_test("returns_various", returns_various_str, returns_various) - def test_returns_various(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - returns_various_result - ) + def expect_signature(self, func: Callable, expected_result: str, func_call_append = ""): + """ + Test whether the function passed in results in the expected signature. Tests both interpreter and script. - @create_test("example_string", example_string_str, example_string) - def test_example_string(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - example_string_result - ) + Args: + func: the function object. Will be used with Jedi Interpreter, and source used for Jedi Script + expected_result: the expected signature result + func_call_append: the string to append at the end of the function call + """ + script_signature = self.get_script_signature(func, func_call_append) + self.assertEqual(_generate_display_sig(script_signature), expected_result) - @create_test("example_code", example_code_str, example_code) - def test_example_code(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual( - _generate_description_markdown(docs, _get_params(signature, docs)), - example_code_result - ) + interpreter_signature = self.get_interpreter_signature(func, func_call_append) + self.assertEqual(_generate_display_sig(interpreter_signature), expected_result) - @create_test("original_signature", original_signature_str, original_signature) - def test_original_signature(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual( - _generate_display_sig(signature), - "original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09)" - ) + def test_args(self): + def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): + """ + Description + + Args: + has_docs: Arg has docs + has_type: Arg has type + not_real: Arg does not exist in signature + *positional: Positional arg has docs + has_default: Arg has default + has_type_default: Arg has type and default + **keyword: Keyword arg has docs + """ + + self.expect_description(args, """\ +Description + +#### **Parameters** + +> **has_docs** +> Arg has docs + +> **has_type**: *str | int* +> Arg has type + +> ***positional** +> Positional arg has docs + +> **has_default** ⋅ (default: *1*) +> Arg has default + +> **has_type_default**: *str | int* ⋅ (default: *1*) +> Arg has type and default + +> ****keyword** +> Keyword arg has docs""") + + def test_args_no_docs(self): + def args_no_docs(no_docs, /, *, keyword_no_docs=None): + """ + Description + + Args: + not_real: Arg does not exist in signature + /: Should not show + *: Should not show + """ + + self.expect_description(args_no_docs, """\ +Description + +#### **Parameters** + +> **no_docs** + + +> **keyword_no_docs** ⋅ (default: *None*)""") + + def test_raises_various(self): + def raises_various(): + """ + Description + + Raises: + Exception: Exception description + ValueError: ValueError description. + This is a continuation of ValueError + """ + + self.expect_description(raises_various, """\ +Description + +#### **Raises** + +> **Exception** +> Exception description + +> **ValueError** +> ValueError description. +This is a continuation of ValueError""") + + def test_returns_various(self): + def returns_various(): + """ + :returns: Return has docs + :returns foo: foo description + :returns bar: bar description + """ + + self.expect_description(returns_various, """\ +#### **Returns** + +> Return has docs + +> **foo** +> foo description + +> **bar** +> bar description""") + + def test_example_string(self): + def example_string(): + """ + Description + + Examples: + Plain text + """ + + self.expect_description(example_string, """\ +Description + +#### **Examples** + +Plain text""") + + def test_example_code(self): + def example_code(): + """ + Description + + Examples: + >>> Code + Still code + """ + + self.expect_description(example_code, """\ +Description + +#### **Examples** + +``` +>>> Code +Still code +```""") + + def test_original_signature(self): + def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): + """ + :returns a: b + """ + + self.expect_signature(original_signature, "original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09)") + + def test_truncate_positional(self): + def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, + aaaaaa10, aaaaaa11, aaaaaa12): + """ + :returns a: b + """ + + self.expect_signature(truncate_positional, "truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, ...)") + self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa01, aaaaaa02, aaaaaa03, ...)", "1, ") + self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)", "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") + self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)", "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") + + def test_truncate_keyword(self): + def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, + aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): + """ + :returns a: b + """ - @create_test("truncate_positional", truncate_positional_str, truncate_positional) - def test_truncate_positional_0(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, ...)") - - @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, ") - def test_truncate_positional_1(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa01, aaaaaa02, aaaaaa03, ...)") - - @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") - def test_truncate_positional_10(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") - - @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") - def test_truncate_positional_11(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") - - @create_test("truncate_positional", truncate_positional_str, truncate_positional, "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") - def test_truncate_positional_12(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)") - - @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword) - def test_truncate_keyword_0(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_keyword(aaaaaa00, aaaaaa01=1, ...)") - - @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword, "1, ") - def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa01=1, ...)") - - @create_test("truncate_keyword", truncate_keyword_str, truncate_keyword, "1, aaaaaa12=") - def test_truncate_keyword_1(self, signature: Signature, docs: Docstring): - self.assertNotEqual(len(docs.meta), 0) - self.assertEqual(_generate_display_sig(signature), "truncate_keyword(..., aaaaaa12=1)") + self.expect_signature(truncate_keyword, "truncate_keyword(aaaaaa00, aaaaaa01=1, ...)") + self.expect_signature(truncate_keyword, "truncate_keyword(..., aaaaaa01=1, ...)", "1, ") + self.expect_signature(truncate_keyword, "truncate_keyword(..., aaaaaa12=1)", "1, aaaaaa12=") \ No newline at end of file From 1d004370f6caff067af99faa378763b898e71a28 Mon Sep 17 00:00:00 2001 From: mikebender Date: Fri, 14 Feb 2025 13:29:33 -0500 Subject: [PATCH 11/13] Clean up some more based on review comments --- .../auto_completer/_completer.py | 4 +- .../auto_completer/_signature_help.py | 84 ++++++++++++++----- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index 7549c00309a..cdd74a5bc7e 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -6,7 +6,7 @@ from typing import Any, Union, List from jedi import Interpreter, Script from jedi.api.classes import Completion, Signature -from ._signature_help import _get_signature_help +from ._signature_help import get_signature_help import sys import warnings @@ -216,7 +216,7 @@ def do_signature_help( # keep checking the latest version as we run, so updated doc can cancel us if not self._versions[uri] == version: return [] - results.append(_get_signature_help(signature)) + results.append(get_signature_help(signature)) return results diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 941d1314951..19b9ebf02aa 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -3,30 +3,54 @@ # from __future__ import annotations from inspect import Parameter -from typing import Any +from typing import Any, TypedDict, Union from docstring_parser import parse, Docstring from jedi.api.classes import Signature -IGNORE_PARAM_NAMES = ("", "/", "*") -MAX_DISPLAY_SIG_LEN = 128 # 3 lines is 150, but there could be overflow so 150 could result in 4 lines -POSITIONAL_KINDS = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD, Parameter.VAR_POSITIONAL) +_IGNORE_PARAM_NAMES = ("", "/", "*") +_MAX_DISPLAY_SIG_LEN = 128 # 3 lines is 150, but there could be overflow so 150 could result in 4 lines +_POSITIONAL_KINDS = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD, Parameter.VAR_POSITIONAL) # key: result from _hash # value: another dictionary that has the following keys: # description: The markdown description (result from _generate_description_markdown) # param_docs: A list of param markdown descriptions (result from _generate_param_markdowns) -result_cache = {} +_result_cache = {} +class ParameterDetails(TypedDict): + """ + Details of a parameter of a function + """ + name: str + """ + Name of the parameter + """ + + description: str + """ + Description of the parameter + """ + + type: Union[str, None] + """ + Type of the parameter + """ -def _hash(signature: Signature) -> str: - """A simple way to identify signatures""" - return f"{signature.to_string()}\n{signature.docstring(raw=True)}" + default: Union[str, None] + """ + Default value of the parameter + """ -def _get_params(signature: Signature, docs: Docstring) -> list[Any]: +def _get_params(signature: Signature, docs: Docstring) -> list[ParameterDetails]: """ - Combines all available parameter information from the signature and docstring. + Returns all available parameter information from the signature and docstring. + + Combines information from the docstring and signature. + Uses the type in the signature if it exists, falls back to the type in the docstring if that exists. + Also includes the default value if it exists in the signature. + Args: signature: The signature from `jedi` @@ -75,7 +99,18 @@ def _get_params(signature: Signature, docs: Docstring) -> list[Any]: return params -def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: +def _generate_description_markdown(docs: Docstring, params: list[ParameterDetails]) -> str: + """ + Generate the description markdown for the signature help. This includes the description, parameters, returns, raises, + and examples. + + Args: + docs: The parsed docstring from `docstring_parser` + params: The list of parameters from `_get_params` + + Returns: + The markdown description + """ if docs.description is None: description = "" else: @@ -84,7 +119,7 @@ def _generate_description_markdown(docs: Docstring, params: list[Any]) -> str: if len(params) > 0: description += "#### **Parameters**\n\n" for param in params: - if param['name'] in IGNORE_PARAM_NAMES: + if param['name'] in _IGNORE_PARAM_NAMES: continue description += f"> **{param['name']}**" @@ -135,7 +170,7 @@ def _generate_display_sig(signature: Signature) -> str: display the current argument. """ - if len(signature.to_string()) <= MAX_DISPLAY_SIG_LEN: + if len(signature.to_string()) <= _MAX_DISPLAY_SIG_LEN: return signature.to_string() # Use 0 as default to display start of signature @@ -147,13 +182,13 @@ def _generate_display_sig(signature: Signature) -> str: # If current arg is positional, display next 2 args # If current arg is keyword, only display current args - if signature.params[index].kind in POSITIONAL_KINDS: + if signature.params[index].kind in _POSITIONAL_KINDS: # Clamp index so that 3 args are shown, even at last index index = max(min(index, len(signature.params) - 3), 0) end_index = index + 3 # If the next arg is not positional, do not show the one after it # Otherwise, this arg will show 2 ahead, and then next arg will show 0 ahead - if signature.params[index + 1].kind not in POSITIONAL_KINDS: + if signature.params[index + 1].kind not in _POSITIONAL_KINDS: end_index -= 1 display_sig += ", ".join([param.to_string() for param in signature.params[index:end_index]]) if index + 3 < len(signature.params): @@ -173,7 +208,7 @@ def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[A param_docs = [] for i in range(len(signature.params)): - if signature.params[i].to_string().strip() in IGNORE_PARAM_NAMES: + if signature.params[i].to_string().strip() in _IGNORE_PARAM_NAMES: continue param = params[i] @@ -190,7 +225,7 @@ def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[A return param_docs -def _get_signature_help(signature: Signature) -> list[Any]: +def get_signature_help(signature: Signature) -> list[Any]: """ Gets the result of a signature to be used by `do_signature_help` If no docstring information is parsed, then the signature will be displayed in Markdown but with plaintext style @@ -201,11 +236,15 @@ def _get_signature_help(signature: Signature) -> list[Any]: """ docstring = signature.docstring(raw=True) - cache_key = _hash(signature) + + # The results are cached based on the signature and docstring + # Even if there is another method in another package with the same signature, + # if the docstring also matches it's fine to use the same cache, as the result will be the same. + cache_key = f"{signature.to_string()}\n{docstring}" # Check cache - if cache_key in result_cache: - result = result_cache[cache_key] + if cache_key in _result_cache: + result = _result_cache[cache_key] return [ _generate_display_sig(signature), result["description"], @@ -213,6 +252,7 @@ def _get_signature_help(signature: Signature) -> list[Any]: signature.index if signature.index is not None else -1, ] + # Parse the docstring to extract information docs = parse(docstring) # Nothing parsed, revert to plaintext @@ -223,7 +263,7 @@ def _get_signature_help(signature: Signature) -> list[Any]: return [ signature.to_string(), # Since signature is a markdown, replace whitespace in a way to preserve how it originally looks - signature.docstring(raw=True).replace(" ", " ").replace("\n", " \n"), + docstring.replace(" ", " ").replace("\n", " \n"), [[param.to_string().strip(), ""] for param in signature.params], signature.index if signature.index is not None else -1, ] @@ -232,7 +272,7 @@ def _get_signature_help(signature: Signature) -> list[Any]: params = _get_params(signature, docs) description = _generate_description_markdown(docs, params) param_docs = _generate_param_markdowns(signature, params) - result_cache[cache_key] = { + _result_cache[cache_key] = { "description": description, "param_docs": param_docs, } From 529e297b47d24c2333163109044a7cadc372046c Mon Sep 17 00:00:00 2001 From: mikebender Date: Fri, 14 Feb 2025 13:45:43 -0500 Subject: [PATCH 12/13] Remove the truncation logic - It was kind of annoying, would rather see all the parameters. That's what VS Code does --- .../auto_completer/_signature_help.py | 44 +--------------- py/server/tests/test_docstring_parser.py | 51 +------------------ 2 files changed, 4 insertions(+), 91 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index 19b9ebf02aa..e1832a5d6ab 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -9,7 +9,6 @@ _IGNORE_PARAM_NAMES = ("", "/", "*") -_MAX_DISPLAY_SIG_LEN = 128 # 3 lines is 150, but there could be overflow so 150 could result in 4 lines _POSITIONAL_KINDS = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD, Parameter.VAR_POSITIONAL) # key: result from _hash @@ -162,45 +161,6 @@ def _generate_description_markdown(docs: Docstring, params: list[ParameterDetail return description.strip() - -def _generate_display_sig(signature: Signature) -> str: - """ - Generate the signature text for the signature help. Truncates the signature if it is too long. If the current - argument is positional, it will display the next 2 arguments. If the current argument is keyword, it will only - display the current argument. - """ - - if len(signature.to_string()) <= _MAX_DISPLAY_SIG_LEN: - return signature.to_string() - - # Use 0 as default to display start of signature - index = signature.index if signature.index is not None else 0 - display_sig = f"{signature.name}(" - - if index > 0: - display_sig += "..., " - - # If current arg is positional, display next 2 args - # If current arg is keyword, only display current args - if signature.params[index].kind in _POSITIONAL_KINDS: - # Clamp index so that 3 args are shown, even at last index - index = max(min(index, len(signature.params) - 3), 0) - end_index = index + 3 - # If the next arg is not positional, do not show the one after it - # Otherwise, this arg will show 2 ahead, and then next arg will show 0 ahead - if signature.params[index + 1].kind not in _POSITIONAL_KINDS: - end_index -= 1 - display_sig += ", ".join([param.to_string() for param in signature.params[index:end_index]]) - if index + 3 < len(signature.params): - display_sig += ", ..." - else: - display_sig += signature.params[index].to_string() - if index + 1 < len(signature.params): - display_sig += ", ..." - - return display_sig + ")" - - def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[Any]: """ Generate markdown for each parameter in the signature. This will be shown on top of the description markdown. @@ -246,7 +206,7 @@ def get_signature_help(signature: Signature) -> list[Any]: if cache_key in _result_cache: result = _result_cache[cache_key] return [ - _generate_display_sig(signature), + signature.to_string(), result["description"], result["param_docs"], signature.index if signature.index is not None else -1, @@ -278,7 +238,7 @@ def get_signature_help(signature: Signature) -> list[Any]: } return [ - _generate_display_sig(signature), + signature.to_string(), description, param_docs, signature.index if signature.index is not None else -1, diff --git a/py/server/tests/test_docstring_parser.py b/py/server/tests/test_docstring_parser.py index 84b0e6f90c0..c333dbf358e 100644 --- a/py/server/tests/test_docstring_parser.py +++ b/py/server/tests/test_docstring_parser.py @@ -7,7 +7,7 @@ from docstring_parser import parse, Docstring from jedi import Script, Interpreter -from deephaven_internal.auto_completer._signature_help import _get_params, _generate_description_markdown, _generate_display_sig +from deephaven_internal.auto_completer._signature_help import _get_params, _generate_description_markdown from tests.testbase import BaseTestCase @@ -52,26 +52,10 @@ def expect_description(self, func: Callable, expected_result: str, func_call_app interpreter_signature = self.get_interpreter_signature(func, func_call_append) interpreter_docstring = interpreter_signature.docstring(raw=True) self.assertEqual( - # Need to use _generate_display_sig for the original_signature ones, not this method... grr. _generate_description_markdown(parse(interpreter_docstring), _get_params(interpreter_signature, parse(interpreter_docstring))), expected_result ) - def expect_signature(self, func: Callable, expected_result: str, func_call_append = ""): - """ - Test whether the function passed in results in the expected signature. Tests both interpreter and script. - - Args: - func: the function object. Will be used with Jedi Interpreter, and source used for Jedi Script - expected_result: the expected signature result - func_call_append: the string to append at the end of the function call - """ - script_signature = self.get_script_signature(func, func_call_append) - self.assertEqual(_generate_display_sig(script_signature), expected_result) - - interpreter_signature = self.get_interpreter_signature(func, func_call_append) - self.assertEqual(_generate_display_sig(interpreter_signature), expected_result) - def test_args(self): def args(has_docs, has_type: str | int, *positional, has_default=1, has_type_default: str | int = 1, **keyword): """ @@ -207,35 +191,4 @@ def example_code(): ``` >>> Code Still code -```""") - - def test_original_signature(self): - def original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09): - """ - :returns a: b - """ - - self.expect_signature(original_signature, "original_signature(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09)") - - def test_truncate_positional(self): - def truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, aaaaaa03, aaaaaa04, aaaaaa05, aaaaaa06, aaaaaa07, aaaaaa08, aaaaaa09, - aaaaaa10, aaaaaa11, aaaaaa12): - """ - :returns a: b - """ - - self.expect_signature(truncate_positional, "truncate_positional(aaaaaa00, aaaaaa01, aaaaaa02, ...)") - self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa01, aaaaaa02, aaaaaa03, ...)", "1, ") - self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)", "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") - self.expect_signature(truncate_positional, "truncate_positional(..., aaaaaa10, aaaaaa11, aaaaaa12)", "1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ") - - def test_truncate_keyword(self): - def truncate_keyword(aaaaaa00, *, aaaaaa01=1, aaaaaa02=1, aaaaaa03=1, aaaaaa04=1, aaaaaa05=1, aaaaaa06=1, aaaaaa07=1, aaaaaa08=1, aaaaaa09=1, - aaaaaa10=1, aaaaaa11=1, aaaaaa12=1): - """ - :returns a: b - """ - - self.expect_signature(truncate_keyword, "truncate_keyword(aaaaaa00, aaaaaa01=1, ...)") - self.expect_signature(truncate_keyword, "truncate_keyword(..., aaaaaa01=1, ...)", "1, ") - self.expect_signature(truncate_keyword, "truncate_keyword(..., aaaaaa12=1)", "1, aaaaaa12=") \ No newline at end of file +```""") \ No newline at end of file From 45e5f96e12cc15311054a818f6f67e8a36738129 Mon Sep 17 00:00:00 2001 From: mikebender Date: Fri, 14 Feb 2025 13:57:28 -0500 Subject: [PATCH 13/13] Clean up based on self-review --- py/server/deephaven_internal/auto_completer/_completer.py | 2 -- .../deephaven_internal/auto_completer/_signature_help.py | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/py/server/deephaven_internal/auto_completer/_completer.py b/py/server/deephaven_internal/auto_completer/_completer.py index cdd74a5bc7e..2589d1bcdb8 100644 --- a/py/server/deephaven_internal/auto_completer/_completer.py +++ b/py/server/deephaven_internal/auto_completer/_completer.py @@ -78,8 +78,6 @@ class Completer: def __init__(self): self._docs = {} self._versions = {} - # Cache for signature markdown - self.signature_cache = {} # we will replace this w/ top-level globals() when we open the document self.__scope = globals() # might want to make this a {uri: []} instead of [] diff --git a/py/server/deephaven_internal/auto_completer/_signature_help.py b/py/server/deephaven_internal/auto_completer/_signature_help.py index e1832a5d6ab..b2be2b0fad2 100644 --- a/py/server/deephaven_internal/auto_completer/_signature_help.py +++ b/py/server/deephaven_internal/auto_completer/_signature_help.py @@ -164,6 +164,13 @@ def _generate_description_markdown(docs: Docstring, params: list[ParameterDetail def _generate_param_markdowns(signature: Signature, params: list[Any]) -> list[Any]: """ Generate markdown for each parameter in the signature. This will be shown on top of the description markdown. + + Args: + signature: The signature from `jedi` + params: The list of parameters from `_get_params` + + Returns: + A list of signature names and description pairs. """ param_docs = []