From 596b2273aa40afdbee5fdab5709e28096185fb70 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 2 Feb 2024 09:18:07 +0100 Subject: [PATCH 01/11] Reintroduce tree-sitter-languages. It seem to be active again, with proper builds, and a newer version of tree-sitter-rst. It allow to not have to do the build-parser step --- .github/workflows/python-package.yml | 11 ++++++----- papyri/ts.py | 28 ++++++++++++++++++---------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 08464717..b20dd02e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -47,11 +47,12 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt flit install --symlink - git clone https://github.com/stsewd/tree-sitter-rst - cd tree-sitter-rst - git checkout 3fc88d2097bc854ab6bf70c52b3c482849cf8e8f - cd - - papyri build-parser + # this is not needed if tree-sitter-languages works + #git clone https://github.com/stsewd/tree-sitter-rst + #cd tree-sitter-rst + #git checkout 3fc88d2097bc854ab6bf70c52b3c482849cf8e8f + #cd - + #papyri build-parser - name: dependency tree run: | pipdeptree diff --git a/papyri/ts.py b/papyri/ts.py index 74283f2c..ec1427d9 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -4,7 +4,6 @@ from textwrap import dedent, indent from typing import List, Any, Dict -from tree_sitter import Language, Parser from .myst_ast import ( MText, @@ -46,16 +45,25 @@ allowed_adorn = "=-`:.'\"~^_*+#<>" pth = str(Path(__file__).parent / "rst.so") -# replace by tree-sitter-languages once it works See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 try: - RST = Language(pth, "rst") -except OSError as e: - raise OSError( - "tree-sitter-rst not found, rst parsing will not work. Please run `papyri build-parser`" - ) from e - -parser = Parser() -parser.set_language(RST) + from tree_sitter_languages import get_language, get_parser + + # language = get_language('python') + parser = get_parser("rst") +except ModuleNotFoundError: + # replace by tree-sitter-languages once it works See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 + try: + from tree_sitter import Language, Parser + + RST = Language(pth, "rst") + except OSError as e: + raise OSError( + "tree-sitter-rst not found, rst parsing will not work. Please run `papyri build-parser`" + ) from e + + parser = Parser() + parser.set_language(RST) + log = logging.getLogger("papyri") From 0794a8510097eac2b5f81c7b1190b681050ecf12 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 14:58:11 +0100 Subject: [PATCH 02/11] nopth --- papyri/ts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papyri/ts.py b/papyri/ts.py index ec1427d9..498e32bd 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -43,7 +43,6 @@ ) allowed_adorn = "=-`:.'\"~^_*+#<>" -pth = str(Path(__file__).parent / "rst.so") try: from tree_sitter_languages import get_language, get_parser @@ -55,6 +54,7 @@ try: from tree_sitter import Language, Parser + pth = str(Path(__file__).parent / "rst.so") RST = Language(pth, "rst") except OSError as e: raise OSError( From 5c21eb1fbeccb0010332025c519318ac60bfe199 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 14:58:34 +0100 Subject: [PATCH 03/11] nuke sub --- tree-sitter-rst | 1 - 1 file changed, 1 deletion(-) delete mode 160000 tree-sitter-rst diff --git a/tree-sitter-rst b/tree-sitter-rst deleted file mode 160000 index 3fc88d20..00000000 --- a/tree-sitter-rst +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3fc88d2097bc854ab6bf70c52b3c482849cf8e8f From 7633537092c6c314848f02ce9247df40614f4acc Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:11:21 +0100 Subject: [PATCH 04/11] refine --- papyri/ts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/papyri/ts.py b/papyri/ts.py index 498e32bd..e4a04aab 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -45,11 +45,12 @@ allowed_adorn = "=-`:.'\"~^_*+#<>" try: - from tree_sitter_languages import get_language, get_parser + from tree_sitter_languages import get_parser # language = get_language('python') parser = get_parser("rst") except ModuleNotFoundError: + assert False # replace by tree-sitter-languages once it works See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 try: from tree_sitter import Language, Parser From bd4adb4184c82d97d62079e139cea278adaf4f8f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:11:56 +0100 Subject: [PATCH 05/11] remove tree_sitter --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 850166bf..8263eef8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ pytest-cov flit matplotlib pytest-trio -tree_sitter coverage jinja2==3.1.3 pipdeptree From eb185bf11da99939f6ada92b24b89111e29a523e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:14:41 +0100 Subject: [PATCH 06/11] another test --- .github/workflows/python-package.yml | 1 + example.py | 3 +++ requirements-dev.txt | 1 + 3 files changed, 5 insertions(+) create mode 100644 example.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b20dd02e..7b7d7b49 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -46,6 +46,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt + python examples.py flit install --symlink # this is not needed if tree-sitter-languages works #git clone https://github.com/stsewd/tree-sitter-rst diff --git a/example.py b/example.py new file mode 100644 index 00000000..9697ac91 --- /dev/null +++ b/example.py @@ -0,0 +1,3 @@ +from tree_sitter_languages import get_parser + +print(get_parser("rst")) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8263eef8..76fc7597 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,3 +10,4 @@ black mypy flake8<8 scipy +tree_sitter_languages From a9942db06a9439cb42e0db6b43a5bd69f85df8e0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:15:20 +0100 Subject: [PATCH 07/11] pyflake --- papyri/ts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/papyri/ts.py b/papyri/ts.py index e4a04aab..de3fdf73 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -50,8 +50,8 @@ # language = get_language('python') parser = get_parser("rst") except ModuleNotFoundError: - assert False - # replace by tree-sitter-languages once it works See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 + # replace by tree-sitter-languages once it works + # See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 try: from tree_sitter import Language, Parser From f86c80ca06d75401a94e770cbf4fb19853047605 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:15:24 +0100 Subject: [PATCH 08/11] Revert "another test" This reverts commit eb185bf11da99939f6ada92b24b89111e29a523e. --- .github/workflows/python-package.yml | 1 - example.py | 3 --- requirements-dev.txt | 1 - 3 files changed, 5 deletions(-) delete mode 100644 example.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7b7d7b49..b20dd02e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -46,7 +46,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - python examples.py flit install --symlink # this is not needed if tree-sitter-languages works #git clone https://github.com/stsewd/tree-sitter-rst diff --git a/example.py b/example.py deleted file mode 100644 index 9697ac91..00000000 --- a/example.py +++ /dev/null @@ -1,3 +0,0 @@ -from tree_sitter_languages import get_parser - -print(get_parser("rst")) diff --git a/requirements-dev.txt b/requirements-dev.txt index 76fc7597..8263eef8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,4 +10,3 @@ black mypy flake8<8 scipy -tree_sitter_languages From 2a83f2b018faf9763694ecaed9612706d203252d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:16:34 +0100 Subject: [PATCH 09/11] always tsl --- papyri/ts.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/papyri/ts.py b/papyri/ts.py index de3fdf73..4b8296b9 100644 --- a/papyri/ts.py +++ b/papyri/ts.py @@ -1,6 +1,5 @@ import logging import itertools -from pathlib import Path from textwrap import dedent, indent from typing import List, Any, Dict @@ -42,28 +41,11 @@ # VisitSubstitutionDefinitionNotImplementedError, ) -allowed_adorn = "=-`:.'\"~^_*+#<>" -try: - from tree_sitter_languages import get_parser - - # language = get_language('python') - parser = get_parser("rst") -except ModuleNotFoundError: - # replace by tree-sitter-languages once it works - # See https://github.com/grantjenks/py-tree-sitter-languages/issues/15 - try: - from tree_sitter import Language, Parser - - pth = str(Path(__file__).parent / "rst.so") - RST = Language(pth, "rst") - except OSError as e: - raise OSError( - "tree-sitter-rst not found, rst parsing will not work. Please run `papyri build-parser`" - ) from e - - parser = Parser() - parser.set_language(RST) +from tree_sitter_languages import get_parser + +parser = get_parser("rst") +allowed_adorn = "=-`:.'\"~^_*+#<>" log = logging.getLogger("papyri") From 96d33830d25e3ffe4e18291b2aa1067eaebf2d3a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 12 Feb 2024 15:23:26 +0100 Subject: [PATCH 10/11] test --- papyri/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/papyri/__init__.py b/papyri/__init__.py index e9f2a062..d991d66d 100644 --- a/papyri/__init__.py +++ b/papyri/__init__.py @@ -596,6 +596,8 @@ def build_parser(): work on all platofrms """ + # this is used just for dev of tree-sitter + return from tree_sitter import Language pth = Path(__file__).parent / "rst.so" From d3e03c762960e1e37038c62bbdab31864492733d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 14 Feb 2024 15:25:37 +0100 Subject: [PATCH 11/11] regen failing --- papyri/rich_render.py | 8 ++++++++ papyri/tests/expected/numpy:linspace.expected | 4 ++-- papyri/tree.py | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/papyri/rich_render.py b/papyri/rich_render.py index 5edb166b..c9478e6a 100644 --- a/papyri/rich_render.py +++ b/papyri/rich_render.py @@ -21,6 +21,10 @@ from .myst_ast import MText +import logging + +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from .myst_ast import MAdmonition, MAdmonitionTitle @@ -163,6 +167,10 @@ def visit_MText(self, node): assert res[-1].value != "\n" return res + def visit_MImage(self, node): + logger.warning("TODO: implement images") + return RToken("Image TODO").partition() + def visit_MEmphasis(self, node): return self.generic_visit(node.children) diff --git a/papyri/tests/expected/numpy:linspace.expected b/papyri/tests/expected/numpy:linspace.expected index 7effcde5..718a9212 100644 --- a/papyri/tests/expected/numpy:linspace.expected +++ b/papyri/tests/expected/numpy:linspace.expected @@ -6,8 +6,8 @@ Return evenly spaced numbers over a specified interval. ## Extended Summary -Returns num evenly spaced samples, calculated over the interval [`start`, stop -]. +Returns num evenly spaced samples, calculated over the interval [start, stop]. + The endpoint of the interval can optionally be excluded. diff --git a/papyri/tree.py b/papyri/tree.py index 84444cfe..50ad126d 100644 --- a/papyri/tree.py +++ b/papyri/tree.py @@ -356,6 +356,7 @@ def generic_visit(self, node) -> List[Node]: "MComment", "MInlineCode", "MInlineMath", + "MImage", "MMath", "MText", "MThematicBreak",