From a50b2a63d4576b86ba0fd35b212d742815960cec Mon Sep 17 00:00:00 2001 From: mgtm98 Date: Thu, 11 Jul 2024 15:24:43 +0300 Subject: [PATCH 1/6] Adding suppor to raise the needed python modules to jac world and use it to get the symbols --- jaclang/compiler/absyntree.py | 1 + .../passes/main/fuse_typeinfo_pass.py | 38 +++--- jaclang/compiler/passes/main/import_pass.py | 113 +++++++++++++++++- .../compiler/passes/main/type_check_pass.py | 3 +- jaclang/compiler/symtable.py | 7 ++ jaclang/utils/treeprinter.py | 15 ++- 6 files changed, 156 insertions(+), 21 deletions(-) diff --git a/jaclang/compiler/absyntree.py b/jaclang/compiler/absyntree.py index 967c514f6..e3f284810 100644 --- a/jaclang/compiler/absyntree.py +++ b/jaclang/compiler/absyntree.py @@ -635,6 +635,7 @@ def __init__( self.test_mod: list[Module] = [] self.mod_deps: dict[str, Module] = {} self.registry = registry + self.py_lib: bool = False AstNode.__init__(self, kid=kid) AstDocNode.__init__(self, doc=doc) diff --git a/jaclang/compiler/passes/main/fuse_typeinfo_pass.py b/jaclang/compiler/passes/main/fuse_typeinfo_pass.py index 8364f54d3..39098b60d 100644 --- a/jaclang/compiler/passes/main/fuse_typeinfo_pass.py +++ b/jaclang/compiler/passes/main/fuse_typeinfo_pass.py @@ -11,7 +11,6 @@ import jaclang.compiler.absyntree as ast from jaclang.compiler.passes import Pass -from jaclang.compiler.symtable import SymbolTable from jaclang.settings import settings from jaclang.utils.helpers import pascal_to_snake from jaclang.vendor.mypy.nodes import Node as VNode # bit of a hack @@ -447,20 +446,25 @@ def exit_assignment(self, node: ast.Assignment) -> None: if self_obj.type_sym_tab and isinstance(right_obj, ast.AstSymbolNode): self_obj.type_sym_tab.def_insert(right_obj) + def exit_atom_trailer(self, node: ast.AtomTrailer) -> None: + """Adding symbol links to AtomTrailer right nodes.""" + # This will fix adding the symbol links to nodes in atom trailer + # self.x.z = 5 # will add symbol links to both x and z + for i in range(1, len(node.as_attr_list)): + left = node.as_attr_list[i - 1] + right = node.as_attr_list[i] + # assert isinstance(left, ast.NameAtom) + # assert isinstance(right, ast.NameAtom) + + if left.type_sym_tab and not isinstance( + right, ast.IndexSlice + ): # TODO check why IndexSlice produce an issue + right.name_spec.sym = left.type_sym_tab.lookup(right.sym_name) + right.type_sym_tab = left.type_sym_tab.find_scope(right.sym_name) + def exit_name(self, node: ast.Name) -> None: - """Add new symbols in the symbol table in case of atom trailer.""" - if isinstance(node.parent, ast.AtomTrailer): - target_node = node.parent.target - if isinstance(target_node, ast.AstSymbolNode): - parent_symbol_table = target_node.type_sym_tab - if isinstance(parent_symbol_table, SymbolTable): - node.sym = parent_symbol_table.lookup(node.sym_name) - - # def exit_in_for_stmt(self, node: ast.InForStmt): - # print(node.loc.mod_path, node.loc) - # print(node.target, node.target.loc) - # # node.sym_tab.def_insert() - # # exit() - - # def after_pass(self) -> None: - # exit() + """Update python nodes.""" + assert self.ir._sym_tab is not None + py_symtab = self.ir._sym_tab.find_py_scope(node.sym_name) + if py_symtab: + node.type_sym_tab = py_symtab diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 58fe5b0e5..5995249af 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -8,6 +8,7 @@ import ast as py_ast import importlib.util import os +import subprocess import sys from typing import Optional @@ -58,7 +59,8 @@ def process_import(self, node: ast.Module, i: ast.ModulePath) -> None: mod_path=node.loc.mod_path, ) elif lang == "py": - self.__py_imports.add(i.path_str) + self.__py_imports.add(i.path_str.split(".")[0]) + self.py_resolve_list.add(i.path_str) def attach_mod_to_node( self, node: ast.ModulePath | ast.ModuleItem, mod: ast.Module | None @@ -214,6 +216,115 @@ def import_jac_mod_from_file(self, target: str) -> ast.Module | None: self.error(f"Module {target} is not a valid Jac module.") return None + def get_py_lib_path(self, import_path: str) -> Optional[str]: + """Try to get the stub path of a python module.""" + base_library = import_path.split(".")[0] + + try: + spec = importlib.util.find_spec(base_library) + lib_path = spec.origin if spec else None + except ModuleNotFoundError: + lib_path = None + + if lib_path is None: + return None + if os.path.sep not in lib_path: + return None + + if lib_path.endswith("py") and os.path.isfile(lib_path.replace(".py", ".pyi")): + lib_path = lib_path.replace(".py", ".pyi") + + base_path = lib_path[: lib_path.rindex("/")] + + for i in import_path.split(".")[1:]: + + # TODO: Check this branch + if os.path.isdir(os.path.join(base_path, i)): + lib_path = os.path.join(base_path, i) + + elif os.path.isfile(os.path.join(base_path, i + ".pyi")): + return os.path.join(base_path, i + ".pyi") + + elif os.path.isfile(os.path.join(base_path, i + ".py")): + return os.path.join(base_path, i + ".py") + + return lib_path + + def grep(self, file_path: str, regex: str) -> list[str]: + """Search for a word inside a directory.""" + command = ["grep", regex, file_path] + result = subprocess.run(command, capture_output=True, text=True) + return result.stdout.split("\n") + + def after_pass(self) -> None: + """Call pass after_pass.""" + from jaclang.compiler.passes.main import PyastBuildPass + + sorted_resolve_list = list(self.py_resolve_list) + sorted_resolve_list.sort() + + py_mod_map: dict[str, tuple[ast.Module, list[str]]] = {} + + for i in sorted_resolve_list: + + expected_file = self.get_py_lib_path(i) + + if expected_file is None: + continue + + if not os.path.isfile(expected_file): + continue + + # final_target = i.split(".")[-1] + # base_file = i.split(".")[0] + + # if "." in i: + # print(self.grep(file_path=expected_file, regex=fr"\s*{final_target}\s*=")) + + try: + + if expected_file not in py_mod_map: + with open(expected_file, "r", encoding="utf-8") as f: + py_mod_map[expected_file] = ( + PyastBuildPass( + input_ir=ast.PythonModuleAst( + py_ast.parse(f.read()), mod_path=expected_file + ), + ).ir, + [i], + ) + SubNodeTabPass( + prior=self, input_ir=py_mod_map[expected_file][0] + ) + py_mod_map[expected_file][0].py_lib = True + else: + py_mod_map[expected_file][1].append(i) + + except Exception: + pass + + attached_modules: dict[str, ast.Module] = {} + for i in py_mod_map: + mode = py_mod_map[i][0] + name_list = py_mod_map[i][1] # List of names that uses the modules + name_list.sort() + mode_name = name_list[0].split(".")[ + -1 + ] # Less name in length will be the module name itself + mode.name = mode_name + + assert isinstance(self.ir, ast.Module) + if mode_name == name_list[0]: + self.attach_mod_to_node(self.ir, mode) + attached_modules[mode.name] = mode + mode.parent = self.ir + else: + # TODO: Fix me when an issue happens + parent_mode = attached_modules[name_list[0].split(".")[-2]] + self.attach_mod_to_node(parent_mode, mode) + # attached_modules[mode] = mode + mode.parent = parent_mode + class PyImportPass(JacImportPass): """Jac statically imports Python modules.""" diff --git a/jaclang/compiler/passes/main/type_check_pass.py b/jaclang/compiler/passes/main/type_check_pass.py index 52fe223ae..230cf0873 100644 --- a/jaclang/compiler/passes/main/type_check_pass.py +++ b/jaclang/compiler/passes/main/type_check_pass.py @@ -30,7 +30,8 @@ def before_pass(self) -> None: def enter_module(self, node: ast.Module) -> None: """Call mypy checks on module level only.""" - self.__modules.append(node) + if not node.py_lib: + self.__modules.append(node) def after_pass(self) -> None: """Call mypy api after traversing all the modules.""" diff --git a/jaclang/compiler/symtable.py b/jaclang/compiler/symtable.py index 93b520b0f..f47bd99dd 100644 --- a/jaclang/compiler/symtable.py +++ b/jaclang/compiler/symtable.py @@ -137,6 +137,13 @@ def find_scope(self, name: str) -> Optional[SymbolTable]: return k return None + def find_py_scope(self, name: str) -> Optional[SymbolTable]: + """Find a scope that was originally a python module in the symbol table.""" + for k in self.kid: + if isinstance(k.owner, ast.Module) and k.owner.py_lib and k.name == name: + return k + return None + def push_scope(self, name: str, key_node: ast.AstNode) -> SymbolTable: """Push a new scope onto the symbol table.""" self.kid.append(SymbolTable(name, key_node, self)) diff --git a/jaclang/utils/treeprinter.py b/jaclang/utils/treeprinter.py index 3aa6c3498..ae58f2acb 100644 --- a/jaclang/utils/treeprinter.py +++ b/jaclang/utils/treeprinter.py @@ -5,6 +5,7 @@ import ast as ast3 import builtins import html +import os from typing import Optional, TYPE_CHECKING import jaclang.compiler.absyntree as ast @@ -101,7 +102,13 @@ def __node_repr_in_tree(node: AstNode) -> str: else "SymbolTable: None" if isinstance(node, AstSymbolNode) else "" ) - if isinstance(node, Token) and isinstance(node, AstSymbolNode): + types_to_ignore = (ast.Int, ast.Bool, ast.Float, ast.String) + + if ( + isinstance(node, Token) + and isinstance(node, AstSymbolNode) + and not isinstance(node, types_to_ignore) + ): out = ( f"{node.__class__.__name__} - {node.value} - " f"Type: {node.sym_type}, {access} {sym_table_link}" @@ -183,8 +190,12 @@ def mapper(draw: bool) -> str: markers += marker if level > 0 else "" if isinstance(root, ast.AstNode): - tree_str = f"{root.loc}\t{markers}{__node_repr_in_tree(root)}\n" + file_name = root.loc.mod_path + file_name = file_name.split(os.path.sep)[-1] + tree_str = f"{file_name} {root.loc}\t{markers}{__node_repr_in_tree(root)}\n" for i, child in enumerate(root.kid): + if isinstance(child, ast.Module) and child.py_lib: + continue is_last = i == len(root.kid) - 1 tree_str += print_ast_tree( child, marker, [*level_markers, not is_last], output_file, max_depth From f247452089393509bc5797b866ba4bd3862f21c8 Mon Sep 17 00:00:00 2001 From: mgtm98 Date: Thu, 11 Jul 2024 22:03:41 +0300 Subject: [PATCH 2/6] Fixing an issue with creating python symbols, Adding new testcase --- jaclang/compiler/passes/main/import_pass.py | 49 ++++++++++++--------- jaclang/tests/test_cli.py | 19 ++++++++ 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 5995249af..2a78f0641 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -260,6 +260,20 @@ def after_pass(self) -> None: """Call pass after_pass.""" from jaclang.compiler.passes.main import PyastBuildPass + # This part to handle importing/creating parent modules in case of doing + # import a.b.c + # without it code will crash as it will create a.b.c as a module and at linking + # it will try to link each a, b and c as separate modules which will crash + more_modules_to_import = [] + for i in self.py_resolve_list: + if "." in i: + name_list = i.split(".") + for index in range(len(name_list)): + more_modules_to_import.append(".".join(name_list[:index])) + + for i in more_modules_to_import: + self.py_resolve_list.add(i) + sorted_resolve_list = list(self.py_resolve_list) sorted_resolve_list.sort() @@ -281,27 +295,20 @@ def after_pass(self) -> None: # if "." in i: # print(self.grep(file_path=expected_file, regex=fr"\s*{final_target}\s*=")) - try: - - if expected_file not in py_mod_map: - with open(expected_file, "r", encoding="utf-8") as f: - py_mod_map[expected_file] = ( - PyastBuildPass( - input_ir=ast.PythonModuleAst( - py_ast.parse(f.read()), mod_path=expected_file - ), - ).ir, - [i], - ) - SubNodeTabPass( - prior=self, input_ir=py_mod_map[expected_file][0] - ) - py_mod_map[expected_file][0].py_lib = True - else: - py_mod_map[expected_file][1].append(i) - - except Exception: - pass + if expected_file not in py_mod_map: + with open(expected_file, "r", encoding="utf-8") as f: + py_mod_map[expected_file] = ( + PyastBuildPass( + input_ir=ast.PythonModuleAst( + py_ast.parse(f.read()), mod_path=expected_file + ), + ).ir, + [i], + ) + SubNodeTabPass(prior=self, input_ir=py_mod_map[expected_file][0]) + py_mod_map[expected_file][0].py_lib = True + else: + py_mod_map[expected_file][1].append(i) attached_modules: dict[str, ast.Module] = {} for i in py_mod_map: diff --git a/jaclang/tests/test_cli.py b/jaclang/tests/test_cli.py index e4880921e..c9e3e6ad2 100644 --- a/jaclang/tests/test_cli.py +++ b/jaclang/tests/test_cli.py @@ -9,6 +9,7 @@ from jaclang.cli import cli from jaclang.plugin.builtin import dotgen from jaclang.utils.test import TestCase +from jaclang.settings import settings class JacCliTests(TestCase): @@ -105,6 +106,24 @@ def test_type_check(self) -> None: stdout_value = captured_output.getvalue() self.assertIn("Errors: 0, Warnings: 1", stdout_value) + def test_symbol_linking(self) -> None: + """Testing for type info inside the ast tool.""" + settings.ast_symbol_info_detailed = True + captured_output = io.StringIO() + sys.stdout = captured_output + cli.tool("ir", ["ast", f"{self.fixture_abs_path('type_info.jac')}"]) + sys.stdout = sys.__stdout__ + stdout_value = captured_output.getvalue() + self.assertIn( + "Name - pygls - Type: NoType, SymbolTable: pygls SymbolPath: type_info.pygls", + stdout_value, + ) + self.assertIn( + "Name - LanguageServer - Type: NoType, SymbolTable: LanguageServer SymbolPath: type_info.pygls.server.LanguageServer", + stdout_value, + ) + settings.ast_symbol_info_detailed = False + def test_type_info(self) -> None: """Testing for type info inside the ast tool.""" captured_output = io.StringIO() From ceaf891564150ce7ac954e4bbf0532ef969fd3d8 Mon Sep 17 00:00:00 2001 From: mgtm98 Date: Fri, 12 Jul 2024 22:11:39 +0300 Subject: [PATCH 3/6] Fixing an issue with the new support that happens in pytest --- jaclang/compiler/passes/main/import_pass.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 2a78f0641..26a62be2a 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -220,10 +220,12 @@ def get_py_lib_path(self, import_path: str) -> Optional[str]: """Try to get the stub path of a python module.""" base_library = import_path.split(".")[0] + print(base_library) + try: spec = importlib.util.find_spec(base_library) lib_path = spec.origin if spec else None - except ModuleNotFoundError: + except Exception: lib_path = None if lib_path is None: From 5649d2883ac9c228eb8071e465ff01b6c6febc8a Mon Sep 17 00:00:00 2001 From: mgtm98 Date: Fri, 12 Jul 2024 22:18:58 +0300 Subject: [PATCH 4/6] Remove unwanted prints slipped in other commits :( --- jaclang/compiler/passes/main/import_pass.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 26a62be2a..623e82de6 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -205,7 +205,6 @@ def import_jac_mod_from_file(self, target: str) -> ast.Module | None: self.warnings_had += mod_pass.warnings_had mod = mod_pass.ir except Exception as e: - print(e) mod = None if isinstance(mod, ast.Module): self.import_table[target] = mod @@ -220,8 +219,6 @@ def get_py_lib_path(self, import_path: str) -> Optional[str]: """Try to get the stub path of a python module.""" base_library = import_path.split(".")[0] - print(base_library) - try: spec = importlib.util.find_spec(base_library) lib_path = spec.origin if spec else None From 995763ca9d10fb75d64798f254fa0f1027f13696 Mon Sep 17 00:00:00 2001 From: mgtm98 Date: Fri, 12 Jul 2024 23:06:21 +0300 Subject: [PATCH 5/6] Fixing an issue where trying to pyraise a non python code files (.so files) --- jaclang/compiler/passes/main/import_pass.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 623e82de6..8c6e07efd 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -230,6 +230,13 @@ def get_py_lib_path(self, import_path: str) -> Optional[str]: if os.path.sep not in lib_path: return None + if ( + os.path.isfile(lib_path) + and not lib_path.endswith(".py") + and not lib_path.endswith(".pyi") + ): + return None + if lib_path.endswith("py") and os.path.isfile(lib_path.replace(".py", ".pyi")): lib_path = lib_path.replace(".py", ".pyi") From 39d77e2fa158ff28dcc9a18d383568cff696370c Mon Sep 17 00:00:00 2001 From: marsninja Date: Sat, 13 Jul 2024 08:38:11 -0400 Subject: [PATCH 6/6] tweak: small --- jaclang/compiler/absyntree.py | 2 +- jaclang/compiler/passes/main/import_pass.py | 4 ++-- jaclang/compiler/passes/main/type_check_pass.py | 2 +- jaclang/compiler/symtable.py | 6 +++++- jaclang/utils/treeprinter.py | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/jaclang/compiler/absyntree.py b/jaclang/compiler/absyntree.py index e3f284810..c15c74c20 100644 --- a/jaclang/compiler/absyntree.py +++ b/jaclang/compiler/absyntree.py @@ -635,7 +635,7 @@ def __init__( self.test_mod: list[Module] = [] self.mod_deps: dict[str, Module] = {} self.registry = registry - self.py_lib: bool = False + self.is_py_raised: bool = False AstNode.__init__(self, kid=kid) AstDocNode.__init__(self, doc=doc) diff --git a/jaclang/compiler/passes/main/import_pass.py b/jaclang/compiler/passes/main/import_pass.py index 8c6e07efd..f2dddb3dc 100644 --- a/jaclang/compiler/passes/main/import_pass.py +++ b/jaclang/compiler/passes/main/import_pass.py @@ -204,7 +204,7 @@ def import_jac_mod_from_file(self, target: str) -> ast.Module | None: self.errors_had += mod_pass.errors_had self.warnings_had += mod_pass.warnings_had mod = mod_pass.ir - except Exception as e: + except Exception: mod = None if isinstance(mod, ast.Module): self.import_table[target] = mod @@ -312,7 +312,7 @@ def after_pass(self) -> None: [i], ) SubNodeTabPass(prior=self, input_ir=py_mod_map[expected_file][0]) - py_mod_map[expected_file][0].py_lib = True + py_mod_map[expected_file][0].is_py_raised = True else: py_mod_map[expected_file][1].append(i) diff --git a/jaclang/compiler/passes/main/type_check_pass.py b/jaclang/compiler/passes/main/type_check_pass.py index 230cf0873..4510fb3ba 100644 --- a/jaclang/compiler/passes/main/type_check_pass.py +++ b/jaclang/compiler/passes/main/type_check_pass.py @@ -30,7 +30,7 @@ def before_pass(self) -> None: def enter_module(self, node: ast.Module) -> None: """Call mypy checks on module level only.""" - if not node.py_lib: + if not node.is_py_raised: self.__modules.append(node) def after_pass(self) -> None: diff --git a/jaclang/compiler/symtable.py b/jaclang/compiler/symtable.py index f47bd99dd..0427c38f5 100644 --- a/jaclang/compiler/symtable.py +++ b/jaclang/compiler/symtable.py @@ -140,7 +140,11 @@ def find_scope(self, name: str) -> Optional[SymbolTable]: def find_py_scope(self, name: str) -> Optional[SymbolTable]: """Find a scope that was originally a python module in the symbol table.""" for k in self.kid: - if isinstance(k.owner, ast.Module) and k.owner.py_lib and k.name == name: + if ( + isinstance(k.owner, ast.Module) + and k.owner.is_py_raised + and k.name == name + ): return k return None diff --git a/jaclang/utils/treeprinter.py b/jaclang/utils/treeprinter.py index ae58f2acb..91ef9c426 100644 --- a/jaclang/utils/treeprinter.py +++ b/jaclang/utils/treeprinter.py @@ -194,7 +194,7 @@ def mapper(draw: bool) -> str: file_name = file_name.split(os.path.sep)[-1] tree_str = f"{file_name} {root.loc}\t{markers}{__node_repr_in_tree(root)}\n" for i, child in enumerate(root.kid): - if isinstance(child, ast.Module) and child.py_lib: + if isinstance(child, ast.Module) and child.is_py_raised: continue is_last = i == len(root.kid) - 1 tree_str += print_ast_tree(