Adding support to raise the needed python modules to jac world and use…

jaclang/compiler/absyntree.py

@@ -635,6 +635,7 @@ def __init__(
         self.test_mod: list[Module] = []
         self.mod_deps: dict[str, Module] = {}
         self.registry = registry
+        self.is_py_raised: bool = False
         AstNode.__init__(self, kid=kid)
         AstDocNode.__init__(self, doc=doc)
 

jaclang/compiler/passes/main/fuse_typeinfo_pass.py

@@ -11,7 +11,6 @@
 
 import jaclang.compiler.absyntree as ast
 from jaclang.compiler.passes import Pass
-from jaclang.compiler.symtable import SymbolTable
 from jaclang.settings import settings
 from jaclang.utils.helpers import pascal_to_snake
 from jaclang.vendor.mypy.nodes import Node as VNode  # bit of a hack
@@ -447,20 +446,25 @@ def exit_assignment(self, node: ast.Assignment) -> None:
                 if self_obj.type_sym_tab and isinstance(right_obj, ast.AstSymbolNode):
                     self_obj.type_sym_tab.def_insert(right_obj)
 
+    def exit_atom_trailer(self, node: ast.AtomTrailer) -> None:
+        """Adding symbol links to AtomTrailer right nodes."""
+        # This will fix adding the symbol links to nodes in atom trailer
+        # self.x.z = 5  # will add symbol links to both x and z
+        for i in range(1, len(node.as_attr_list)):
+            left = node.as_attr_list[i - 1]
+            right = node.as_attr_list[i]
+            # assert isinstance(left, ast.NameAtom)
+            # assert isinstance(right, ast.NameAtom)
+
+            if left.type_sym_tab and not isinstance(
+                right, ast.IndexSlice
+            ):  # TODO check why IndexSlice produce an issue
+                right.name_spec.sym = left.type_sym_tab.lookup(right.sym_name)
+                right.type_sym_tab = left.type_sym_tab.find_scope(right.sym_name)
+
     def exit_name(self, node: ast.Name) -> None:
-        """Add new symbols in the symbol table in case of atom trailer."""
-        if isinstance(node.parent, ast.AtomTrailer):
-            target_node = node.parent.target
-            if isinstance(target_node, ast.AstSymbolNode):
-                parent_symbol_table = target_node.type_sym_tab
-                if isinstance(parent_symbol_table, SymbolTable):
-                    node.sym = parent_symbol_table.lookup(node.sym_name)
-
-    # def exit_in_for_stmt(self, node: ast.InForStmt):
-    #     print(node.loc.mod_path, node.loc)
-    #     print(node.target, node.target.loc)
-    #     # node.sym_tab.def_insert()
-    #     # exit()
-
-    # def after_pass(self) -> None:
-    #     exit()
+        """Update python nodes."""
+        assert self.ir._sym_tab is not None
+        py_symtab = self.ir._sym_tab.find_py_scope(node.sym_name)
+        if py_symtab:
+            node.type_sym_tab = py_symtab

jaclang/compiler/passes/main/import_pass.py

@@ -8,6 +8,7 @@
 import ast as py_ast
 import importlib.util
 import os
+import subprocess
 import sys
 from typing import Optional
 
@@ -58,7 +59,8 @@ def process_import(self, node: ast.Module, i: ast.ModulePath) -> None:
                 mod_path=node.loc.mod_path,
             )
         elif lang == "py":
-            self.__py_imports.add(i.path_str)
+            self.__py_imports.add(i.path_str.split(".")[0])
+            self.py_resolve_list.add(i.path_str)
 
     def attach_mod_to_node(
         self, node: ast.ModulePath | ast.ModuleItem, mod: ast.Module | None
@@ -202,8 +204,7 @@ def import_jac_mod_from_file(self, target: str) -> ast.Module | None:
             self.errors_had += mod_pass.errors_had
             self.warnings_had += mod_pass.warnings_had
             mod = mod_pass.ir
-        except Exception as e:
-            print(e)
+        except Exception:
             mod = None
         if isinstance(mod, ast.Module):
             self.import_table[target] = mod
@@ -214,6 +215,129 @@ def import_jac_mod_from_file(self, target: str) -> ast.Module | None:
             self.error(f"Module {target} is not a valid Jac module.")
             return None
 
+    def get_py_lib_path(self, import_path: str) -> Optional[str]:
+        """Try to get the stub path of a python module."""
+        base_library = import_path.split(".")[0]
+
+        try:
+            spec = importlib.util.find_spec(base_library)
+            lib_path = spec.origin if spec else None
+        except Exception:
+            lib_path = None
+
+        if lib_path is None:
+            return None
+        if os.path.sep not in lib_path:
+            return None
+
+        if (
+            os.path.isfile(lib_path)
+            and not lib_path.endswith(".py")
+            and not lib_path.endswith(".pyi")
+        ):
+            return None
+
+        if lib_path.endswith("py") and os.path.isfile(lib_path.replace(".py", ".pyi")):
+            lib_path = lib_path.replace(".py", ".pyi")
+
+        base_path = lib_path[: lib_path.rindex("/")]
+
+        for i in import_path.split(".")[1:]:
+
+            # TODO: Check this branch
+            if os.path.isdir(os.path.join(base_path, i)):
+                lib_path = os.path.join(base_path, i)
+
+            elif os.path.isfile(os.path.join(base_path, i + ".pyi")):
+                return os.path.join(base_path, i + ".pyi")
+
+            elif os.path.isfile(os.path.join(base_path, i + ".py")):
+                return os.path.join(base_path, i + ".py")
+
+        return lib_path
+
+    def grep(self, file_path: str, regex: str) -> list[str]:
+        """Search for a word inside a directory."""
+        command = ["grep", regex, file_path]
+        result = subprocess.run(command, capture_output=True, text=True)
+        return result.stdout.split("\n")
+
+    def after_pass(self) -> None:
+        """Call pass after_pass."""
+        from jaclang.compiler.passes.main import PyastBuildPass
+
+        # This part to handle importing/creating parent modules in case of doing
+        # import a.b.c
+        # without it code will crash as it will create a.b.c as a module and at linking
+        # it will try to link each a, b and c as separate modules which will crash
+        more_modules_to_import = []
+        for i in self.py_resolve_list:
+            if "." in i:
+                name_list = i.split(".")
+                for index in range(len(name_list)):
+                    more_modules_to_import.append(".".join(name_list[:index]))
+
+        for i in more_modules_to_import:
+            self.py_resolve_list.add(i)
+
+        sorted_resolve_list = list(self.py_resolve_list)
+        sorted_resolve_list.sort()
+
+        py_mod_map: dict[str, tuple[ast.Module, list[str]]] = {}
+
+        for i in sorted_resolve_list:
+
+            expected_file = self.get_py_lib_path(i)
+
+            if expected_file is None:
+                continue
+
+            if not os.path.isfile(expected_file):
+                continue
+
+            # final_target = i.split(".")[-1]
+            # base_file = i.split(".")[0]
+
+            # if "." in i:
+            #     print(self.grep(file_path=expected_file, regex=fr"\s*{final_target}\s*="))
+
+            if expected_file not in py_mod_map:
+                with open(expected_file, "r", encoding="utf-8") as f:
+                    py_mod_map[expected_file] = (
+                        PyastBuildPass(
+                            input_ir=ast.PythonModuleAst(
+                                py_ast.parse(f.read()), mod_path=expected_file
+                            ),
+                        ).ir,
+                        [i],
+                    )
+                    SubNodeTabPass(prior=self, input_ir=py_mod_map[expected_file][0])
+                    py_mod_map[expected_file][0].is_py_raised = True
+            else:
+                py_mod_map[expected_file][1].append(i)
+
+        attached_modules: dict[str, ast.Module] = {}
+        for i in py_mod_map:
+            mode = py_mod_map[i][0]
+            name_list = py_mod_map[i][1]  # List of names that uses the modules
+            name_list.sort()
+            mode_name = name_list[0].split(".")[
+                -1
+            ]  # Less name in length will be the module name itself
+            mode.name = mode_name
+
+            assert isinstance(self.ir, ast.Module)
+            if mode_name == name_list[0]:
+                self.attach_mod_to_node(self.ir, mode)
+                attached_modules[mode.name] = mode
+                mode.parent = self.ir
+            else:
+                # TODO: Fix me when an issue happens
+                parent_mode = attached_modules[name_list[0].split(".")[-2]]
+                self.attach_mod_to_node(parent_mode, mode)
+                # attached_modules[mode] = mode
+                mode.parent = parent_mode
+
 
 class PyImportPass(JacImportPass):
     """Jac statically imports Python modules."""

jaclang/compiler/passes/main/type_check_pass.py

@@ -30,7 +30,8 @@ def before_pass(self) -> None:
 
     def enter_module(self, node: ast.Module) -> None:
         """Call mypy checks on module level only."""
-        self.__modules.append(node)
+        if not node.is_py_raised:
+            self.__modules.append(node)
 
     def after_pass(self) -> None:
         """Call mypy api after traversing all the modules."""

jaclang/compiler/symtable.py

@@ -137,6 +137,17 @@ def find_scope(self, name: str) -> Optional[SymbolTable]:
                 return k
         return None
 
+    def find_py_scope(self, name: str) -> Optional[SymbolTable]:
+        """Find a scope that was originally a python module in the symbol table."""
+        for k in self.kid:
+            if (
+                isinstance(k.owner, ast.Module)
+                and k.owner.is_py_raised
+                and k.name == name
+            ):
+                return k
+        return None
+
     def push_scope(self, name: str, key_node: ast.AstNode) -> SymbolTable:
         """Push a new scope onto the symbol table."""
         self.kid.append(SymbolTable(name, key_node, self))

jaclang/tests/test_cli.py

@@ -9,6 +9,7 @@
 from jaclang.cli import cli
 from jaclang.plugin.builtin import dotgen
 from jaclang.utils.test import TestCase
+from jaclang.settings import settings
 
 
 class JacCliTests(TestCase):
@@ -105,6 +106,24 @@ def test_type_check(self) -> None:
         stdout_value = captured_output.getvalue()
         self.assertIn("Errors: 0, Warnings: 1", stdout_value)
 
+    def test_symbol_linking(self) -> None:
+        """Testing for type info inside the ast tool."""
+        settings.ast_symbol_info_detailed = True
+        captured_output = io.StringIO()
+        sys.stdout = captured_output
+        cli.tool("ir", ["ast", f"{self.fixture_abs_path('type_info.jac')}"])
+        sys.stdout = sys.__stdout__
+        stdout_value = captured_output.getvalue()
+        self.assertIn(
+            "Name - pygls - Type: NoType,  SymbolTable: pygls SymbolPath: type_info.pygls",
+            stdout_value,
+        )
+        self.assertIn(
+            "Name - LanguageServer - Type: NoType,  SymbolTable: LanguageServer SymbolPath: type_info.pygls.server.LanguageServer",
+            stdout_value,
+        )
+        settings.ast_symbol_info_detailed = False
+
     def test_type_info(self) -> None:
         """Testing for type info inside the ast tool."""
         captured_output = io.StringIO()

jaclang/utils/treeprinter.py

@@ -5,6 +5,7 @@
 import ast as ast3
 import builtins
 import html
+import os
 from typing import Optional, TYPE_CHECKING
 
 import jaclang.compiler.absyntree as ast
@@ -101,7 +102,13 @@ def __node_repr_in_tree(node: AstNode) -> str:
             else "SymbolTable: None" if isinstance(node, AstSymbolNode) else ""
         )
 
-        if isinstance(node, Token) and isinstance(node, AstSymbolNode):
+        types_to_ignore = (ast.Int, ast.Bool, ast.Float, ast.String)
+
+        if (
+            isinstance(node, Token)
+            and isinstance(node, AstSymbolNode)
+            and not isinstance(node, types_to_ignore)
+        ):
             out = (
                 f"{node.__class__.__name__} - {node.value} - "
                 f"Type: {node.sym_type}, {access} {sym_table_link}"
@@ -183,8 +190,12 @@ def mapper(draw: bool) -> str:
     markers += marker if level > 0 else ""
 
     if isinstance(root, ast.AstNode):
-        tree_str = f"{root.loc}\t{markers}{__node_repr_in_tree(root)}\n"
+        file_name = root.loc.mod_path
+        file_name = file_name.split(os.path.sep)[-1]
+        tree_str = f"{file_name} {root.loc}\t{markers}{__node_repr_in_tree(root)}\n"
         for i, child in enumerate(root.kid):
+            if isinstance(child, ast.Module) and child.is_py_raised:
+                continue
             is_last = i == len(root.kid) - 1
             tree_str += print_ast_tree(
                 child, marker, [*level_markers, not is_last], output_file, max_depth