feature: notebook as python file

zero_true/__init__.py

@@ -21,4 +21,14 @@
 from zt_backend.models.components.html import HTML, pygwalker
 from zt_backend.models.state.state import state
 from zt_backend.models.state.state import global_state
-from zt_backend.models.components.chat_ui import chat_ui
+from zt_backend.models.components.chat_ui import chat_ui
+from zt_backend.utils.pyfile_parser import cell,notebook
+
+def sql(var_name,sql_string):
+    return(var_name,sql_string)
+
+def md(md_string):
+    return(md_string)
+
+def text(txt_string):
+    return(txt_string)

zt_backend/main.py

@@ -2,7 +2,7 @@
 from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
 from zt_backend.config import settings
-from zt_backend.utils.notebook import get_notebook, write_notebook
+from zt_backend.utils.notebook import get_notebook, write_notebook_to_python, write_notebook
 from zt_backend.utils.dependencies import parse_dependencies, write_dependencies
 from copilot.copilot import copilot_app
 import zt_backend.router as router
@@ -45,9 +45,11 @@ def open_project():
         except Exception as e:
             logger.error("Unexpected error with matplotlib configuration: %s", e)
         notebook_path = Path(settings.zt_path) / "notebook.ztnb"
-        if not notebook_path.exists():
+        notebook_path = Path(settings.zt_path) / "notebook.py"
+        ztnb_path= Path(settings.zt_path) / "notebook.ztnb"
+        if not notebook_path.exists() and not ztnb_path.exists():
             logger.info("No notebook file found, creating with empty notebook")
-            write_notebook()
+            write_notebook_to_python()
         requirements_path = Path(settings.zt_path) / "requirements.txt"
         if not requirements_path.exists():
             logger.info("No requirements file found, creating empty file")

zt_backend/models/components/card.py

@@ -25,4 +25,4 @@ class Card(ZTComponent):
         description="Density of the component",
     )
     width: Optional[Union[int, str]] = Field("100%", description="Width of the card")
-    style: Optional[str] = Field("", description="CSS style to apply to the component")
+    style: Optional[str] = Field("", description="CSS style to apply to the card")

zt_backend/models/state/notebook_state.py

@@ -14,7 +14,7 @@ def __init__(self):
 
         self.notebook_db_path = notebook_db_path
         codeCell = notebook.CodeCell(
-            id=str(uuid.uuid4()),
+            id=str(uuid.uuid4()).split('-')[0],
             code="",
             components=[],
             variable_name="",

zt_backend/router.py

@@ -54,7 +54,7 @@
 from typing import Dict, Tuple, Optional
 import tempfile
 from zt_backend.utils.file_utils import *
-
+from zt_backend.utils.notebook import notebook_state
 
 router = APIRouter()
 manager = ConnectionManager()
@@ -193,8 +193,9 @@ def run_all():
 def create_cell(cellRequest: request.CreateRequest):
     if app_state.run_mode == "dev":
         logger.debug("Code cell addition request started")
+        num_cells = str(uuid.uuid4()).split('-')[0]
         createdCell = notebook.CodeCell(
-            id=str(uuid.uuid4()),
+            id="cell_"+str(num_cells),
             code="",
             components=[],
             output="",
@@ -1063,7 +1064,7 @@ async def move_item(move_request: request.MoveItemRequest) -> Dict:
                 )
 
             # Protected files check
-            protected_files = ["requirements.txt", "notebook.ztnb", 
+            protected_files = ["requirements.txt", "notebook.ztnb", "notebook.py",
                              "zt_db.db", "zt_db.db.wal"]
             if source_path.name in protected_files:
                 raise HTTPException(

zt_backend/runner/execute_code.py

@@ -21,6 +21,33 @@
 now = datetime.now()
 logger = logging.getLogger("__name__")
 
+def better_exec(code_, globals_=None, locals_=None, /, *, closure=None):
+    import ast
+    import linecache
+
+    if not hasattr(better_exec, "saved_sources"):
+        old_getlines = linecache.getlines
+        better_exec.saved_sources = []
+
+        def patched_getlines(filename, module_globals=None):
+            if "<exec#" in filename:
+                index = int(filename.split("#")[1].split(">")[0])
+                return better_exec.saved_sources[index].splitlines(True)
+            else:
+                return old_getlines(filename, module_globals)
+
+        linecache.getlines = patched_getlines
+    better_exec.saved_sources.append(code_)
+    exec(
+        compile(
+            ast.parse(code_),
+            filename=f"<exec#{len(better_exec.saved_sources) - 1}>",
+            mode="exec",
+        ),
+        globals_
+    )
+
+
 initial_cell = request.CodeRequest(
     id="initial_cell",
     code="""
@@ -236,7 +263,7 @@ def flush(self):
                     component_values=execution_state.component_values,
                 )
             execution_state.context_globals["exec_mode"] = True
-            exec(code_cell.code, temp_globals)
+            better_exec(code_cell.code,temp_globals)
 
         except Exception:
             logger.debug("Error during code execution")

zt_backend/tests/test_convert.py

@@ -1,51 +1,36 @@
 from zt_backend.models.state.notebook_state import NotebookState
-from zt_backend.models import notebook
-import rtoml
 import subprocess
 from pathlib import Path
 import os
-
-notebook_state = NotebookState()
+import importlib.util
 
 IPYNB_PATH = Path("zt_backend/tests/test_file.ipynb").resolve()
-OUTPUT_PATH = Path("zt_backend/tests/notebook.ztnb").resolve()
-NOTEBOOK_PATH = Path("zt_backend/tests/test_notebook.ztnb").resolve()
+OUTPUT_PATH = Path("zt_backend/tests/notebook.py").resolve()
+NOTEBOOK_PATH = Path("zt_backend/tests/test_notebook.py").resolve()
+
+def dynamic_import(module_path):
+    spec = importlib.util.spec_from_file_location("notebook_module", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
 
 def test_ipynb_to_ztnb():
 
+    # Step 1: Convert the IPYNB file to a Python file
     convert = subprocess.Popen(
         ["zero-true", "jupyter-convert", IPYNB_PATH, OUTPUT_PATH]
     )
     convert.wait()
 
-    with open(NOTEBOOK_PATH, "r", encoding="utf-8") as file:
-        expected_data = rtoml.loads(file.read().replace("\\", "\\\\"))
-
-    expected_notebook_data = {
-        "notebookId": "test_id",
-        "notebookName": expected_data.get("notebookName", "Zero True"),
-        "userId": "",
-        "cells": {
-            f"{index}": notebook.CodeCell(id=f"{index}", **cell_data, output="")
-            for index, (cell_id, cell_data) in enumerate(expected_data["cells"].items())
-        },
-    }
-    expected_notebook = notebook.Notebook(**expected_notebook_data)
-
-    with open(OUTPUT_PATH, "r", encoding="utf-8") as file:
-        output_data = rtoml.loads(file.read().replace("\\", "\\\\"))
-
-    output_notebook_data = {
-        "notebookId": "test_id",
-        "notebookName": output_data.get("notebookName", "Zero True"),
-        "userId": "",
-        "cells": {
-            f"{index}": notebook.CodeCell(id=f"{index}", **cell_data, output="")
-            for index, (cell_id, cell_data) in enumerate(output_data["cells"].items())
-        },
-    }
-    output_notebook = notebook.Notebook(**output_notebook_data)
-
-    assert expected_notebook == output_notebook
-
-    os.remove(OUTPUT_PATH)
+    # Step 2: Import the expected notebook from test_notebook.py
+    from test_notebook import notebook as expected_notebook
+    output_module = dynamic_import(OUTPUT_PATH)
+    output_notebook = output_module.notebook
+    # Step 3: Import the generated notebook from notebook.py
+    expected_notebook.notebookId='0'
+    output_notebook.notebookId='0'
+    # Step 4: Assert that the generated notebook matches the expected notebook
+    assert output_notebook == expected_notebook
+
+    # Step 5: Clean up the generated file
+    os.remove(OUTPUT_PATH)

zt_backend/tests/test_e2e.py

@@ -1,5 +1,6 @@
 import pytest
 import subprocess
+import textwrap
 import os
 from selenium import webdriver
 from selenium.webdriver.common.by import By
@@ -14,7 +15,6 @@
 
 notebook_id = str(uuid.uuid4())
 
-
 expected_code = """
 import zero_true as zt
 import time
@@ -23,23 +23,24 @@
 zt.TextInput(id='text')"""
 
 
-notebook_str = '''notebookId = "''' + notebook_id + '''"
-notebookName = "Zero True"
+# Define the expected Python code for the notebook
+notebook_str = f"""
+import zero_true as zt
+import time
 
-[cells.57fbbd59-8f30-415c-87bf-8caae0374070]
-cellName = ""
-cellType = "code"
-hideCell = "False"
-hideCode = "False"
-expandCode = "False"
-showTable = "False"
-nonReactive = "False"
-code = """
-'''+expected_code+'''"""
+def cell_0():
+"""+textwrap.indent(expected_code,'    ')+"""
 
-'''
+notebook = zt.notebook(
+    id="{notebook_id}",
+    name="Zero True",
+    cells=[
+        zt.cell(cell_0, type="code")
+    ]
+)
+"""
 
-notebook_filename = "notebook.ztnb"
+notebook_filename = "notebook.py"
 
 @pytest.fixture(scope="session", autouse=True)
 def start_stop_app():
@@ -81,6 +82,7 @@ def find_code_cells(driver):
     EC.presence_of_element_located((By.XPATH, "//div[contains(@id, 'codeCard')]"))
     )
     code_cells = driver.find_elements(By.XPATH, "//div[contains(@id, 'codeCard')]")
+    print(code_cells)
     return code_cells
 
 def find_el_by_id_w_exception(driver,element_id):
@@ -168,7 +170,7 @@ def test_notebook_loading(driver):
 
 def test_initial_code_cell(driver):
     code_cells = find_code_cells(driver)
-    assert len(code_cells) == 1 and code_cells[0].get_attribute('id') == 'codeCard57fbbd59-8f30-415c-87bf-8caae0374070', "Expected code cell not found."
+    assert len(code_cells) == 1 and code_cells[0].get_attribute('id') == 'codeCardcell_0', "Expected code cell not found."
 
 def test_intial_code_cell_content(driver):
     code_cells = find_code_cells(driver)
@@ -226,7 +228,7 @@ def test_adding_new_code_cell(driver):
 
     code_cells = find_code_cells(driver)
     assert len(code_cells) == 2, "New code cell was not added"
-    assert code_cells[0].get_attribute('id') == 'codeCard57fbbd59-8f30-415c-87bf-8caae0374070', "Expected code cell not found"
+    assert code_cells[0].get_attribute('id') == 'codeCardcell_0', "Expected code cell not found"
 
 def test_execution_of_new_code_cell(driver):
     code_cells = find_code_cells(driver)
@@ -339,7 +341,7 @@ def test_app_mode(driver):
 
     #assert that there is only cell in app mode because the second cell was hidden
 
-    cell_id_0 = '57fbbd59-8f30-415c-87bf-8caae0374070'
+    cell_id_0 = 'cell_0'
 
     assert len(code_cells) == 1 and code_cells[0].get_attribute('id') == f'codeCard{cell_id_0}', "Expected code cell not found."
 
@@ -385,7 +387,7 @@ def test_deletion_of_new_code_cell(driver):
     delete_btn.click()
     time.sleep(2)
     code_cells = find_code_cells(driver)
-    assert len(code_cells) == 1 and code_cells[0].get_attribute('id') == 'codeCard57fbbd59-8f30-415c-87bf-8caae0374070', "Expected code cell not found."
+    assert len(code_cells) == 1 and code_cells[0].get_attribute('id') == 'codeCardcell_0', "Expected code cell not found."
 
 # test hiding code cell
 

zt_backend/tests/test_notebook.py

@@ -0,0 +1,16 @@
+import zero_true as zt
+
+def cell_0():
+    print('Hello, world!')
+
+def cell_1():
+    zt.markdown("""# This is a markdown cell""")
+
+notebook = zt.notebook(
+    id='ca156d73-1c20-48c6-afbb-ae177c6bafa5',
+    name='Zero True',
+    cells=[
+        zt.cell(cell_0, type='code'),
+        zt.cell(cell_1, type='markdown'),
+    ]
+)

zt_backend/utils/completions.py

@@ -12,7 +12,7 @@ async def get_code_completions(cell_id: str, code: str, line: int, column: int)
 
     await text_document_did_change({
         "textDocument": {
-            "uri": "file:///notebook.ztnb",
+            "uri": "file:///notebook.py",
             "version": 1
         },
         "contentChanges": [{