Implement new theme in the UI

src/jinjat/core/dbt/dbt_project.py

@@ -266,8 +266,6 @@ def clear_caches(self) -> None:
         self.get_ref_node.cache_clear()
         self.get_source_node.cache_clear()
         self.get_macro_function.cache_clear()
-        self.get_columns.cache_clear()
-        self.compile_sql.cache_clear()
 
     @lru_cache(maxsize=10)
     def get_ref_node(self, target_model_name: str) -> MaybeNonSource:

src/jinjat/core/models.py

@@ -79,8 +79,8 @@ class JinjatAnalysisConfig(BaseModel):
     response: Optional[ResponseSchema] = ResponseSchema()
 
 
-async def generate_dbt_context_from_request(request: Request, openapi: dict,
-                                            transform_request: typing.Callable[[dict], dict]):
+async def generate_dbt_context_from_request(request: Request, openapi: dict = None,
+                                            transform_request: typing.Callable[[dict], dict] = None):
     if request.method in ['PATCH', 'PUT', 'POST']:
         body = transform_request(await request.json())
         validate(instance=body, schema=openapi)

src/jinjat/core/routes/analysis.py

@@ -2,9 +2,10 @@
 import itertools
 import json
 import re
+import typing
 from copy import deepcopy
 from pathlib import Path
-from typing import List, Optional, Callable, Mapping
+from typing import List, Optional, Callable
 
 import jmespath
 import jsonref
@@ -17,15 +18,14 @@
 from pydantic import ValidationError
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response
-from starlette.routing import Route
 
 from jinjat.core.dbt.dbt_project import DbtProject
 from jinjat.core.exceptions import InvalidJinjaConfig
 from jinjat.core.models import generate_dbt_context_from_request, JinjatExecutionResult, JinjatAnalysisConfig, \
     JinjatProjectConfig, JSON_COLUMNS_QUERY_PARAM, RequestSchema, ResponseSchema
 from jinjat.core.routes.admin import _execute_jinjat_query
 from jinjat.core.util.api import get_human_readable_error, rapidoc_html, register_jsonapi_exception_handlers, \
-    CustomButton, register_openapi_validators, unregister_openapi_validators
+    CustomButton, register_openapi_validators, unregister_openapi_validators, extract_key_value_pairs
 from jinjat.core.util.jmespath import extract_jmespath
 
 ANALYSIS_FILE_PATH_REGEX = re.compile(r"^analysis\/(.*)\.sql$")
@@ -139,8 +139,7 @@ async def custom_openapi(project, jinjat_project_config, api, package_name, req:
 
     if jinjat_project_config.openapi is not None:
         always_merger.merge(openapi_schema, jinjat_project_config.openapi)
-    if extract_path:
-        openapi_schema = jsonref.replace_refs(deepcopy(openapi_schema), base_uri="", proxies=False, lazy_load=False)
+    openapi_schema = jsonref.replace_refs(deepcopy(openapi_schema), base_uri="", proxies=False, lazy_load=False)
 
     api.openapi_schema = openapi_schema
 
@@ -150,7 +149,7 @@ async def custom_openapi(project, jinjat_project_config, api, package_name, req:
 def enrich_openapi_schema(project: DbtProject, openapi: Operation, request: RequestSchema, response: ResponseSchema):
     if request.body is not None:
         openapi.requestBody = RequestBody(content={"application/json": MediaType(schema=request.body)})
-    if response is not None:
+    if response.content is not None:
         response_body_schema = get_final_response(response.transform, response.content)
         openapi.responses = {response.status: APIResponse(description=response.description,
                                                           content={"application/json": MediaType(
@@ -175,13 +174,20 @@ def create_analysis_apps(jinjat_project_config: JinjatProjectConfig, project: Db
     analysis_lookup = {}
 
     async def lookup_by_id(request: Request, response: Response):
-        analysis_func = analysis_lookup["analysis."+request.path_params.get('id')]
+        analysis_name = request.path_params.get('id')
+        rest_of_path = request.path_params.get('rest_of_path')
+        path_params = extract_key_value_pairs(rest_of_path)
+        # Override it to propagate dbt context
+        request.scope['path_params'] = path_params
+        if '.' not in analysis_name:
+            analysis_name = f'{project.project_name}.{analysis_name}'
+        analysis_func = analysis_lookup.get("analysis." + analysis_name)
         if analysis_func is None:
-            return None
+            response.status_code = 404
         else:
             return await analysis_func(request, response)
 
-    api.add_api_route("/_id/{id}", endpoint=lookup_by_id)
+    api.add_api_route("/_analysis/{id}{rest_of_path:path}", endpoint=lookup_by_id)
 
     for package_name, analyses in nodes_by_packages.items():
         sub_app = FastAPI(redoc_url=None, docs_url=None, openapi_url=None)

src/jinjat/core/routes/notebook.py

@@ -0,0 +1,30 @@
+import os
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response, FileResponse, PlainTextResponse
+
+from jinjat.core.dbt.dbt_project import DbtProject
+from jinjat.core.models import JinjatProjectConfig, generate_dbt_context_from_request
+
+
+async def lookup_notebook_by_id(jinjat_project_config: JinjatProjectConfig, project: DbtProject,
+                                request: Request):
+    notebook_id = request.path_params.get('id')
+    [package_name, analysis] = notebook_id.split('.', 2)
+
+    current_project = project.config.dependencies[package_name]
+    # TODO: process all analysis_paths
+    directory = os.path.join(current_project.project_root, current_project.analysis_paths[0])
+    path = Path(directory).rglob(f'{analysis}.mdx')
+    found_file = next(path)
+    file_content = found_file.open('r').read()
+    result = file_content
+    # result = project.compile_sql(file_content,
+    #                              await generate_dbt_context_from_request(request)).compiled_sql
+    return JSONResponse({
+        "file_path": found_file.name,
+        "content": result
+    })

src/jinjat/core/server.py

@@ -20,6 +20,7 @@
 from jinjat.core.models import JinjatProjectConfig
 from jinjat.core.routes.admin import app as admin_app
 from jinjat.core.routes.analysis import create_analysis_apps
+from jinjat.core.routes.notebook import lookup_notebook_by_id
 from jinjat.core.util import filesystem
 from jinjat.core.util.api import register_jsonapi_exception_handlers, rapidoc_html, CustomButton, DBT_PROJECT_HEADER, \
     DBT_PROJECT_NAME, StaticFilesWithFallbackIndex, extract_host
@@ -95,7 +96,7 @@ def generate_app(config: JinjatProjectConfig, project: DbtProject, changed_file:
 def homepage_without_ui(host, project: DbtProject, dbt_target: DbtTarget) -> dict:
     return {
         "analysis_api_docs": f"{host}{project.config.project_name}/docs" if len(app.routes) > 2 else None,
-        "dependencies": [route.path for route in app.routes[2].routes[2:]],
+        "dependencies": [route.path for route in app.routes[3].routes[2:]],
         "admin_api_docs": f"{host}admin/docs",
         "magic": "https://jinj.at",
         "options": dbt_target.dict()
@@ -120,6 +121,7 @@ def mount_app(app: FastAPI, project: DbtProject, dbt_target: DbtTarget):
     current_app = generate_app(config, project)
 
     def watch(event: FileSystemEvent):
+        relative_dir = os.path.relpath(event.src_path, project.project_root)
         logger().info(f"Reloading project, file changed: {event.src_path}")
         project.safe_parse_project(reinit=True)
         logger().info(f"Updating the endpoints")
@@ -139,6 +141,7 @@ def watch(event: FileSystemEvent):
                                include_in_schema=False)
     admin_app.version = project.config.version
     app.mount("/admin", admin_app)
+    app.add_api_route("/_notebook/{id}", endpoint=functools.partial(lookup_notebook_by_id, config, project, ))
 
     default_refine_project = os.path.join(get_project_root(), *["src", "jinjat", "jinjat-refine"])
     project_static_files = os.path.join(project.project_root, "static")

src/jinjat/core/util/api.py

@@ -1,5 +1,6 @@
 import functools
 import os
+import re
 import typing
 from dataclasses import dataclass
 from enum import Enum
@@ -243,6 +244,9 @@ async def get_response(self, path: str, scope: Scope) -> Response:
 
 
 def convert_openapi_ref(default_project: str, cls, value, parent, model, _):
+    if value[0] == "#":
+        return value
+
     values = value.split(".", 3)
     if len(values) == 1:
         name = values[0]
@@ -270,3 +274,15 @@ def register_openapi_validators(project: DbtProject):
 
 def unregister_openapi_validators():
     Schema.__fields__.get("ref").post_validators = []
+
+
+def extract_key_value_pairs(path: str) -> typing.Dict[str, typing.Any]:
+    pattern = r"/(\w+):([^/]+)"
+    matches = re.findall(pattern, path)
+
+    output = {}
+    for match in matches:
+        key, value = match
+        output[key] = value
+
+    return output

src/jinjat/core/util/filesystem.py

@@ -7,12 +7,14 @@
 
 def watch(directory: str, callback):
     class DbtProjectWatcher(PatternMatchingEventHandler):
-        patterns = ['analysis/**/*.sql',
+        patterns = ['analyses/**/*.sql',
                     'analyses/**/*.yml',
-                    'macros/**/*.sql',
-                    'macros/**/*.yml',
+                    'analyses/**/*.mdx',
                     'models/**/*.sql',
                     'models/**/*.yml',
+                    'seeds/**/*.csv',
+                    'seeds/**/*.yml',
+                    'jinjat_project.yml'
                     'dbt_project.yml']
 
         def on_any_event(self, event):

src/jinjat/main.py

@@ -157,66 +157,6 @@ def serve(
     sys.exit(server.exitcode)
 
 
-@cli.command(
-    context_settings=dict(
-        ignore_unknown_options=True,
-        allow_extra_args=True,
-    )
-)
-@shared_single_project_opts
-@shared_server_opts
-@click.pass_context
-def streamlit(
-        ctx,
-        project_dir: str,
-        profiles_dir: str,
-        target: str,
-        host: str,
-        port: int,
-):
-    """Start the jinjat playground
-    \f
-    Pass the --options command to see streamlit specific options that can be passed to the app,
-    pass --config to see the output of streamlit config show
-    """
-
-    logger().info(":water_wave: Executing Jinjat Playground\n")
-
-    if "--options" in ctx.args:
-        subprocess.run(["streamlit", "run", "--help"])
-        ctx.exit()
-
-    import os
-
-    if "--config" in ctx.args:
-        subprocess.run(
-            ["streamlit", "config", "show"],
-            env=os.environ,
-            cwd=Path.cwd(),
-        )
-        ctx.exit()
-
-    script_args = ["--"]
-    if project_dir:
-        script_args.append("--project-dir")
-        script_args.append(project_dir)
-    if profiles_dir:
-        script_args.append("--profiles-dir")
-        script_args.append(profiles_dir)
-    if target:
-        script_args.append("--target")
-        script_args.append(target)
-
-    streamlit_command = ["streamlit", "run", "--runner.magicEnabled=false",
-                         Path(__file__).parent / "playground.py", ] + ctx.args + script_args
-    print(streamlit_command)
-    subprocess.run(
-        streamlit_command,
-        env=os.environ,
-        cwd=Path.cwd(),
-    )
-
-
 def run_server(host="localhost", port=8581, target=DbtTarget):
     uvicorn.run(
         lambda: get_multi_tenant_app(target),