refactor: Consolidate fixes and code enhancements (#97)

* feat: Version each of bot & df_designer dirs

* style: Black up

* fix: Delete is_alive & slot interface

* fix: Run python3 if python wasn't found

* test: Add tests for new json_converter

* fix: Handle building if there's no slot_node in graph

* refactor: Add dev group in toml file

* refactor: Use ast instead of the manual old indexing

* chore: Update to the new template repo

* chore: Add logging to show weather it's gonna build

* fix: Save tg_token in .env

* feat: Support automatic transitions (previous, ...)

* style: Black up

* style: Flake8 up

* fix: Check for unique group names

* feat: Integrate slots in responses by FillTemplate

* feat: Delete websocket and use http_interface

* fix: Get last flow not last tag with `/flows`

* chore: Write logs out in case of stopped process

* test: Fix json-converter tests

* fix: Check aliveness of a tg|http process properly

* fix: Version with git only when using UI

* fix: Save built script after end of process

* fix: Return http health checking

* fix: Set a logger for service_replacer

* chore: Clean old config refreshing

* fix: Save script in periodic-check& separate is_alive

* fix: Return config reload refreshing feature

* fix: Terminate a group process with its children

* fix: Get env vars after reloading if any

* fix: Get env vars properly 2

* fix: Handle running processes at shuttdown

* fix: Override token envvar

* fix: Update database after shutdown

* style: Black up

* test: Add integration test_bot

* style: Black up

* chore: Update chatsky-ui version

* ci: Run tests with chatsky-ui env in dev mode

* ci: Try2

* chore: Checkou main template branch

* chore: Update readme

* refactor: Rename json-converter

* refactor: Add chatsky 0.9.0.dev1

* style: isort up

* ci: Test within chatsky-ui env

.github/workflows/backend_check.yml

@@ -75,13 +75,9 @@ jobs:
           python -m poetry run chatsky.ui init --destination ../ --no-input --overwrite-if-exists
         working-directory: backend
 
-      - name: Install chatsky-ui into new project poetry environment
-        run: |
-          ../bin/add_ui_to_toml.sh
-        working-directory: my_project
-
       - name: run tests
         run: |
-          python -m poetry install --no-root
-          python -m poetry run pytest ../backend/chatsky_ui/tests/ --verbose
+          poetry install --with dev -C ../backend
+          POETRY_ENV=$(poetry env info --path -C ../backend)
+          $POETRY_ENV/bin/pytest ../backend/chatsky_ui/tests/ --verbose
         working-directory: my_project

.github/workflows/e2e_test.yml

@@ -5,6 +5,7 @@
 #   push:
 #     branches:
 #     - dev
+#     - master
 #   pull_request:
 #     branches:
 #     - dev

Dockerfile

@@ -33,7 +33,6 @@ ENV PATH="${PATH}:${POETRY_VENV}/bin"
 COPY ./backend /temp/backend
 COPY --from=frontend-builder /temp/frontend/dist /temp/backend/chatsky_ui/static
 
-
 # Build the wheel
 WORKDIR /temp/backend
 RUN poetry build
@@ -47,6 +46,9 @@ ARG PROJECT_DIR
 # Install pip and upgrade
 RUN pip install --upgrade pip
 
+# Install Git
+RUN apt-get update && apt-get install -y git
+
 # Copy only the necessary files
 COPY --from=backend-builder /temp/backend/dist /src/dist
 COPY ./${PROJECT_DIR} /src/project_dir

Makefile

@@ -153,7 +153,7 @@ init_proj: install_backend_env ## Initiates a new project using chatsky-ui
 
 .PHONY: init_with_cc
 init_with_cc: ## Initiates a new project using cookiecutter
-	cookiecutter https://github.com/Ramimashkouk/df_d_template.git
+	cookiecutter https://github.com/deeppavlov/chatsky-ui-template.git
 
 
 .PHONY: build_docs

README.md

@@ -1,9 +1,9 @@
 # Quick Start
 ## System Requirements
-Ensure you have Python version 3.8.1 or higher installed.
+Ensure you have Python version 3.9 or higher installed (Excluding 3.9.7).
 
 ## Installation
-To install the necessary package, run the following command:
+To install the package and necessary dependencies, run the following command:
 ```bash
 pip install chatsky-ui
 ```

backend/chatsky_ui/api/api_v1/endpoints/bot.py

@@ -1,15 +1,14 @@
 import asyncio
 from typing import Any, Dict, List, Optional, Union
 
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, WebSocket, WebSocketException, status
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+from httpx import AsyncClient
 
 from chatsky_ui.api import deps
+from chatsky_ui.core.config import settings
 from chatsky_ui.schemas.pagination import Pagination
 from chatsky_ui.schemas.preset import Preset
-from chatsky_ui.schemas.process_status import Status
-from chatsky_ui.services.index import Index
 from chatsky_ui.services.process_manager import BuildManager, ProcessManager, RunManager
-from chatsky_ui.services.websocket_manager import WebSocketManager
 
 router = APIRouter()
 
@@ -45,7 +44,6 @@ async def start_build(
     preset: Preset,
     background_tasks: BackgroundTasks,
     build_manager: BuildManager = Depends(deps.get_build_manager),
-    index: Index = Depends(deps.get_index),
 ) -> Dict[str, Union[str, int]]:
 
     """Starts a `build` process with the given preset.
@@ -61,7 +59,7 @@ async def start_build(
 
     await asyncio.sleep(preset.wait_time)
     build_id = await build_manager.start(preset)
-    background_tasks.add_task(build_manager.check_status, build_id, index)
+    background_tasks.add_task(build_manager.check_status, build_id)
     build_manager.logger.info("Build process '%s' has started", build_id)
     return {"status": "ok", "build_id": build_id}
 
@@ -145,7 +143,7 @@ async def start_run(
     build_id: int,
     preset: Preset,
     background_tasks: BackgroundTasks,
-    run_manager: RunManager = Depends(deps.get_run_manager)
+    run_manager: RunManager = Depends(deps.get_run_manager),
 ) -> Dict[str, Union[str, int]]:
     """Starts a `run` process with the given preset.
 
@@ -236,47 +234,20 @@ async def get_run_logs(
         return await run_manager.fetch_run_logs(run_id, pagination.offset(), pagination.limit)
 
 
-@router.websocket("/run/connect")
-async def connect(
-    websocket: WebSocket,
-    websocket_manager: WebSocketManager = Depends(deps.get_websocket_manager),
-    run_manager: RunManager = Depends(deps.get_run_manager),
-) -> None:
-    """Establishes a WebSocket connection to communicate with an alive run process identified by its 'run_id'.
-
-    The WebSocket URL should adhere to the format: /bot/run/connect?run_id=<run_id>.
-    """
-
-    run_manager.logger.debug("Connecting to websocket")
-    run_id = websocket.query_params.get("run_id")
-
-    # Validate run_id
-    if run_id is None:
-        run_manager.logger.error("No run_id provided")
-        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)
-    if not run_id.isdigit():
-        run_manager.logger.error("A non-digit run run_id provided")
-        raise WebSocketException(code=status.WS_1003_UNSUPPORTED_DATA)
-    run_id = int(run_id)
-    if run_id not in run_manager.processes:
-        run_manager.logger.error("process with run_id '%s' exited or never existed", run_id)
-        raise WebSocketException(code=status.WS_1014_BAD_GATEWAY)
-
-    await websocket_manager.connect(websocket)
-    run_manager.logger.info("Websocket for run process '%s' has been opened", run_id)
-
-    output_task = asyncio.create_task(
-        websocket_manager.send_process_output_to_websocket(run_id, run_manager, websocket)
-    )
-    input_task = asyncio.create_task(
-        websocket_manager.forward_websocket_messages_to_process(run_id, run_manager, websocket)
-    )
-
-    # Wait for either task to finish
-    _, websocket_manager.pending_tasks[websocket] = await asyncio.wait(
-        [output_task, input_task],
-        return_when=asyncio.FIRST_COMPLETED,
-    )
-    websocket_manager.disconnect(websocket)
-    if await run_manager.get_status(run_id) in [Status.ALIVE, Status.RUNNING]:
-        await run_manager.stop(run_id)
+@router.post("/chat", status_code=201)
+async def respond(
+    user_id: str,
+    user_message: str,
+):
+    async with AsyncClient() as client:
+        try:
+            response = await client.post(
+                f"http://localhost:{settings.chatsky_port}/chat",
+                params={"user_id": user_id, "user_message": user_message},
+            )
+            return response.json()
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail=f"Please check that service's up and running on the port '{settings.chatsky_port}'.",
+            ) from e

backend/chatsky_ui/api/api_v1/endpoints/chatsky_services.py

@@ -1,32 +1,36 @@
 import re
 from io import StringIO
-from typing import Dict, Optional, Union
+from typing import Dict, Union
 
 import aiofiles
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter
 from pylint.lint import Run, pylinter
 from pylint.reporters.text import TextReporter
 
-from chatsky_ui.api.deps import get_index
 from chatsky_ui.clients.chatsky_client import get_chatsky_conditions
 from chatsky_ui.core.config import settings
 from chatsky_ui.schemas.code_snippet import CodeSnippet
-from chatsky_ui.services.index import Index
+from chatsky_ui.services.json_converter.logic_component_converter.service_replacer import get_all_classes
 from chatsky_ui.utils.ast_utils import get_imports_from_file
 
 router = APIRouter()
 
 
-@router.get("/search/{service_name}", status_code=200)
-async def search_service(service_name: str, index: Index = Depends(get_index)) -> Dict[str, Optional[Union[str, list]]]:
-    """Searches for a custom service by name and returns its code.
-
-    A service could be a condition, reponse, or pre/postservice.
-    """
-    response = await index.search_service(service_name)
+@router.get("/search/condition/{condition_name}", status_code=200)
+async def search_condition(condition_name: str) -> Dict[str, Union[str, list]]:
+    """Searches for a custom condition by name and returns its code."""
+    custom_classes = get_all_classes(settings.conditions_path)
+    response = [custom_class["body"] for custom_class in custom_classes if custom_class["name"] == condition_name]
     return {"status": "ok", "data": response}
 
 
+@router.get("/get_all_custom_conditions", status_code=200)
+async def get_all_custom_conditions_names() -> Dict[str, Union[str, list]]:
+    all_classes = get_all_classes(settings.conditions_path)
+    custom_classes = [custom_class["body"] for custom_class in all_classes]
+    return {"status": "ok", "data": custom_classes}
+
+
 @router.post("/lint_snippet", status_code=200)
 async def lint_snippet(snippet: CodeSnippet) -> Dict[str, str]:
     """Lints a snippet with Pylint.

backend/chatsky_ui/api/api_v1/endpoints/flows.py

@@ -1,17 +1,42 @@
-from typing import Dict, Union
+from pathlib import Path
+from typing import Dict, Optional, Union
 
-from fastapi import APIRouter
+from dotenv import set_key
+from fastapi import APIRouter, HTTPException, status
+from git.exc import GitCommandError
 from omegaconf import OmegaConf
 
 from chatsky_ui.core.config import settings
+from chatsky_ui.core.logger_config import get_logger
 from chatsky_ui.db.base import read_conf, write_conf
+from chatsky_ui.utils.git_cmd import commit_changes, get_repo
 
 router = APIRouter()
 
 
 @router.get("/")
-async def flows_get() -> Dict[str, Union[str, Dict[str, Union[list, dict]]]]:
+async def flows_get(build_id: Optional[int] = None) -> Dict[str, Union[str, Dict[str, Union[list, dict]]]]:
     """Get the flows by reading the frontend_flows.yaml file."""
+    repo = get_repo(settings.frontend_flows_path.parent)
+
+    if build_id is not None:
+        tag = int(build_id)
+        try:
+            repo.git.checkout(tag, settings.frontend_flows_path.name)
+        except GitCommandError as e:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Build_id {tag} not found",
+            ) from e
+    else:
+        try:
+            repo.git.checkout("HEAD", settings.frontend_flows_path.name)
+        except GitCommandError as e:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Failed to checkout the latest commit",
+            ) from e
+
     omega_flows = await read_conf(settings.frontend_flows_path)
     dict_flows = OmegaConf.to_container(omega_flows, resolve=True)
     return {"status": "ok", "data": dict_flows}  # type: ignore
@@ -20,5 +45,24 @@ async def flows_get() -> Dict[str, Union[str, Dict[str, Union[list, dict]]]]:
 @router.post("/")
 async def flows_post(flows: Dict[str, Union[list, dict]]) -> Dict[str, str]:
     """Write the flows to the frontend_flows.yaml file."""
+    logger = get_logger(__name__)
+    repo = get_repo(settings.frontend_flows_path.parent)
+
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    repo.git.checkout(tags[-1], settings.frontend_flows_path.name)
+
     await write_conf(flows, settings.frontend_flows_path)
+    logger.info("Flows saved to DB")
+
+    commit_changes(repo, "Save frontend flows")
+
     return {"status": "ok"}
+
+
+@router.post("/tg_token")
+async def post_tg_token(token: str):
+    dotenv_path = Path(settings.work_directory) / ".env"
+    dotenv_path.touch(exist_ok=True)
+
+    set_key(dotenv_path, "TG_BOT_TOKEN", token)
+    return {"status": "ok", "message": "Token saved successfully"}

backend/chatsky_ui/api/deps.py

@@ -1,7 +1,4 @@
-from chatsky_ui.core.config import settings
-from chatsky_ui.services.index import Index
 from chatsky_ui.services.process_manager import BuildManager, RunManager
-from chatsky_ui.services.websocket_manager import WebSocketManager
 
 build_manager = BuildManager()
 
@@ -17,20 +14,3 @@ def get_build_manager() -> BuildManager:
 def get_run_manager() -> RunManager:
     run_manager.set_logger()
     return run_manager
-
-
-websocket_manager = WebSocketManager()
-
-
-def get_websocket_manager() -> WebSocketManager:
-    websocket_manager.set_logger()
-    return websocket_manager
-
-
-index = Index()
-
-
-def get_index() -> Index:
-    index.set_logger()
-    index.set_path(settings.index_path)
-    return index