diff --git a/.gitignore b/.gitignore index 5a050c7..3a49bec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ output/ .vscode/ ..vscode/ .git/ +token.txt +secrets.txt # package files config.json diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index bb7b60b..ad7a7cb 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -15,3 +15,53 @@ # ports: # - 2000:2000 # - 5678:5678 + +version: '3' + +services: + web: + build: + context: . + dockerfile: ./src/codecarto/containers/web/Dockerfile + ports: + - '2000:2000' + networks: + - external_network + - internal_network + # volumnes are for quicker docker builds during development + volumes: + - ./src/codecarto/containers/web/api:/app/api + - ./src/codecarto/containers/web/src:/app/src + + processor: + build: + context: . + dockerfile: ./src/codecarto/containers/processor/Dockerfile + secrets: + - github_token + networks: + - internal_network + # volumnes are for quicker docker builds during development + volumes: + - ./src/codecarto/containers/processor/api:/app/api + - ./src/codecarto/containers/processor/src:/app/src + + database: + image: mongo:latest + ports: + - '27017:27017' + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: examplepassword + networks: + - internal_network + +networks: + external_network: + driver: bridge + internal_network: + driver: bridge + +secrets: + github_token: + file: ./token.txt diff --git a/docker-compose.yml b/docker-compose.yml index cc354db..d64b7be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: build: context: . dockerfile: ./src/codecarto/containers/processor/Dockerfile + secrets: + - github_token networks: - internal_network @@ -35,3 +37,7 @@ networks: driver: bridge internal_network: driver: bridge + +secrets: + github_token: + file: ./token.txt diff --git a/src/codecarto/containers/processor/Dockerfile b/src/codecarto/containers/processor/Dockerfile index c520316..0a22026 100644 --- a/src/codecarto/containers/processor/Dockerfile +++ b/src/codecarto/containers/processor/Dockerfile @@ -10,7 +10,8 @@ ENV PYTHONUNBUFFERED=1 # Install pip requirements COPY ./src/codecarto/containers/processor/requirements.txt . -RUN python -m pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install -r requirements.txt # Directory WORKDIR /app @@ -19,10 +20,14 @@ COPY ./src/codecarto/containers/processor/src /app/src ENV PYTHONPATH=/app # # Creates a non-root user with an explicit UID and adds permission to access the /app folder -# # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers # RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app # USER appuser +# # Good idea to also remove the shell access and ensure there is no home directory for the user +# RUN addgroup --gid 1001 --system app && \ +# adduser --no-create-home --shell /bin/false --disabled-password --uid 1001 --system --group app +# USER app + # # During debugging, this entry point will be overridden. # For more information, please refer to https://aka.ms/vscode-docker-python-debug -CMD ["gunicorn", "--bind", "0.0.0.0:2020", "-k", "uvicorn.workers.UvicornWorker", "api.main:app"] \ No newline at end of file +CMD ["gunicorn", "--bind", "0.0.0.0:2020", "-k", "uvicorn.workers.UvicornWorker", "api.main:app"] \ No newline at end of file diff --git a/src/codecarto/containers/processor/api/routers/palette_router.py b/src/codecarto/containers/processor/api/routers/palette_router.py index 5072dfd..f91832c 100644 --- a/src/codecarto/containers/processor/api/routers/palette_router.py +++ b/src/codecarto/containers/processor/api/routers/palette_router.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse from src.plotter.palette import Theme -from api.util import generate_return, proc_exception +from api.util import generate_return, proc_exception, proc_error PaletteRoute: APIRouter = APIRouter() default_palette_path: str = "src/plotter/default_palette.json" @@ -33,7 +33,7 @@ async def get_palette(user_id: int = -1) -> dict: with open(file_path, "r") as f: pal_data = load(f) - return generate_return("success", "Proc - Success", pal_data) + return generate_return(200, "Proc - Success", pal_data) except Exception as e: proc_exception( "get_palette", @@ -73,10 +73,11 @@ async def set_palette(user_id: int = -1, new_pal_data: dict = {}) -> dict: return pal_data else: - proc_exception( + return proc_error( "set_palette", "No new palette data provided", {"user_id": user_id, "new_pal_data": new_pal_data}, + 500, ) except Exception as e: proc_exception( diff --git a/src/codecarto/containers/processor/api/routers/parser_router.py b/src/codecarto/containers/processor/api/routers/parser_router.py index 378de87..8f4e0f0 100644 --- a/src/codecarto/containers/processor/api/routers/parser_router.py +++ b/src/codecarto/containers/processor/api/routers/parser_router.py @@ -1,7 +1,7 @@ import httpx from fastapi import APIRouter, HTTPException -from api.util import generate_return, proc_exception +from api.util import generate_return, proc_exception, proc_error # DEBUG import logging @@ -20,68 +20,83 @@ async def parse(): @ParserRoute.get("/handle_github_url") async def handle_github_url(github_url: str) -> dict: try: + import time + client = httpx.AsyncClient() logger.info( f" Started Proc.handle_github_url(): github_url - {github_url}" ) + # get current time to calculate total time taken + start_time = time.time() + # check that the url is a github url if "github.com" not in github_url: - proc_exception( + return proc_error( "handle_github_url", - "URL is not a valid GitHub URL", + "Invalid GitHub URL", {"github_url": github_url}, - status=404, + 404, ) # Extract owner and repo from the URL # Assuming the URL is like: https://github.com/owner/repo parts = github_url.split("/") if len(parts) < 5 or parts[2] != "github.com": - proc_exception( - "read_github_file", + return proc_error( + "handle_github_url", "Invalid GitHub URL format", {"github_url": github_url}, + 404, ) owner, repo = parts[3], parts[4] # get content from url - url_content: list[dict] = await read_github_content(github_url, owner, repo) + url_content: list[dict] | dict = await read_github_content( + github_url, owner, repo, "", True + ) if not url_content: - proc_exception( + return proc_error( "handle_github_url", "Empty file content received from GitHub", {"github_url": github_url}, + 500, ) + else: + # check if url_content is an error + if "status" in url_content: + return url_content - # url_content = await fetch_directory(github_url) + # parse the content repo_contents = { "package_owner": owner, "package_name": repo, "contents": {}, } - logger.info(f" Started Proc.parse_github_content(): {owner}/{repo}") - repo_contents["contents"]: dict = await parse_github_content( - url_content, owner, repo - ) - logger.info(f" Finished Proc.parse_github_content()") - if repo_contents: - return generate_return( - "success", "Proc.handle_github_url() - Success", repo_contents - ) + logger.info(f"\tStarted\tProc.parse_github_content(): {owner}/{repo}") + contents: dict = await parse_github_content(url_content, owner, repo) + logger.info(f"\tFinished\tProc.parse_github_content()") + + # check contents + if contents: + # check if contents dict has status key + # if it does, then it is an error + logger.info(f"\n\n\tContents: {contents}\n\n") + if "status" in contents: + return contents + else: + # otherwise good to return + repo_contents["contents"] = contents + return generate_return( + 200, "Proc.handle_github_url() - Success", repo_contents + ) else: - proc_exception( + # contents is empty + return proc_error( "handle_github_url", "Could not parse file content", {"github_url": github_url}, + 500, ) - except HTTPException as exc: - # Handle network errors - proc_exception( - "handle_github_url", - "An error occurred while requesting", - {"github_url": github_url}, - exc, - ) except Exception as exc: proc_exception( "handle_github_url", @@ -92,76 +107,108 @@ async def handle_github_url(github_url: str) -> dict: finally: await client.aclose() logger.info(f" Finished Proc.handle_github_url()") + # calculate total time taken + end_time = time.time() + total_time = time.strftime("%H:%M:%S", time.gmtime(end_time - start_time)) + logger.info(f" Total time taken: {total_time}") + # TODO: Log this in database later async def read_github_content( - url: str, owner: str, repo: str, path: str = "" + url: str, + owner: str, + repo: str, + path: str = "", + first: bool = False, ) -> list[dict]: try: - import os - from dotenv import load_dotenv - client = httpx.AsyncClient() - logger.info(f"Started Proc.read_github_content(): {url}") # Construct the API URL - api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}" - load_dotenv() - git_api_key = os.getenv("GIT_API_KEY") + with open("/run/secrets/github_token", "r") as file: + GIT_API_KEY = file.read().strip() + if not GIT_API_KEY or GIT_API_KEY == "": + return proc_error( + "read_github_content", + "No GitHub API key found", + {"url": url, "api_url": api_url}, + 403, + ) headers = { "Accept": "application/vnd.github.v3+json", # Uncomment and set your token if you have one - "Authorization": f"Bearer {git_api_key}", + "Authorization": f"token {GIT_API_KEY}", } + # get the size of the whole repo + if first: + api_url = f"https://api.github.com/repos/{owner}/{repo}" + response = await client.get( + api_url, headers=headers, follow_redirects=False + ) + if response.status_code == 200: + json_data = response.json() + if json_data["size"]: + size = json_data["size"] + logger.info(f" Repo Size: {size} bytes") + if size > 1000000: + return proc_error( + "read_github_content", + "GitHub repo is too large", + {"url": url, "api_url": api_url}, + 500, + ) + + # get the actual contents of the repo + api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}" response = await client.get(api_url, headers=headers, follow_redirects=False) + # Check the response if response.status_code == 200: json_data = response.json() if not json_data: - proc_exception( + return proc_error( "read_github_content", - "No data returned from GitHub API for UR", + "No data returned from GitHub API for URL", {"url": url, "api_url": api_url}, + 404, ) + else: + # Remove unnecessary data from the response + # this will leave us with {name, path, size, html_url, download_url, type} + # html url is the url to view the file in the browser + # download url is the url to see just the raw file contents + for item in json_data: + item.pop("sha", None) + item.pop("url", None) + item.pop("git_url", None) + item.pop("_links", None) - # Remove unnecessary data from the response - # this will leave us with {name, path, size, html_url, download_url, type} - # html url is the url to view the file in the browser - # download url is the url to see just the raw file contents - for item in json_data: - item.pop("sha", None) - item.pop("url", None) - item.pop("git_url", None) - item.pop("_links", None) - - logger.info(f" json_data: {json_data}") - return json_data + return json_data else: if response.status_code == 404: - proc_exception( + return proc_error( "read_github_content", "GitHub API returned 404", {"url": url, "api_url": api_url}, - HTTPException, 404, ) if response.status_code == 403: error_message = f"GitHub API returned 403: {response.text}" - # if "rate_limit" in response.text: - # error_message = f"GitHub API rate limit exceeded: {response.text}" - proc_exception( + if "rate_limit" in response.text: + error_message = f"GitHub API rate limit exceeded: {response.text}" + return proc_error( "read_github_content", error_message, {"url": url, "api_url": api_url}, - HTTPException, 403, ) else: - proc_exception( + return proc_error( "read_github_content", "Error with client response", {"url": url, "status_code": response.status_code}, + 500, ) except httpx.RequestError as exc: proc_exception( @@ -170,18 +217,17 @@ async def read_github_content( {"url": url}, exc, ) - finally: - logger.info(f"Finished Proc.read_github_content(): {url}") async def parse_github_content(file_content, owner, repo) -> dict: try: # Check that the file content is a list if not file_content or not isinstance(file_content, list): - proc_exception( + return proc_error( "parse_github_content", "Invalid file content format", - {"file_content": file_content}, + {}, + 404, ) # Process directories @@ -224,7 +270,7 @@ def raw_to_graph(raw_data: str, filename: str): proc_exception( "text_to_json", "Error when transforming raw data to JSON", - {"raw_data": raw_data}, + {}, exc, ) finally: diff --git a/src/codecarto/containers/processor/api/routers/plotter_router.py b/src/codecarto/containers/processor/api/routers/plotter_router.py index 7db5f54..d116a02 100644 --- a/src/codecarto/containers/processor/api/routers/plotter_router.py +++ b/src/codecarto/containers/processor/api/routers/plotter_router.py @@ -4,7 +4,7 @@ import mpld3 import matplotlib.lines as mlines -from api.util import generate_return, proc_exception +from api.util import generate_return, proc_exception, proc_error PlotterRoute: APIRouter = APIRouter() @@ -124,7 +124,7 @@ async def plot( results = grid_plot(graph) else: results = single_plot(graph=graph, title=layout, file_name=filename) - return generate_return("success", "Proc - Plot generated successfully", results) + return generate_return(200, "Proc - Plot generated successfully", results) except Exception as e: proc_exception( "plot", diff --git a/src/codecarto/containers/processor/api/routers/polygraph_router.py b/src/codecarto/containers/processor/api/routers/polygraph_router.py index 2de7235..230bea7 100644 --- a/src/codecarto/containers/processor/api/routers/polygraph_router.py +++ b/src/codecarto/containers/processor/api/routers/polygraph_router.py @@ -1,7 +1,7 @@ import httpx from fastapi import APIRouter -from api.util import generate_return, proc_exception +from api.util import generate_return, proc_exception, proc_error # DEBUG import logging @@ -21,7 +21,7 @@ async def get_graph_desc() -> dict: graph_desc: dict = get_graph_description() return generate_return( - "success", + 200, "Graph description successfully fetched from processor.", graph_desc, ) @@ -46,7 +46,7 @@ async def raw_to_json(file_url: str) -> dict: filename = file_url.split("/")[-1] graph = raw_to_graph(raw_data, filename) json_data = graph_to_json(graph) - return generate_return("success", "Proc - Success", json_data) + return generate_return(200, "Proc - Success", json_data) except Exception as exc: proc_exception( "raw_to_json", @@ -62,20 +62,22 @@ async def read_raw_data_from_url(url: str) -> str: try: logger.info(f" Started Proc.read_raw_data_from_url(): url - {url}") if not url.endswith(".py"): - proc_exception( + return proc_error( "read_raw_data_from_url", "URL is not a valid Python file", {"url": url}, + 404, ) client = httpx.AsyncClient() response = await client.get(url) if response.status_code == 200: return response.text else: - proc_exception( + return proc_error( "read_raw_data_from_url", "Could not read raw data from URL", {"url": url}, + 404, ) except Exception as exc: proc_exception( diff --git a/src/codecarto/containers/processor/api/util.py b/src/codecarto/containers/processor/api/util.py index 6ed7fe7..0cf6a8b 100644 --- a/src/codecarto/containers/processor/api/util.py +++ b/src/codecarto/containers/processor/api/util.py @@ -3,15 +3,24 @@ logger = logging.getLogger(__name__) -def generate_return(status: str, message: str, results) -> dict: - logger.info(f"{message} - Results: {results}") +def generate_return(status: int = 200, message: str = "", results: str = {}): return { - "status": status, # success or error - "message": message, # friendly message - "results": results, # the actual results or the error message + "status": status, + "message": message, + "results": results, } +def proc_error(called_from: str, message: str, params: dict = {}, status: int = 500): + """Return error results when something is wrong but did not throw exception""" + # Generate msg based on proc error status + error_message = f"\n\n\tProc.{called_from}() \n\tstatus: {status} \n\tmessage: {message} \n\tparam: {params} \n" + logger.error(f"{error_message}") + + # return the error + return generate_return(status=status, message=message, results=params) + + def proc_exception( called_from: str, message: str, @@ -19,22 +28,25 @@ def proc_exception( exc: Exception = None, status: int = 500, ) -> dict: + """Raise an exception if there is an exception thrown in processor""" import traceback from fastapi import HTTPException # log the error and stack trace - error_message = f"Proc.{called_from}() - status: {status} - param: {params} - message: {message}" + error_message = f"\n\n\tProc.{called_from}() \n\tstatus: {status} message: {message} \n\tparam: {params}\n" logger.error(error_message) + + # create a stack trace if exc: - error_message = f"{error_message} - exception: {str(exc)}" + error_message = f"\tProc.exception: {str(exc)} \n\tmessage:{error_message}\n" tbk_str = traceback.format_exception(type(exc), exc, exc.__traceback__) tbk_str = "".join(tbk_str) - logger.error(tbk_str) + logger.error(f"\n\t{tbk_str}") # raise the exception - if status == 404: + if status != 500: raise HTTPException( - status_code=404, + status_code=status, detail=error_message, ) else: diff --git a/src/codecarto/containers/web/Dockerfile b/src/codecarto/containers/web/Dockerfile index cddb762..265194c 100644 --- a/src/codecarto/containers/web/Dockerfile +++ b/src/codecarto/containers/web/Dockerfile @@ -10,7 +10,8 @@ ENV PYTHONUNBUFFERED=1 # Install pip requirements COPY ./src/codecarto/containers/web/requirements.txt . -RUN python -m pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install -r requirements.txt # Directory WORKDIR /app @@ -19,9 +20,13 @@ COPY ./src/codecarto/containers/web/src /app/src ENV PYTHONPATH=/app # # Creates a non-root user with an explicit UID and adds permission to access the /app folder -# # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers # RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app # USER appuser +# # Good idea to also remove the shell access and ensure there is no home directory for the user +# RUN addgroup --gid 1001 --system app && \ +# adduser --no-create-home --shell /bin/false --disabled-password --uid 1001 --system --group app +# USER app + # # During debugging, this entry point will be overridden. # For more information, please refer to https://aka.ms/vscode-docker-python-debug diff --git a/src/codecarto/containers/web/api/routers/palette_router.py b/src/codecarto/containers/web/api/routers/palette_router.py index 7b43b14..2d08c12 100644 --- a/src/codecarto/containers/web/api/routers/palette_router.py +++ b/src/codecarto/containers/web/api/routers/palette_router.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Request from fastapi.templating import Jinja2Templates -from api.util import generate_return, web_exception +from api.util import generate_return, web_exception, proc_error # Create a router PaletteRoute: APIRouter = APIRouter() @@ -25,22 +25,31 @@ async def get_palette() -> dict: async with httpx.AsyncClient() as client: try: response = await client.get(PROC_API_GET_PALETTE) - response.raise_for_status() - if not response.status_code == 200: - web_exception( + # returning a json response from the processor + # even in the case of an error in the processor + data: dict = response.json() + status_code = data.get("status", 500) + + # check if the response is an error + if status_code != 200: + error_message = data.get("message", "No error message") + results = proc_error( "get_palette", - "Could not fetch palette from processor", + "Error from processor", {}, + status=status_code, + proc_error=error_message, ) - return response.json() + else: + results = data + + return results except Exception as exc: - error_message = exc.response.json().get("detail", str(exc)) web_exception( "get_palette", - "Error from processor", + "Error with request to processor", {}, exc, - proc_error=error_message, ) diff --git a/src/codecarto/containers/web/api/routers/parser_router.py b/src/codecarto/containers/web/api/routers/parser_router.py index 7ffbbf2..acab299 100644 --- a/src/codecarto/containers/web/api/routers/parser_router.py +++ b/src/codecarto/containers/web/api/routers/parser_router.py @@ -2,7 +2,12 @@ from fastapi import APIRouter, Request from fastapi.templating import Jinja2Templates -from api.util import generate_return, web_exception +from api.util import generate_return, web_exception, proc_error + +# Debug +import logging + +logger = logging.getLogger(__name__) # Create a router ParserRoute: APIRouter = APIRouter() @@ -27,7 +32,7 @@ async def handle_github_url(github_url: str) -> dict: # TODO: call the proc api to start it, will get a job id # TODO: check the database every X secs on job id for results # TODO: Temp work around to see if working - async with httpx.AsyncClient(timeout=60.0) as client: + async with httpx.AsyncClient(timeout=180.0) as client: try: response = await client.get( PROC_API_GITHUB_URL, @@ -35,20 +40,32 @@ async def handle_github_url(github_url: str) -> dict: "github_url": github_url, }, ) - response.raise_for_status() - if not response.status_code == 200: - web_exception( + + # returning a json response from the processor + # even in the case of an error in the processor + data: dict = response.json() + status_code = data.get("status", 500) + + # check if the response is an error + if status_code != 200: + error_message = data.get("message", "No error message") + results = proc_error( "handle_github_url", - "Could not fetch github contents from processor", + "Error from processor", {"github_url": github_url}, + status=status_code, + proc_error=error_message, ) - return response.json() + else: + results = data + + return results except Exception as exc: - error_message = exc.response.json().get("detail", str(exc)) + # should only get here if there + # is an error with web container web_exception( "handle_github_url", - "Error from processor", + "Error with request to processor", {"github_url": github_url}, exc, - proc_error=error_message, ) diff --git a/src/codecarto/containers/web/api/routers/plotter_router.py b/src/codecarto/containers/web/api/routers/plotter_router.py index dc38e70..a0a8094 100644 --- a/src/codecarto/containers/web/api/routers/plotter_router.py +++ b/src/codecarto/containers/web/api/routers/plotter_router.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Request from fastapi.templating import Jinja2Templates -from api.util import generate_return, web_exception +from api.util import generate_return, web_exception, proc_error # Create a router PlotterRoute: APIRouter = APIRouter() @@ -88,20 +88,32 @@ async def plot( try: response = await client.get(PROC_API_PLOT, params=params) - response.raise_for_status() - if not response.status_code == 200: - web_exception( + + # returning a json response from the processor + # even in the case of an error in the processor + data: dict = response.json() + status_code = data.get("status", 500) + + # check if the response is an error + if status_code != 200: + error_message = data.get("message", "No error message") + results = proc_error( "plot", - "Could not fetch plot from processor", + "Error from processor", params, + status=status_code, + proc_error=error_message, ) - return response.json() + else: + results = data + + return results except Exception as exc: - error_message = exc.response.json().get("detail", str(exc)) + # should only get here if there + # is an error with web container web_exception( "plot", - "Error from processor", + "Error with request to processor", params, exc, - proc_error=error_message, ) diff --git a/src/codecarto/containers/web/api/routers/polygraph_router.py b/src/codecarto/containers/web/api/routers/polygraph_router.py index 441ef68..fecfbf9 100644 --- a/src/codecarto/containers/web/api/routers/polygraph_router.py +++ b/src/codecarto/containers/web/api/routers/polygraph_router.py @@ -2,7 +2,7 @@ from fastapi import APIRouter from fastapi.templating import Jinja2Templates -from api.util import generate_return, web_exception +from api.util import generate_return, web_exception, proc_error # Create a router PolyGraphRoute: APIRouter = APIRouter() @@ -20,21 +20,31 @@ async def get_graph_desc() -> dict: async with httpx.AsyncClient() as client: try: response = await client.get(PROC_API_GRAPH_DESC) - response.raise_for_status() - if not response.status_code == 200: - web_exception( + # returning a json response from the processor + # even in the case of an error in the processor + data: dict = response.json() + status_code = data.get("status", 500) + + # check if the response is an error + if status_code != 200: + error_message = data.get("message", "No error message") + results = proc_error( "get_graph_desc", - "Could not fetch graph description from processor", + "Error from processor", + {}, + status=status_code, + proc_error=error_message, ) - return response.json() + else: + results = data + + return results except Exception as exc: - error_message = exc.response.json().get("detail", str(exc)) web_exception( "get_graph_desc", - "Error from processor", + "Error with request to processor", {}, exc, - proc_error=error_message, ) @@ -48,14 +58,29 @@ async def raw_to_json(file_url: str) -> dict: "file_url": file_url, }, ) - response.raise_for_status() - return response.json() + # returning a json response from the processor + # even in the case of an error in the processor + data: dict = response.json() + status_code = data.get("status", 500) + + # check if the response is an error + if status_code != 200: + error_message = data.get("message", "No error message") + results = proc_error( + "raw_to_json", + "Error from processor", + {}, + status=status_code, + proc_error=error_message, + ) + else: + results = data + + return results except Exception as exc: - error_message = exc.response.json().get("detail", str(exc)) web_exception( "raw_to_json", - "Error from processor", + "Error with request to processor", {}, exc, - proc_error=error_message, ) diff --git a/src/codecarto/containers/web/api/util.py b/src/codecarto/containers/web/api/util.py index 2fb1be6..44d0eba 100644 --- a/src/codecarto/containers/web/api/util.py +++ b/src/codecarto/containers/web/api/util.py @@ -1,37 +1,71 @@ -def generate_return(status: str, message: str, results: str): +import logging + +logger = logging.getLogger(__name__) + + +def generate_return(status: int = 200, message: str = "", results: str = {}): return { - "status": status, # success or error - "message": message, # friendly message - "results": results, # the actual results or the error message + "status": status, + "message": message, + "results": results, } +def proc_error( + called_from: str, + message: str, + params: dict = {}, + status: int = 500, + proc_error: str = "", +): + """Return error results when something is wrong in processor container but did not throw exception""" + # Generate msg based on proc error status + msg: str = "" + if status != 500: + if status == 403: + msg = "Github API - Forbidden" + elif status == 404: + msg = "URL Not Found" + else: + msg = {message} + + error_message = f"\n\n\tWeb.{called_from}() \n\tstatus: {status} \n\tmessage: {msg} \n\tparam: {params} \n\tproc_error: {proc_error}" + logger.error(f"{error_message}\n") + + # return the error + return generate_return( + status=status, + message=msg, + results={"proc_error": proc_error}, + ) + + def web_exception( called_from: str, message: str, params: dict = {}, exc: Exception = None, status: int = 500, - proc_error: str = "", ) -> dict: + """Raise an exception if there is an error with the web container""" import traceback - import logging from fastapi import HTTPException # log the error and stack trace - error_message = f"Web.{called_from}() - status: {status} - param: {params} - message: {message} - proc_error: {proc_error}" - logger = logging.getLogger(__name__) + error_message = f"\n\n\Web.{called_from}() \n\tstatus: {status} message: {message} \n\tparam: {params}\n" logger.error(error_message) + + # create a stack trace if exc: - error_message = f"{error_message} - exception: {str(exc)}" + error_message = f"\Web.exception: {str(exc)} \n\tmessage:{error_message}\n" tbk_str = traceback.format_exception(type(exc), exc, exc.__traceback__) tbk_str = "".join(tbk_str) - logger.error(tbk_str) + logger.error(f"\n\t{tbk_str}") # raise the exception - if status == 404: + if status != 500: raise HTTPException( - status_code=404, + status_code=status, detail=error_message, ) else: diff --git a/src/codecarto/containers/web/src/pages/palette/palette.js b/src/codecarto/containers/web/src/pages/palette/palette.js index 81d918f..559c5a3 100644 --- a/src/codecarto/containers/web/src/pages/palette/palette.js +++ b/src/codecarto/containers/web/src/pages/palette/palette.js @@ -9,6 +9,13 @@ async function getPalette() { const responseData = await response.json() if (response.ok) { + if (responseData.status !== 200) { + displayError( + 'pal_data', + responseData.message, + `Error with response data: ${responseData.detail}` + ) + } if (responseData.status === 'error') { console.error(`Error with response data: ${responseData.message}`) document.getElementById('pal_data').innerHTML = responseData.message diff --git a/src/codecarto/containers/web/src/pages/parse/parse.js b/src/codecarto/containers/web/src/pages/parse/parse.js index 830c36f..d25167a 100644 --- a/src/codecarto/containers/web/src/pages/parse/parse.js +++ b/src/codecarto/containers/web/src/pages/parse/parse.js @@ -9,8 +9,8 @@ async function getGraphDesc() { const responseData = await response.json() if (response.ok) { - // Check if the response is an error from the backend - if (responseData.status === 'error') { + // Check if responseData.status is not 200 + if (responseData.status !== 200) { displayError( 'graph_desc', responseData.message, @@ -87,7 +87,7 @@ async function handleGithubURL() { if (response.ok) { // Check if the response is an error from the backend - if (responseData.status === 'error') { + if (responseData.status !== 200) { displayError( 'url_content', responseData.message, diff --git a/src/codecarto/containers/web/src/pages/plot/plot.js b/src/codecarto/containers/web/src/pages/plot/plot.js index 168349c..1b248a0 100644 --- a/src/codecarto/containers/web/src/pages/plot/plot.js +++ b/src/codecarto/containers/web/src/pages/plot/plot.js @@ -11,6 +11,13 @@ if (fileUrlDiv) { 'fileUrl' ).innerHTML = `File:  ${fileName}` document.getElementById('fileUrl').style.display = 'inline' + + // check if the file ends with .py + if (!fileName.endsWith('.py')) { + document.getElementById('single').disabled = true + document.getElementById('grid').disabled = true + document.getElementById('plot').innerHTML = '
Invalid file type' + } } } else { console.error('fileUrlDiv not found') @@ -21,17 +28,22 @@ if (fileUrlDiv) { * @param {boolean} all - If true, plot all layouts. */ async function plot(all = false) { - // Clear plot and show spinner - document.getElementById('plot').innerHTML = '' - document.getElementById('plot_loader').style.display = 'inline' + try { + // Clear plot and show spinner + document.getElementById('plot').innerHTML = '' + document.getElementById('plot_loader').style.display = 'inline' - // Fetch plot data - const endpoint = generateEndpoint(all) - const responseData = await fetchPlotData(endpoint) - handlePlotResponse(responseData) + // Fetch plot data + const endpoint = generateEndpoint(all) + const responseData = await fetchPlotData(endpoint) + handlePlotResponse(responseData) - // Hide spinner - document.getElementById('plot_loader').style.display = 'none' + // Hide spinner + document.getElementById('plot_loader').style.display = 'none' + } catch (error) { + console.error('Error - plot.js - plot():', error) + document.getElementById('plot_loader').style.display = 'none' + } } /** @@ -41,36 +53,41 @@ async function plot(all = false) { * */ function generateEndpoint(all) { - // Get the selected layout - const layoutElement = document.getElementById('layouts') - const selectedLayout = all ? 'all' : layoutElement.value - - // Create a data object to send in the POST request - let postData = { - layout: selectedLayout, - file: '', - url: '', - debug: false, - } + try { + // Get the selected layout + const layoutElement = document.getElementById('layouts') + const selectedLayout = all ? 'all' : layoutElement.value - // If fileUrl is defined, add it to the postData object - if (window.fileUrl && window.fileUrl !== '') { - postData.url = window.fileUrl - } else { - // Otherwise, get the file selector value - const runElement = document.getElementById('files') - const selectedRun = runElement.value - postData.debug = selectedRun === 'debug' - if (!postData.debug) { - postData.file = selectedRun + // Create a data object to send in the POST request + let postData = { + layout: selectedLayout, + file: '', + url: '', + debug: false, } - } - // Construct the endpoint - let endpoint = `/plotter/plot?file=${postData.file}&url=${postData.url}&layout=${postData.layout}&debug=${postData.debug}` + // If fileUrl is defined, add it to the postData object + if (window.fileUrl && window.fileUrl !== '') { + postData.url = window.fileUrl + } else { + // Otherwise, get the file selector value + const runElement = document.getElementById('files') + const selectedRun = runElement.value + postData.debug = selectedRun === 'debug' + if (!postData.debug) { + postData.file = selectedRun + } + } - // Return the endpoint - return endpoint + // Construct the endpoint + let endpoint = `/plotter/plot?file=${postData.file}&url=${postData.url}&layout=${postData.layout}&debug=${postData.debug}` + + // Return the endpoint + return endpoint + } catch (error) { + console.error('Error - plot.js - generateEndpoint():', error) + document.getElementById('plot_loader').style.display = 'none' + } } /** @@ -92,6 +109,7 @@ async function fetchPlotData(endpoint) { } } catch (error) { console.error('Error - plot.js - fetchPlotData():', error) + document.getElementById('plot_loader').style.display = 'none' return { status: 'error', message: 'Network error' } } } @@ -101,16 +119,21 @@ async function fetchPlotData(endpoint) { * @param {object} responseData - The response data from the plot API. */ function handlePlotResponse(responseData) { - // Check the inner response status (derived from the web container) - if (responseData.status === 'error') { - console.error(`Error with response data: ${responseData.message}`) - document.getElementById('plot').innerHTML = responseData.message - } else { - console.log(`Received response: ${responseData.message}`) - // Style the plot HTML and insert it into the page - const plotHTML = responseData.results - let newPlotHTML = stylePlotHTML(plotHTML) - insertHTMLWithScripts('plot', newPlotHTML) + try { + // Check the inner response status (derived from the web container) + if (responseData.status === 'error') { + console.error(`Error with response data: ${responseData.message}`) + document.getElementById('plot').innerHTML = responseData.message + } else { + console.log(`Received response: ${responseData.message}`) + // Style the plot HTML and insert it into the page + const plotHTML = responseData.results + let newPlotHTML = stylePlotHTML(plotHTML) + insertHTMLWithScripts('plot', newPlotHTML) + } + } catch (error) { + console.error('Error - plot.js - handlePlotResponse():', error) + document.getElementById('plot_loader').style.display = 'none' } } @@ -120,25 +143,30 @@ function handlePlotResponse(responseData) { * @param {string} html - The HTML to insert. */ function insertHTMLWithScripts(divId, html) { - // Clear the existing content - const container = document.getElementById(divId) - container.innerHTML = '' - - // Insert the new content - const parser = new DOMParser() - const doc = parser.parseFromString(html, 'text/html') - container.appendChild(doc.body.firstChild) - - // Extract and append scripts - const scripts = doc.querySelectorAll('script') - scripts.forEach(oldScript => { - const newScript = document.createElement('script') - Array.from(oldScript.attributes).forEach(attr => - newScript.setAttribute(attr.name, attr.value) - ) - newScript.appendChild(document.createTextNode(oldScript.innerHTML)) - container.appendChild(newScript) - }) + try { + // Clear the existing content + const container = document.getElementById(divId) + container.innerHTML = '' + + // Insert the new content + const parser = new DOMParser() + const doc = parser.parseFromString(html, 'text/html') + container.appendChild(doc.body.firstChild) + + // Extract and append scripts + const scripts = doc.querySelectorAll('script') + scripts.forEach(oldScript => { + const newScript = document.createElement('script') + Array.from(oldScript.attributes).forEach(attr => + newScript.setAttribute(attr.name, attr.value) + ) + newScript.appendChild(document.createTextNode(oldScript.innerHTML)) + container.appendChild(newScript) + }) + } catch (error) { + console.error('Error - plot.js - insertHTMLWithScripts():', error) + document.getElementById('plot_loader').style.display = 'none' + } } /** @@ -147,13 +175,18 @@ function insertHTMLWithScripts(divId, html) { * @returns {string} The styled HTML. */ function stylePlotHTML(plotHTML) { - // background color - let newPlotHTML = plotHTML.replace( - /"axesbg": "#FFFFFF"/g, - '"axesbg": "#e8dbad"' - ) - // font size - newPlotHTML = newPlotHTML.replace(/"fontsize": 12.0/g, '"fontsize": 30.0') - // return new HTML - return newPlotHTML + try { + // background color + let newPlotHTML = plotHTML.replace( + /"axesbg": "#FFFFFF"/g, + '"axesbg": "#e8dbad"' + ) + // font size + newPlotHTML = newPlotHTML.replace(/"fontsize": 12.0/g, '"fontsize": 30.0') + // return new HTML + return newPlotHTML + } catch (error) { + console.error('Error - plot.js - stylePlotHTML():', error) + document.getElementById('plot_loader').style.display = 'none' + } }