Merge pull request #61 from BharatSahAIyak/dev

Dev -> Main v3.5.0
BharatSahAIyak · May 20, 2024 · 4ffc163 · 4ffc163
2 parents 45d20ad + 28a098a
commit 4ffc163
Show file tree

Hide file tree

Showing 44 changed files with 2,871 additions and 522 deletions.
diff --git a/autotune/redis.py b/autotune/redis.py
@@ -1,8 +1,4 @@
-from django.conf import settings
 import redis
+from django.conf import settings
 
-redis_conn = redis.Redis(
-    host=settings.REDIS_HOST,
-    port=settings.REDIS_PORT,
-    db=settings.REDIS_DB
-)
+redis_conn = redis.Redis.from_url(settings.REDIS_URL)
diff --git a/autotune/settings.py b/autotune/settings.py
@@ -10,9 +10,11 @@
 https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 
+import logging
 import os
 from pathlib import Path
 
+import coloredlogs
 import dj_database_url
 from dotenv import load_dotenv
 
@@ -59,9 +61,15 @@
 
 HUGGING_FACE_USERNAME = os.getenv("HUGGING_FACE_USERNAME")
 
-MAX_CONCURRENT_FETCHES = os.getenv("MAX_CONCURRENT_FETCHES")
+MINIO_BASE_URL = os.getenv("MINIO_BASE_URL")
 
-MAX_ITERATIONS = os.getenv("MAX_ITERATIONS")
+MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY")
+
+MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
+
+MINIO_SECURE_CONN = os.getenv("MINIO_SECURE_CONN") == "True"
+
+MINIO_BUCKET_NAME = os.getenv("MINIO_BUCKET")
 
 # Application definition
 
@@ -75,6 +83,7 @@
     "drf_yasg",
     "rest_framework",
     "workflow",
+    "workflowV2",
 ]
 
 MIDDLEWARE = [
@@ -171,7 +180,7 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/1",
+        "LOCATION": REDIS_URL,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
         },
@@ -190,17 +199,43 @@
 
 CSRF_TRUSTED_ORIGINS = ["http://localhost:8000"]
 
+level_styles = {
+    "info": {"color": "green"},  # Info logs are green
+    "warning": {"color": "yellow"},
+    "error": {"color": "red"},
+    "critical": {"color": "red", "bold": True},  # Critical logs in bold red
+}
+
+coloredlogs.install(
+    level="INFO",
+    logger=logging.getLogger(),  # Root logger
+    fmt="%(levelname)s - %(name)s - %(asctime)s - %(message)s",
+    level_styles=level_styles,  # Apply custom level styles
+)
+
 
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
+            "formatter": "colored",
         },
     },
-    "root": {
-        "handlers": ["console"],
-        "level": "INFO",
+    "formatters": {
+        "verbose": {
+            "format": "%(asctime)s [%(levelname)s] %(message)s (%(filename)s:%(lineno)s)",
+        },
+        "colored": {
+            "format": "%(asctime)s [%(levelname)s] %(message)s (%(filename)s:%(lineno)s)",
+        },
+    },
+    "loggers": {
+        "django.request": {
+            "handlers": ["console"],  # Add both file and console handlers
+            "level": "DEBUG",
+            "propagate": True,
+        },
     },
 }
diff --git a/autotune/urls.py b/autotune/urls.py
@@ -49,6 +49,7 @@
         name="schema-swagger-ui",
     ),
     path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
-    path("workflow/", include("workflow.urls")),
     path("task/status/<uuid:task_id>/", TaskView.as_view(), name="task-status"),
+    path("v1/workflow/", include(("workflow.urls", "workflow"), namespace="v1")),
+    path("v2/workflow/", include(("workflowV2.urls", "workflowV2"), namespace="v2")),
 ]
diff --git a/docs/API v2/GET status.md b/docs/API v2/GET status.md
@@ -0,0 +1,53 @@
+# Endpoint
+
+GET /status-v2/
+
+# Request
+
+### Headers
+
+- workflow_id
+- task_id
+
+```bash
+curl --location 'localhost:8000/status-v2/'
+```
+
+# Response
+
+### task_id is provided
+
+```json
+{
+  "task_id": "UUID",
+  "status": "",
+  "percentage": ""
+}
+```
+
+### workflow_id is provided
+
+```json
+{
+  "workflow_id": "UUID",
+  "status": "",
+  "task_ids": []
+}
+```
+
+### If both task_id and workflow_id is provided
+
+```json
+{
+  "workflow": {
+    "workflow_id": "UUID",
+    "status": "",
+    "task_ids": []
+  },
+  "task": {
+    "task_id": "UUID",
+    "status": "",
+    "percentage": ""
+  }
+}
+```
diff --git a/docs/API v2/GET workflow.md b/docs/API v2/GET workflow.md
@@ -0,0 +1,83 @@
+- Will return the workflows for a given user
+
+# Endpoint
+
+GET /workflow-v2 and GET /workflow-v2/{workflow_id}
+
+# Request
+
+### Headers
+
+- User Auth
+
+# Response
+
+```json
+[
+  {
+    "workflow_id": "<UUID>",
+    "workflow_name": "String",
+    "user_id": "<UUID>",
+    "config": {
+      "config_name": "QnA",
+      "system_prompt": "You are a helpful data generation assistant working as a teacher. You are an expert in this field. Don't Hallucinate",
+      "user_prompt_template": "{{workflow.user_prompt}}",
+      "temperature": 1,
+      "schema_example": {
+        "question": "4 + 5",
+        "choices": [
+          {
+            "text": "9",
+            "score": "1"
+          },
+          {
+            "text": "4",
+            "score": "0"
+          },
+          {
+            "text": "2",
+            "score": "0"
+          },
+          {
+            "text": "1",
+            "score": "0"
+          }
+        ]
+      }
+    },
+    "split": [100, 0, 0],
+    "llm_model": "LLM Models",
+    "tags": [],
+    "user_prompt": "",
+    "cost": "",
+    "estimated_dataset_cost": "",
+    "examples": [
+      {
+        "text": {
+          "question": "question text",
+          "choices": [
+            {
+              "text": "9",
+              "score": "1"
+            },
+            {
+              "text": "4",
+              "score": "0"
+            },
+            {
+              "text": "2",
+              "score": "0"
+            },
+            {
+              "text": "1",
+              "score": "0"
+            }
+          ]
+        }
+      }
+    ]
+  }
+]
+```
+
+Dataset Cost is estimated based on the tokens used in the previous iteration.
diff --git a/docs/API v2/POST data.md b/docs/API v2/POST data.md
@@ -0,0 +1,53 @@
+Returns the dataset associated with a given id
+
+# Endpoint
+
+POST /workflow-v2/data
+
+# Request
+
+### Headers
+
+- User Auth
+- workflow_id
+- task_id
+
+need to send either one of `workflow_id` or `task_id`
+
+# Response
+
+### If workflow_id is provided:
+
+returns dataset for all task_ids associated with the given workflow_id
+
+```json
+{
+  "workflow_id": "",
+  "data": [
+    {
+      "task_id": "",
+      "dataset": "MINIO LINK for the csv of a dataset"
+    },
+    {
+      "task_id": "",
+      "dataset": "MINIO LINK for the csv of a dataset"
+    }
+  ]
+}
+```
+
+### If task_id is provided
+
+dataset for just one task_id is provided
+
+```json
+{
+  "workflow_id": "",
+  "data": [
+    {
+      "task_id": "",
+      "dataset": "MINIO LINK for the csv of a dataset"
+    }
+  ]
+}
+```
diff --git a/docs/API v2/POST generate.md b/docs/API v2/POST generate.md
@@ -0,0 +1,22 @@
+# Endpoint
+
+/workflow-v2/generate/{workflow_id}
+
+# Request
+
+### Headers
+
+- User Auth
+
+# Response
+
+```json
+{
+  "expected_cost": "",
+  "task_id": ""
+}
+```
+
+## Implementation Details
+
+The prompt which goes to GPT will only include those examples which have been provided a reason.