feature/litellm

.github/workflows/integration.yml

@@ -45,11 +45,11 @@ jobs:
     - name: Build Containers
       run: |
         cd integration
-        chmod -R 777 data/
+        chmod -R 777 .
         cp tests/.env.integration .env
         echo AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }} >> .env
         echo AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
-        docker compose up -d --wait minio db
+        docker compose up -d --wait minio db litellm
         docker compose up -d --wait django-app worker
         docker ps
 

Makefile

@@ -154,6 +154,12 @@ docker_build: ## Build the docker container
 	docker pull $$PREV_IMAGE; \
 	docker compose build lit-ssr;
 
+	echo "Building liellm..."; \
+	PREV_IMAGE="$(ECR_REPO_URL)-litellm:$(PREV_IMAGE_TAG)"; \
+	echo "Pulling previous image: $$PREV_IMAGE"; \
+	docker pull $$PREV_IMAGE; \
+	docker compose build litellm;
+
 
 
 

django_app/redbox_app/settings.py

@@ -389,6 +389,8 @@ def filter_transactions(event):
 GOOGLE_ANALYTICS_LINK = env.str("GOOGLE_ANALYTICS_LINK", " ")
 LIT_SSR_URL = env.str("LIT_SSR_URL", "localhost")
 
+LITELLM_MASTER_KEY = os.environ["LITELLM_MASTER_KEY"]
+LITELLM_URL = f"http://{os.environ["LITELLM_URL"]}:4000"
 
 MESSAGE_THROTTLE_SECONDS_MIN = env.int("MESSAGE_THROTTLE_SECONDS_MIN", 1)
 MESSAGE_THROTTLE_SECONDS_MAX = env.int("MESSAGE_THROTTLE_SECONDS_MAX", 10)

docker-compose.yml

@@ -9,6 +9,8 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      litellm:
+        condition: service_started
     networks:
       - redbox-app-network
     env_file:
@@ -111,6 +113,20 @@ services:
       retries: 24
       start_period: 30s
 
+  litellm:
+    image: ghcr.io/berriai/litellm:main-latest
+    env_file: .env
+    volumes:
+      - ./litellm_proxy_config.yml:/app/config.yml
+    networks:
+      - redbox-app-network
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - "4000:4000"
+    command: --config /app/config.yml --detailed_debug
+
 networks:
   redbox-app-network:
     driver: bridge

infrastructure/aws/data.tf

@@ -1,8 +1,10 @@
 locals {
   record_prefix     = terraform.workspace == "prod" ? var.project_name : "${var.project_name}-${terraform.workspace}"
   django_host       = "${local.record_prefix}.${var.domain_name}"
+  litellm_host      = "litellm-${local.record_prefix}.${var.domain_name}"
   name              = "${var.team_name}-${terraform.workspace}-${var.project_name}"
-  ssr_url = "${aws_service_discovery_service.lit_ssr_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
+  ssr_url           = "${aws_service_discovery_service.lit_ssr_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
+  litellm_url       = "${aws_service_discovery_service.litellm_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
 
 
   django_app_environment_variables = {
@@ -11,6 +13,10 @@ locals {
 
     "LIT_SSR_URL": local.ssr_url,
 
+    "LITELLM_CONFIG_BUCKET_NAME": aws_s3_bucket.user_data.bucket,
+    "LITELLM_CONFIG_BUCKET_OBJECT_KEY": "litellm_proxy_config.yml",
+    "LITELLM_URL": local.litellm_url,
+
     "OBJECT_STORE" : "s3",
     "BUCKET_NAME" : aws_s3_bucket.user_data.bucket,
     "POSTGRES_DB" : module.rds.db_instance_name,
@@ -41,27 +47,56 @@ locals {
     "ALLOWED_EMAIL_DOMAINS": var.allowed_email_domains,
   }
 
+  litellm_environment_variables = {
+    "AZURE_OPENAI_MODEL" : var.azure_openai_model,
+
+    "LITELLM_CONFIG_BUCKET_NAME": aws_s3_bucket.user_data.bucket,
+    "LITELLM_CONFIG_BUCKET_OBJECT_KEY": "litellm_proxy_config.yml",
+    "LITELLM_URL": local.litellm_url,
+    "LITELLM_MASTER_KEY": var.litellm_master_key,
+
+    "BUCKET_NAME" : aws_s3_bucket.user_data.bucket,
+    "ENVIRONMENT" : upper(terraform.workspace),
+    "DEBUG" : terraform.workspace == "dev",
+  }
   django_app_secrets = {
     "ELASTIC__API_KEY" : var.elastic_api_key,
     "ELASTIC__CLOUD_ID" : var.cloud_id,
 
     "AZURE_OPENAI_API_KEY": var.azure_openai_api_key,
-    "AZURE_OPENAI_ENDPOINT" : var.azure_openai_endpoint,
+    "LITELLM_MASTER_KEY": var.azure_openai_api_key,
+
+    "AZURE_OPENAI_ENDPOINT" : "http://${local.litellm_url}:4000", #var.azure_openai_endpoint,
     "OPENAI_API_VERSION": var.openai_api_version,
-    "GOOGLE_APPLICATION_CREDENTIALS_JSON": var.google_application_credentials_json,
 
     "DJANGO_SECRET_KEY" : var.django_secret_key,
     "POSTGRES_PASSWORD" : module.rds.rds_instance_db_password,
     "POSTGRES_HOST" : module.rds.db_instance_address,
     "POSTGRES_USER" : module.rds.rds_instance_username,
+    "POSTGRES_URL": "postgresql://${module.rds.rds_instance_username}:${module.rds.rds_instance_db_password}@${module.rds.db_instance_address}:5432/redbox",
     "GOVUK_NOTIFY_API_KEY" : var.govuk_notify_api_key,
     "SENTRY_DSN" : var.sentry_dsn,
     "SLACK_NOTIFICATION_URL" : var.slack_url
     "ELASTIC__API_KEY" : var.elastic_api_key,
     "ELASTIC__CLOUD_ID" : var.cloud_id,
   }
 
+  litellm_secrets = {
+    "AZURE_OPENAI_API_KEY": var.azure_openai_api_key,
+    "AZURE_OPENAI_ENDPOINT" : var.azure_openai_endpoint,
+    "OPENAI_API_VERSION": var.openai_api_version,
+
+    "POSTGRES_PASSWORD" : module.rds.rds_instance_db_password,
+    "POSTGRES_HOST" : module.rds.db_instance_address,
+    "POSTGRES_USER" : module.rds.rds_instance_username,
+    "DATABASE_URL": "postgresql://${module.rds.rds_instance_username}:${module.rds.rds_instance_db_password}@${module.rds.db_instance_address}:5432/litellm",
+    "UI_USERNAME": "redbox",
+    "UI_PASSWORD": random_password.litellm_ui_password.result 
+  }
+
+
   reconstructed_django_secrets = [for k, _ in local.django_app_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.django-app-secret.arn}:${k}::" }]
+  reconstructed_litellm_secrets = [for k, _ in local.litellm_secrets : { name = k, valueFrom = "${aws_secretsmanager_secret.litellm-secret.arn}:${k}::" }]
 }
 
 data "terraform_remote_state" "vpc" {

infrastructure/aws/ecs.tf

@@ -17,6 +17,18 @@ resource "aws_route53_record" "type_a_record" {
   }
 }
 
+resource "aws_route53_record" "litellm" {
+  zone_id = var.hosted_zone_id
+  name    = local.litellm_host
+  type    = "A"
+
+  alias {
+    name                   = module.load_balancer.load_balancer_dns_name
+    zone_id                = module.load_balancer.load_balancer_zone_id
+    evaluate_target_health = true
+  }
+}
+
 resource "aws_service_discovery_private_dns_namespace" "private_dns_namespace" {
   name        = "${local.name}-internal"
   description = "redbox private dns namespace"
@@ -43,19 +55,54 @@ resource "aws_service_discovery_service" "lit_ssr_service_discovery_service" {
   }
 }
 
+resource "aws_service_discovery_service" "litellm_service_discovery_service" {
+  name = "${local.name}-litellm"
+
+  dns_config {
+    namespace_id = aws_service_discovery_private_dns_namespace.private_dns_namespace.id
+
+    dns_records {
+      ttl  = 10
+      type = "A"
+    }
+
+    routing_policy = "MULTIVALUE"
+  }
+
+  health_check_custom_config {
+    failure_threshold = 1
+  }
+}
+
 resource "aws_secretsmanager_secret" "django-app-secret" {
   name = "${local.name}-django-app-secret"
   tags = {
     "platform:secret-purpose" = "general"
   }
 }
 
+resource "aws_secretsmanager_secret" "litellm-secret" {
+  name = "${local.name}-litellm-secret"
+  tags = {
+    "platform:secret-purpose" = "general"
+  }
+}
+
+resource "random_password" "litellm_ui_password" {
+  length           = 16
+  special          = true
+}
 
 resource "aws_secretsmanager_secret_version" "django-app-json-secret" {
   secret_id     = aws_secretsmanager_secret.django-app-secret.id
   secret_string = jsonencode(local.django_app_secrets)
 }
 
+resource "aws_secretsmanager_secret_version" "litellm-json-secret" {
+  secret_id     = aws_secretsmanager_secret.litellm-secret.id
+  secret_string = jsonencode(local.litellm_secrets)
+}
+
 
 module "django-app" {
   # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash
@@ -162,6 +209,49 @@ module "lit-ssr" {
   certificate_arn           = data.terraform_remote_state.universal.outputs.certificate_arn
 }
 
+data "aws_lb_listener" "lb_listener_443" {
+  load_balancer_arn = module.load_balancer.alb_arn
+  port              = 443
+}
+
+module "litellm" {
+  # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash
+  #source                       = "../../i-dot-ai-core-terraform-modules//modules/infrastructure/ecs" # For testing local changes
+  source                        = "git::https://github.com/i-dot-ai/i-dot-ai-core-terraform-modules.git//modules/infrastructure/ecs?ref=v1.0.0-ecs"
+  service_discovery_service_arn = aws_service_discovery_service.litellm_service_discovery_service.arn
+  memory                        = 4096
+  cpu                           = 2048
+  create_listener               = false
+  create_networking             = true
+  name                          = "${local.name}-litellm"
+  image_tag                     = "main-latest"
+  ecr_repository_uri            = "ghcr.io/berriai/litellm"
+  ecs_cluster_id                = module.cluster.ecs_cluster_id
+  ecs_cluster_name              = module.cluster.ecs_cluster_name
+  autoscaling_minimum_target    = 1
+  autoscaling_maximum_target    = 1
+  health_check = {
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    accepted_response   = "200"
+    path                = "/test"
+    timeout             = 5
+  }
+  state_bucket                 = var.state_bucket
+  vpc_id                       = data.terraform_remote_state.vpc.outputs.vpc_id
+  private_subnets              = data.terraform_remote_state.vpc.outputs.private_subnets
+  host                         = local.litellm_host
+  container_port               = 4000
+  load_balancer_security_group = module.load_balancer.load_balancer_security_group_id
+  aws_lb_arn                   = module.load_balancer.alb_arn
+  https_listener_arn           = data.aws_lb_listener.lb_listener_443.arn
+  ephemeral_storage            = 30
+  environment_variables        = local.litellm_environment_variables
+  secrets                      = local.reconstructed_litellm_secrets
+  auto_scale_off_peak_times    = true
+  wait_for_ready_state         = true
+}
+
 
 resource "aws_security_group_rule" "ecs_ingress_django_to_lit_ssr" {
   type                     = "ingress"
@@ -173,3 +263,12 @@ resource "aws_security_group_rule" "ecs_ingress_django_to_lit_ssr" {
   security_group_id        = module.lit-ssr.ecs_sg_id
 }
 
+resource "aws_security_group_rule" "ecs_django_to_litellm" {
+  type                     = "ingress"
+  description              = "Allow all traffic from the django-app to litellm"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+  source_security_group_id = module.django-app.ecs_sg_id
+  security_group_id        = module.litellm.ecs_sg_id
+}

infrastructure/aws/iam.tf

@@ -42,7 +42,9 @@ data "aws_iam_policy_document" "ecs_exec_role_policy" {
     ]
     resources = [
       aws_secretsmanager_secret.django-app-secret.arn,
-      "${aws_secretsmanager_secret.django-app-secret.arn}:*"
+      "${aws_secretsmanager_secret.django-app-secret.arn}:*",
+      aws_secretsmanager_secret.litellm-secret.arn,
+      "${aws_secretsmanager_secret.litellm-secret.arn}:*"
     ]
   }
 }
@@ -58,6 +60,7 @@ resource "aws_iam_role_policy_attachment" "redbox_role_policy" {
     {
       "worker" = module.worker.ecs_task_execution_exec_role_name,
       "django" = module.django-app.ecs_task_execution_exec_role_name,
+      "litellm" = module.litellm.ecs_task_execution_exec_role_name,
     }
   )
   role       = each.value

infrastructure/aws/provider.tf

@@ -4,6 +4,11 @@ terraform {
       source  = "hashicorp/aws"
       version = ">= 4.2.0"
     }
+
+    random = {
+      source = "hashicorp/random"
+      version = ">=3.6.0"
+    }
   }
   required_version = ">= 1.3.5"
 

infrastructure/aws/rds.tf

@@ -1,7 +1,8 @@
 module "rds" {
   # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash
   # source = "../../../i-dot-ai-core-terraform-modules//modules/infrastructure/rds"  # For testing local changes
-  source                  = "git::https://github.com/i-dot-ai/i-dot-ai-core-terraform-modules.git//modules/infrastructure/rds?ref=v1.0.0-rds"
+  #source                  = "git::https://github.com/i-dot-ai/i-dot-ai-core-terraform-modules.git//modules/infrastructure/rds?ref=v1.0.0-rds"
+  source                  = "../core/rds/"
   name                    = local.name
   db_name                 = var.project_name
   domain_name             = var.domain_name

infrastructure/aws/variables.tf

@@ -64,6 +64,12 @@ variable "elastic_api_key" {
   description = "API Key for elastic cloud instance"
 }
 
+variable "litellm_master_key" {
+  type = string
+  description = "master key for lite-llm"
+  default = "sk-1234"
+}
+
 variable "env" {
   type        = string
   description = "Environment"