feat: evaluate api

backend/Makefile

@@ -18,4 +18,8 @@ run_dev_server:
 
 run_dev_celery_worker:
 	@echo "Running celery..."
-	@rye run celery -A app.celery worker
+	@rye run celery -A app.celery worker -Q default
+
+run_eval_dev_celery_worker:
+	@echo "Running evaluation celery..."
+	@rye run celery -A app.celery worker -Q evaluation --loglevel=debug --pool=solo

backend/app/alembic/env.py

@@ -7,8 +7,11 @@
 
 from app.core.config import settings
 from app.models import *  # noqa
-from app.models.knowledge_base_scoped.table_naming import KB_CHUNKS_TABLE_PATTERN, KB_ENTITIES_TABLE_PATTERN, \
-    KB_RELATIONSHIPS_TABLE_PATTERN
+from app.models.knowledge_base_scoped.table_naming import (
+    KB_CHUNKS_TABLE_PATTERN,
+    KB_ENTITIES_TABLE_PATTERN,
+    KB_RELATIONSHIPS_TABLE_PATTERN,
+)
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -60,7 +63,7 @@ def run_migrations_offline():
         target_metadata=target_metadata,
         include_name=include_name,
         literal_binds=True,
-        compare_type=True
+        compare_type=True,
     )
 
     with context.begin_transaction():
@@ -88,7 +91,7 @@ def run_migrations_online():
             connection=connection,
             target_metadata=target_metadata,
             include_name=include_name,
-            compare_type=True
+            compare_type=True,
         )
 
         with context.begin_transaction():

backend/app/alembic/versions/27a6723b767a_.py

@@ -5,28 +5,37 @@
 Create Date: 2024-11-29 20:38:05.773083
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
 from tidb_vector.sqlalchemy import VectorType
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.
-revision = '27a6723b767a'
-down_revision = 'd2ad44deab20'
+revision = "27a6723b767a"
+down_revision = "d2ad44deab20"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('chat_messages', sa.Column('is_best_answer', sa.Boolean(), server_default='0', nullable=False))
-    op.create_index('ix_chat_message_is_best_answer', 'chat_messages', ['is_best_answer'], unique=False)
+    op.add_column(
+        "chat_messages",
+        sa.Column("is_best_answer", sa.Boolean(), server_default="0", nullable=False),
+    )
+    op.create_index(
+        "ix_chat_message_is_best_answer",
+        "chat_messages",
+        ["is_best_answer"],
+        unique=False,
+    )
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index('ix_chat_message_is_best_answer', table_name='chat_messages')
-    op.drop_column('chat_messages', 'is_best_answer')
+    op.drop_index("ix_chat_message_is_best_answer", table_name="chat_messages")
+    op.drop_column("chat_messages", "is_best_answer")
     # ### end Alembic commands ###

backend/app/alembic/versions/749767db5505_add_recommend_questions.py

@@ -5,36 +5,58 @@
 Create Date: 2024-10-15 16:02:14.203584
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
 from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.
-revision = '749767db5505'
-down_revision = '8093333c0d87'
+revision = "749767db5505"
+down_revision = "8093333c0d87"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('recommend_questions',
-    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
-    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('questions', sa.JSON(), nullable=True),
-    sa.Column('chat_message_id', sa.Integer(), nullable=False),
-    sa.ForeignKeyConstraint(['chat_message_id'], ['chat_messages.id'], ),
-    sa.PrimaryKeyConstraint('id')
+    op.create_table(
+        "recommend_questions",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("questions", sa.JSON(), nullable=True),
+        sa.Column("chat_message_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chat_message_id"],
+            ["chat_messages.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_recommend_questions_chat_message_id"),
+        "recommend_questions",
+        ["chat_message_id"],
+        unique=False,
     )
-    op.create_index(op.f('ix_recommend_questions_chat_message_id'), 'recommend_questions', ['chat_message_id'], unique=False)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_recommend_questions_chat_message_id'), table_name='recommend_questions')
-    op.drop_table('recommend_questions')
+    op.drop_index(
+        op.f("ix_recommend_questions_chat_message_id"), table_name="recommend_questions"
+    )
+    op.drop_table("recommend_questions")
     # ### end Alembic commands ###

backend/app/alembic/versions/a54f966436ce_evaluation.py

@@ -0,0 +1,140 @@
+"""evaluation
+
+Revision ID: a54f966436ce
+Revises: 27a6723b767a
+Create Date: 2024-12-09 16:46:21.077517
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from tidb_vector.sqlalchemy import VectorType
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "a54f966436ce"
+down_revision = "27a6723b767a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "evaluation_datasets",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
+        sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "evaluation_tasks",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
+        sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
+        sa.Column("dataset_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "evaluation_dataset_items",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("query", sa.Text(), nullable=True),
+        sa.Column("reference", sa.Text(), nullable=True),
+        sa.Column("retrieved_contexts", sa.JSON(), nullable=True),
+        sa.Column("extra", sa.JSON(), nullable=True),
+        sa.Column("evaluation_dataset_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["evaluation_dataset_id"],
+            ["evaluation_datasets.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "evaluation_task_items",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "chat_engine", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False
+        ),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("query", sa.Text(), nullable=True),
+        sa.Column("reference", sa.Text(), nullable=True),
+        sa.Column("response", sa.Text(), nullable=True),
+        sa.Column("retrieved_contexts", sa.JSON(), nullable=True),
+        sa.Column("extra", sa.JSON(), nullable=True),
+        sa.Column("error_msg", sa.Text(), nullable=True),
+        sa.Column("factual_correctness", sa.Float(), nullable=True),
+        sa.Column("semantic_similarity", sa.Float(), nullable=True),
+        sa.Column("evaluation_task_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["evaluation_task_id"],
+            ["evaluation_tasks.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("evaluation_task_items")
+    op.drop_table("evaluation_dataset_items")
+    op.drop_table("evaluation_tasks")
+    op.drop_table("evaluation_datasets")
+    # ### end Alembic commands ###

backend/app/alembic/versions/c7f016a904c1_.py

@@ -5,26 +5,27 @@
 Create Date: 2024-10-30 13:28:17.345385
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
 from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.
-revision = 'c7f016a904c1'
-down_revision = '749767db5505'
+revision = "c7f016a904c1"
+down_revision = "749767db5505"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('chat_messages', sa.Column('meta', sa.JSON(), nullable=True))
+    op.add_column("chat_messages", sa.Column("meta", sa.JSON(), nullable=True))
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('chat_messages', 'meta')
+    op.drop_column("chat_messages", "meta")
     # ### end Alembic commands ###