src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from documentation.how_tos.example_data import (\n",
    "    ComplexDummyAggregationLogic,\n",
    "    ComplexDummyEvaluationLogic,\n",
    "    ComplexDummyTask,\n",
    "    example_data,\n",
    ")\n",
    "from intelligence_layer.connectors import StudioClient\n",
    "from intelligence_layer.evaluation import (\n",
    "    StudioBenchmarkRepository,\n",
    "    StudioDatasetRepository,\n",
    ")\n",
    "\n",
    "load_dotenv()\n",
    "my_example_data = example_data()\n",
    "examples = my_example_data.complex_examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to execute Benchmarks\n",
    "<div class=\"alert alert-info\">  \n",
    "\n",
    "Make sure your account has permissions to use the Studio application.\n",
    "\n",
    "For an on-prem or local installation, please contact the corresponding team.\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0. Initialize a `StudioClient` with a project.\n",
    "    - Use an existing project or create a new one with the `StudioClient.create_project` function.\n",
    "    \n",
    "1. Create a `StudioDatasetRepository` and create a new `Dataset` via `StudioDatasetRepository.create_dataset`, which will automatically upload this new `Dataset` to Studio.\n",
    "\n",
    "2. Create a `StudioBenchmarkRepository` and instantiate a benchmark with your `evaluation_logic` and `aggregation_logic` using the `create_benchmark` function.\n",
    "\n",
    "3. Execute the `Benchmark` with your initialized `Task`\n",
    "\n",
    "### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 0\n",
    "\n",
    "studio_client = StudioClient(\n",
    "    project=my_example_data.studio_project_name, create_project=True\n",
    ")\n",
    "\n",
    "# Step 1\n",
    "studio_dataset_repository = StudioDatasetRepository(studio_client)\n",
    "dataset = studio_dataset_repository.create_dataset(examples, \"my_dataset\")\n",
    "\n",
    "# Step 2\n",
    "studio_benchmark_repository = StudioBenchmarkRepository(studio_client)\n",
    "evaluation_logic = ComplexDummyEvaluationLogic()\n",
    "aggregation_logic = ComplexDummyAggregationLogic()\n",
    "benchmark = studio_benchmark_repository.create_benchmark(\n",
    "    dataset.id, evaluation_logic, aggregation_logic, f\"my_benchmark-{uuid4()}\"\n",
    ")\n",
    "\n",
    "# Step 3\n",
    "\n",
    "task = ComplexDummyTask()\n",
    "benchmark.execute(task, \"my_task\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython"
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}