src/documentation/how_tos/how_to_evaluate_runs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from example_data import DummyEvaluationLogic, example_data\n",
    "\n",
    "from intelligence_layer.evaluation import Evaluator, InMemoryEvaluationRepository"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to evaluate runs\n",
    "0. Run your tasks on the datasets where you want to evaluate them on (see [here](./how_to_run_a_task_on_a_dataset.ipynb))\n",
    "   - When evaluating multiple runs, all of them need the same data types \n",
    "2. Initialize all necessary repositories for the `Evaluator`, and an `EvaluationLogic`.\n",
    "3. Run the evaluator to evaluate all examples and create a single `EvaluationOverview`\n",
    "4. (Optional) Save the evaluation id for later use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 0\n",
    "\n",
    "my_example_data = example_data()\n",
    "print()\n",
    "run_ids = [my_example_data.run_overview_1.id, my_example_data.run_overview_2.id]\n",
    "\n",
    "# Step 1\n",
    "dataset_repository = my_example_data.dataset_repository\n",
    "run_repository = my_example_data.run_repository\n",
    "evaluation_repository = InMemoryEvaluationRepository()\n",
    "evaluation_logic = DummyEvaluationLogic()\n",
    "\n",
    "# Step 3\n",
    "evaluator = Evaluator(\n",
    "    dataset_repository,\n",
    "    run_repository,\n",
    "    evaluation_repository,\n",
    "    \"My dummy evaluation\",\n",
    "    evaluation_logic,\n",
    ")\n",
    "\n",
    "evaluation_overview = evaluator.evaluate_runs(\n",
    "    *run_ids, labels=set({\"label\"}), metadata=dict({\"key\": \"value\"})\n",
    ")\n",
    "\n",
    "# Step 4\n",
    "print(evaluation_overview.id)\n",
    "print(evaluation_overview.metadata)\n",
    "print(evaluation_overview.labels)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython"
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}