BugsInPy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b23e6d91-24da-4904-9e22-b225f7d9959c",
   "metadata": {},
   "source": [
    "# Experimental Evaluation with BugsInPy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8de1be4-4380-4d08-915d-e3147d0db268",
   "metadata": {},
   "source": [
    "To run these experiments you need to clone BugsInPy from [here](https://github.com/smythi93/BugsInPy). Install it as explained in the [README.md](https://github.com/smythi93/BugsInPy/blob/master/README.md)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e571e550-8bc7-4f7e-b873-76c9161af7d0",
   "metadata": {},
   "source": [
    "## General Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6fe1d278",
   "metadata": {},
   "source": [
    "!pip install whatthepatch"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "71944b13-ca1f-4f5b-b1dd-7f0447e01626",
   "metadata": {},
   "source": [
    "import json\n",
    "import hashlib\n",
    "import os\n",
    "import re\n",
    "import shutil\n",
    "import subprocess\n",
    "import time\n",
    "import whatthepatch\n",
    "from typing import List, Set\n",
    "from xml.etree import ElementTree as etree\n",
    "from sflkit.analysis.suggestion import Location"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "3c9d21f2-f316-4dbf-bfd7-dc307795981c",
   "metadata": {},
   "source": [
    "## Configuration of Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7a4db981-8306-4bb0-88d8-5b3f711adaee",
   "metadata": {},
   "source": [
    "subject_selection = False\n",
    "continue_subject_selection = True"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6d269bfe-e9eb-43ac-977f-6bed9080173e",
   "metadata": {},
   "source": [
    "events_extraction = True\n",
    "continue_events_extraction = True"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "41cfb43a-fa07-49d4-82e3-b1a0cb04971f",
   "metadata": {},
   "source": [
    "tmp = '/tmp'\n",
    "subjects_file = 'subjects.json'\n",
    "default_project = 'project'\n",
    "events_path = 'EVENTS_PATH'\n",
    "config_tmp = 'tmp.ini'"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "84c154e6-9bd3-4a46-9604-73a895457f45",
   "metadata": {},
   "source": [
    "def get_dir(p):\n",
    "    return os.path.join(tmp, p)\n",
    "\n",
    "def clean_project(project):\n",
    "    shutil.rmtree(get_dir(project), ignore_errors=True)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "54609932-e26f-4fd1-86b6-b6e49fa78fde",
   "metadata": {},
   "source": [
    "sflkit_dir = os.path.abspath('.')\n",
    "sflkit_dir"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "93d35204-e3a9-4911-a7cf-a588d08fc2ef",
   "metadata": {},
   "source": [
    "events_output = os.path.abspath('events')\n",
    "events_output"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "deb300c5-7fd1-4ada-9198-566cdd859b61",
   "metadata": {},
   "source": [
    "PASSED = 'PASSED'\n",
    "FAILED = 'FAILED'"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "271ee261-dd22-4759-ac89-f0afcb12d091",
   "metadata": {},
   "source": [
    "## BugsInPy Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "106b4a86-48a2-46b1-94fe-e447df2b7647",
   "metadata": {},
   "source": [
    "bip_info = 'bugsinpy-info'\n",
    "bip_checkout = 'bugsinpy-checkout'\n",
    "bip_compile = 'bugsinpy-compile'\n",
    "bip_test = 'bugsinpy-test'\n",
    "bip_sdtools = 'bugsinpy-sflkit'\n",
    "info_file = 'bugsinpy_bug.info'\n",
    "run_test_file = 'bugsinpy_run_test.sh'\n",
    "patch_files = [\n",
    "    'bugsinpy_bug_patch.txt',\n",
    "    'bugsinpy_patchfile.info'\n",
    "]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cb6dc70f-82cc-4475-9cbe-3d481e914cea",
   "metadata": {},
   "source": [
    "projects = [\n",
    "    'PySnooper',\n",
    "    'ansible',\n",
    "    'black',\n",
    "    'cookiecutter',\n",
    "    'fastapi',\n",
    "    'httpie',\n",
    "    'keras',\n",
    "    'luigi',\n",
    "    'matplotlib',\n",
    "    'pandas',\n",
    "    'sanic',\n",
    "    'scrapy',\n",
    "    'spacy',\n",
    "    'thefuck',\n",
    "    'tornado',\n",
    "    'tqdm',\n",
    "    'youtube-dl'\n",
    "]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "65b9d0a4-2d20-4326-88a2-8a7a6a9ee8f2",
   "metadata": {},
   "source": [
    "for project in projects:\n",
    "    globals()[project.replace('-', '_')] = project\n",
    "    clean_project(project)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "37e38835-1276-4db0-9a64-41d3e13df982",
   "metadata": {},
   "source": [
    "excluded_subjects = [matplotlib, pandas, spacy]\n",
    "max_bugs_per_subject = 500"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ce05a37b-ae11-4009-9e48-d1a5b65ff51a",
   "metadata": {},
   "source": [
    "test_project = PySnooper\n",
    "test_bug_id = 3"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "362b469d-6c69-4642-8932-a5ea97873716",
   "metadata": {},
   "source": [
    "number_of_bugs_pattern = re.compile(r'Number\\s*of\\s*bugs\\s*:\\s*(?P<bugs>\\d+)')\n",
    "\n",
    "def get_bugs(project):\n",
    "    process = subprocess.run([bip_info, '-p', project], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)\n",
    "    match = number_of_bugs_pattern.search(process.stdout.decode('utf8'))\n",
    "    if match:\n",
    "        return int(match.group('bugs'))"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b67456e2-3039-4a4d-a45c-bc9c40d3583d",
   "metadata": {},
   "source": [
    "assert get_bugs(test_project) == 3"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "97fb587c-8706-4436-94f9-7ff3dcd79e4b",
   "metadata": {},
   "source": [
    "test_file_pattern = re.compile(r'Triggering\\s*test\\s*file\\s*(?P<test_file>[^\\n]+)')\n",
    "\n",
    "def get_test_file(project: str, bug_id: int):\n",
    "    process = subprocess.run([bip_info, '-p', project, '-i', str(bug_id)], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)\n",
    "    match = test_file_pattern.search(process.stdout.decode('utf8'))\n",
    "    if match:\n",
    "        return match.group('test_file')"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f832b7ac-1b57-4c42-acf3-c05e2796850e",
   "metadata": {},
   "source": [
    "test_test_file = get_test_file(test_project, test_bug_id)\n",
    "assert test_test_file == 'tests/test_pysnooper.py'"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4d2c4307-626c-4c7c-8494-d641fbe5cfa0",
   "metadata": {},
   "source": [
    "def get_project(project: str, buggy: bool = True, bug_id: int = 0, delete: bool = False, verbose: bool = False):\n",
    "    if delete:\n",
    "        clean_project(project)\n",
    "    process = subprocess.run([bip_checkout, '-p', project, '-v', '0' if buggy else '1', '-i', str(bug_id), '-w', tmp], \n",
    "                             stdout=None if verbose else subprocess.DEVNULL,\n",
    "                             stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL)\n",
    "    assert process.returncode == 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a02a4869-32d7-4e92-bd5a-7651820f8332",
   "metadata": {},
   "source": [
    "def is_unittest(project):\n",
    "    with open(os.path.join(get_dir(project), run_test_file), 'r') as fp:\n",
    "        s = fp.read()\n",
    "    return 'unittest' in s"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9dea83dd-812b-43d7-8579-7963a3d6e6cf",
   "metadata": {},
   "source": [
    "version_pattern = re.compile(r'python_version=\"(?P<version>[^\\\"]+)\"')\n",
    "\n",
    "original_env = os.environ.copy()\n",
    "    \n",
    "def env_on(project: str, verbose=False):\n",
    "    if os.path.exists(os.path.join(get_dir(project), info_file)):\n",
    "        with open(os.path.join(get_dir(project), info_file), 'r') as fp:\n",
    "            s = fp.read()\n",
    "        match = version_pattern.search(s)\n",
    "        if match:\n",
    "            version = match.group('version')\n",
    "            os.environ = original_env.copy()\n",
    "            process = subprocess.run(['pyenv', 'install', version], \n",
    "                                     stdout=None if verbose else subprocess.DEVNULL,\n",
    "                                     stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL)\n",
    "            os.environ['PATH'] = f'{os.path.join(os.environ[\"PYENV_ROOT\"], \"versions\", version, \"bin\")}:{os.environ[\"PATH\"]}'\n",
    "            process = subprocess.run(['pip', 'install', '--upgrade', 'pip'], \n",
    "                                     stdout=None if verbose else subprocess.DEVNULL,\n",
    "                                     stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "21c43449-662d-42c9-b8e4-2ad5099948b9",
   "metadata": {},
   "source": [
    "def setup_project(project: str, verbose: bool = False):\n",
    "    env_on(project, verbose=verbose)\n",
    "    process = subprocess.run([bip_compile, '-w', get_dir(project)],  \n",
    "                             stdout=None if verbose else subprocess.DEVNULL,\n",
    "                             stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL, env=os.environ)\n",
    "    assert process.returncode == 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8fcfecfb-ddd0-4173-9b87-1da35a89ff87",
   "metadata": {},
   "source": [
    "def install_project(project: str, bug_id: int, buggy: bool = True, delete: bool = True, verbose: bool = False):\n",
    "    get_project(project, buggy=buggy, bug_id=bug_id, verbose=verbose)\n",
    "    setup_project(project, verbose=verbose)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1cf603b1-a683-4d47-8fe9-bb461722548c",
   "metadata": {},
   "source": [
    "NEWLINE_TOKEN = 'SDNEWLINE'\n",
    "\n",
    "def replace_important(s: str):\n",
    "    important = False\n",
    "    result = ''\n",
    "    escaped = False\n",
    "    while s:\n",
    "        if not important:\n",
    "            if s.startswith('name=\"'):\n",
    "                result += 'name=\"'\n",
    "                s = s[6:]\n",
    "                important = True\n",
    "            elif s.startswith('classname=\"'):\n",
    "                result += 'classname=\"'\n",
    "                s = s[11:]\n",
    "                important = True\n",
    "            else:\n",
    "                result += s[0]\n",
    "                s = s[1:]\n",
    "        else:\n",
    "            if s[0] == '\\n':\n",
    "                result += NEWLINE_TOKEN\n",
    "                s = s[1:]\n",
    "            elif s[0] == '\"' and not escaped:\n",
    "                result += '\"'\n",
    "                s = s[1:]\n",
    "                important = False\n",
    "            elif s[0] == '\\\\' and not escaped:\n",
    "                result += '\\\\'\n",
    "                s = s[1:]\n",
    "                escaped = True\n",
    "            else:\n",
    "                result += s[0]\n",
    "                s = s[1:]\n",
    "                escaped = False\n",
    "    return result\n",
    "\n",
    "def get_test_results(project: str, tmp_file: str):\n",
    "    tests = list()\n",
    "    is_unittest_ = is_unittest(project) \n",
    "    try:\n",
    "        with open(tmp_file, 'r') as fp:\n",
    "            s = fp.read()\n",
    "        tree = etree.fromstring(replace_important(s))\n",
    "    except FileNotFoundError:\n",
    "        print('pytest did not generate file')\n",
    "        return tests\n",
    "    except etree.ParseError:\n",
    "        print('pytest produced empty file')\n",
    "        return tests\n",
    "    directory = get_dir(project)\n",
    "    for testcase in tree.findall('.//testcase'):\n",
    "        if is_unittest_:\n",
    "            test = testcase.get('classname').replace(NEWLINE_TOKEN, '\\n') + '.' + testcase.get('name').replace(NEWLINE_TOKEN, '\\n')\n",
    "        else:\n",
    "            path = testcase.get('classname').replace(NEWLINE_TOKEN, '\\n').split(\".\")\n",
    "            file = ''\n",
    "            classes = '::'\n",
    "            for i in range(1, len(path) + 1):\n",
    "                file = os.path.join(*path[:i]) + '.py'\n",
    "                if os.path.exists(os.path.join(directory, file)):\n",
    "                    if len(path[i:]) > 0:\n",
    "                        classes = '::' + '::'.join(path[i:]) + '::'\n",
    "                    break\n",
    "            test = file + classes + testcase.get('name').replace(NEWLINE_TOKEN, '\\n')\n",
    "        if testcase.find('failure') is not None or testcase.find('error') is not None:\n",
    "            tests.append((test, FAILED))\n",
    "        elif len(list(testcase)) == 0 or (\n",
    "            (len(list(testcase)) == 1 and (testcase.find('system-out') is not None or testcase.find('system-err') is not None)) or\n",
    "            (len(list(testcase)) == 2 and (testcase.find('system-out') is not None and testcase.find('system-err') is not None))):\n",
    "            tests.append((test, PASSED))\n",
    "    return tests"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d6e48520-de25-44aa-8e80-6f6ab1a6f086",
   "metadata": {},
   "source": [
    "def get_tests(project: str, test_file: str, verbose: bool = False):\n",
    "    tmp_file = os.path.join(get_dir(project), 'tmp.xml')\n",
    "    try:\n",
    "        os.remove(tmp_file)\n",
    "    except OSError:\n",
    "        pass\n",
    "\n",
    "    process = subprocess.Popen([bip_test, '-t', test_file, '-o', tmp_file, '-w', get_dir(project)],  \n",
    "                               stdout=None if verbose else subprocess.DEVNULL,\n",
    "                               stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL, env=os.environ)\n",
    "    while process.poll() is None:\n",
    "        if os.path.exists(tmp_file):\n",
    "            try:\n",
    "                process.wait(5)\n",
    "            except subprocess.TimeoutExpired:\n",
    "                process.terminate()\n",
    "        try:\n",
    "            process.wait(1)\n",
    "        except subprocess.TimeoutExpired:\n",
    "            pass\n",
    "    passing, failing = list(), list()\n",
    "    for t, s in get_test_results(project, tmp_file):\n",
    "        if s == PASSED:\n",
    "            passing.append(t)\n",
    "        elif s == FAILED:\n",
    "            failing.append(t)\n",
    "    return passing, failing"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d22a6a38-393a-4e91-8f0a-fa513a87d37f",
   "metadata": {},
   "source": [
    "def verify_correct_version(project: str, bug_id: int, test_file: str, verbose: bool = False):\n",
    "    install_project(project, bug_id, buggy=False, verbose=verbose)\n",
    "    passing, failing = get_tests(project, test_file, verbose=verbose)\n",
    "    return len(failing) == 0 and len(passing) > 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7ddd63f4-5d9b-46e1-b1ad-e6bbe630cb6b",
   "metadata": {},
   "source": [
    "assert verify_correct_version(test_project, test_bug_id, test_test_file)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "589f7dd6-6c2d-4091-ab6e-e0af3180bf82",
   "metadata": {},
   "source": [
    "install_project(test_project, bug_id=test_bug_id)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "45ef8657-48f6-42a9-9e35-1280e2225ce2",
   "metadata": {},
   "source": [
    "assert not is_unittest(test_project)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0ab6bf87-130b-48e3-ba9d-75092a27f2d9",
   "metadata": {},
   "source": [
    "test_passing, test_failing = get_tests(test_project, test_test_file)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6639244a-b19c-4ebf-9dc3-8689268edb20",
   "metadata": {},
   "source": [
    "test_passing"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cffdf1ad-0e61-4370-af5d-4980dd8177fa",
   "metadata": {},
   "source": [
    "test_failing"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e9d91d53-dafe-4352-aa5a-5e2dd59a14a2",
   "metadata": {},
   "source": [
    "def run_test(project: str, test_case: str, verbose: bool = False):\n",
    "    passing, failing = get_tests(project, test_case, verbose=verbose)\n",
    "    return len(passing) > 0 and len(failing) == 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8867aec5-03e9-4658-88bc-749b4765f3a3",
   "metadata": {},
   "source": [
    "for t in test_passing:\n",
    "    assert run_test(test_project, t)\n",
    "for t in test_failing:\n",
    "    assert not run_test(test_project, t)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5ed1595e-dd34-42a3-a20c-ab026e452c3d",
   "metadata": {},
   "source": [
    "def get_faulty_lines(project: str):\n",
    "    locations = list()\n",
    "    for patch_file in patch_files:\n",
    "        try:\n",
    "            with open(os.path.join(get_dir(project), patch_file), 'r') as fp:\n",
    "                s = fp.read()\n",
    "            for diff in whatthepatch.parse_patch(s):\n",
    "                last = None\n",
    "                for change in diff.changes:\n",
    "                    if change.old is None:\n",
    "                        if last is None:\n",
    "                            last = change.new\n",
    "                            \n",
    "                    else:\n",
    "                        last = change.old\n",
    "                    location = Location(diff.header.old_path, last)\n",
    "                    if location not in locations:\n",
    "                        locations.append(location)\n",
    "        except (IOError, whatthepatch.exceptions.WhatThePatchException):\n",
    "            pass\n",
    "    return locations"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ca900924-c5a9-4a2e-a1c4-951d69618eb4",
   "metadata": {},
   "source": [
    "test_faulty_lines = get_faulty_lines(test_project)\n",
    "test_faulty_lines"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8e4b9e48-aa2f-4158-b7bc-735532c78c5c",
   "metadata": {},
   "source": [
    "class Subject:\n",
    "    def __init__(self, project: str, bug_id: int, passing: List[str], failing: List[str], is_unittest_: bool, faulty_lines: List[Location]):\n",
    "        self.project = project\n",
    "        self.bug_id = bug_id\n",
    "        self.passing = passing\n",
    "        self.failing = failing\n",
    "        self.is_unittest = is_unittest_\n",
    "        self.faulty_lines = faulty_lines\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return f'Subject(project={self.project},bugId={self.bug_id},#passing={len(self.passing)},#failing={len(self.failing)})'\n",
    "    \n",
    "    def __str__(self):\n",
    "        return repr(self)\n",
    "    \n",
    "    def __hash__(self):\n",
    "        return hash((self.project, self.bug_id))\n",
    "    \n",
    "    def __eq__(self, other):\n",
    "        return isinstance(other, Subject) and other.project == self.project and other.bug_id == self.bug_id\n",
    "        \n",
    "    def serialize(self):\n",
    "        return {\n",
    "            'project': self.project,\n",
    "            'bugId': self.bug_id,\n",
    "            'passing': self.passing,\n",
    "            'failing': self.failing,\n",
    "            'is_unittest': self.is_unittest,\n",
    "            'faulty_lines':  [(location.file, location.line) for location in self.faulty_lines],\n",
    "        }\n",
    "    \n",
    "    @staticmethod\n",
    "    def deserialize(values: dict):\n",
    "        return Subject(values['project'], values['bugId'], values['passing'], values['failing'], values['is_unittest'], [Location(file, line) for file, line in values['faulty_lines']])"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "864d96db-0323-44c5-82cc-cd1b11c5b415",
   "metadata": {},
   "source": [
    "def verify_project(project: str, bug_id: int, fast: bool = False, verbose=False) -> Subject:\n",
    "    install_project(project, bug_id=bug_id, verbose=verbose)\n",
    "    test_file = get_test_file(project, bug_id)\n",
    "    passing, failing = get_tests(project, test_file, verbose=verbose)\n",
    "    if len(failing) == 0:\n",
    "        print(f'X {project:<40}{bug_id:<3} - No failing tests on buggy version')\n",
    "    elif len(passing) == 0:\n",
    "        print(f'X {project:<40}{bug_id:<3} - No passing tests on buggy version')\n",
    "    else:\n",
    "        if fast or verify_correct_version(project, bug_id, test_file, verbose=verbose):\n",
    "            print(f'  {project:<40}{bug_id:<3}')    \n",
    "        else:\n",
    "            print(f'  {project:<40}{bug_id:<3} - Failing tests on correct version')\n",
    "        return Subject(project, bug_id, passing, failing, is_unittest(project), get_faulty_lines(project))"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "32ff893b-4149-496d-ab58-327f2212ffbb",
   "metadata": {},
   "source": [
    "test_subject = verify_project(test_project, test_bug_id)\n",
    "test_subject"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "94366ba5-55e0-4f71-961e-5ab8091b3199",
   "metadata": {},
   "source": [
    "test_subject.faulty_lines"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "025371d6-23e6-43c8-82de-37d504f5485e",
   "metadata": {},
   "source": [
    "def get_all_valid(found_subjects: list = None, fast: bool = False, excludes: list = None, max_number: int = None):\n",
    "    \n",
    "    if excludes is None:\n",
    "        excludes = list()\n",
    "    if found_subjects is None:\n",
    "        found_subjects = list()\n",
    "    subjects = found_subjects\n",
    "    try:\n",
    "        for p in projects:\n",
    "            if p not in excludes:\n",
    "                bug_count = len(list(filter(lambda s: s.project == p, subjects)))\n",
    "                for b in range(get_bugs(p)):\n",
    "                    if max_number and max_number > bug_count:\n",
    "                        if not any(s.project == p and s.bug_id == b + 1 for s in found_subjects):\n",
    "                            s = verify_project(p, b + 1, fast=fast)\n",
    "                            if s:\n",
    "                                subjects.append(s)\n",
    "                                bug_count += 1\n",
    "        return subjects\n",
    "    finally:\n",
    "        globals()['subjects'] = subjects"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1827e90b-3bf2-416b-b658-3667b312186e",
   "metadata": {},
   "source": [
    "if subject_selection:\n",
    "    if continue_subject_selection:\n",
    "        subjects = list()\n",
    "        if os.path.exists(subjects_file):\n",
    "            with open(subjects_file, 'r') as fp:\n",
    "                json_subjects = json.load(fp)\n",
    "            for values in json_subjects:\n",
    "                subjects.append(Subject.deserialize(values))\n",
    "        subjects = get_all_valid(subjects, fast=True, excludes=excluded_subjects, \n",
    "                                 max_number=max_bugs_per_subject)\n",
    "    else:\n",
    "        subjects = get_all_valid(fast=True, excludes=excludes)\n",
    "else:\n",
    "    subjects = list()\n",
    "    if os.path.exists(subjects_file):\n",
    "        with open(subjects_file, 'r') as fp:\n",
    "            json_subjects = json.load(fp)\n",
    "        for values in json_subjects:\n",
    "            subjects.append(Subject.deserialize(values))"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a0aa5102-a5d9-40c2-a35c-e65f00bc4855",
   "metadata": {},
   "source": [
    "if subject_selection:\n",
    "    with open(subjects_file, 'w') as fp:\n",
    "        json.dump(list(map(Subject.serialize, subjects)), fp, indent = 4)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1d7cb84e-25c2-4ce1-b1c1-d8f9cadab2a5",
   "metadata": {},
   "source": [
    "len(subjects)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "95e0f584-f802-4540-a485-ccdafc23df42",
   "metadata": {},
   "source": [
    "for subject in subjects:\n",
    "    globals()[f'subject_{subject.project.replace(\"-\", \"_\")}_{subject.bug_id}'] = subject"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "6efc3f7d-7470-482f-8bbd-dfd864cc9286",
   "metadata": {},
   "source": [
    "stats = {p: {'bugs': get_bugs(p), 'used': 0, 'tests': 0} for p in projects}\n",
    "\n",
    "for subject in subjects:\n",
    "    stats[subject.project]['used'] += 1\n",
    "    stats[subject.project]['tests'] += len(subject.passing) + len(subject.failing)\n",
    "\n",
    "print(f'{\"project\":<20}{\"bugs\":<6}{\"used\":<6}{\"tests\":<6}')\n",
    "\n",
    "for p in stats:\n",
    "    print(f'{p:<20}{stats[p][\"bugs\"]:<6}{stats[p][\"used\"]:<6}{stats[p][\"tests\"]:<6}')"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "7a92f75e-1934-49f5-942c-ead44b069984",
   "metadata": {},
   "source": [
    "## SFLKit Framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6a2c2c4a-d229-44e3-b8b3-a47a4064d42b",
   "metadata": {},
   "source": [
    "from sflkit import instrument, analyze\n",
    "from sflkit.config import Config\n",
    "from sflkit.analysis.analysis_type import AnalysisType\n",
    "from sflkit.analysis.suggestion import Suggestion"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "725ee1e9-5475-4bb1-90f8-89916be26fd4",
   "metadata": {},
   "source": [
    "from configparser import ConfigParser"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3f4f90e5-21d8-475e-a577-f6a5b69f33b1",
   "metadata": {},
   "source": [
    "def generate_config(project, dst, excluded_project_files, passing, failing):\n",
    "    config = ConfigParser()\n",
    "    config['target'] = dict()\n",
    "    config['events'] = dict()\n",
    "    config['instrumentation'] = dict()\n",
    "    config['test'] = dict()\n",
    "\n",
    "    config['target']['path'] = get_dir(project)\n",
    "    config['target']['language'] = 'python'\n",
    "    config['events']['events'] = 'line'\n",
    "    config['events']['predicates'] = 'line'\n",
    "    config['events']['metrics'] = 'Ochiai,Tarantula,Jaccard'\n",
    "    config['events']['passing'] = passing\n",
    "    config['events']['failing'] = failing\n",
    "    config['instrumentation']['path'] = get_dir(dst)\n",
    "    config['instrumentation']['exclude'] = '\"' + '\",\"'.join(excluded_project_files) + '\"'\n",
    "    with open(config_tmp, 'w') as fp:\n",
    "        config.write(fp)"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "4c448c7b-f58d-474a-b8ba-511fe98a6195",
   "metadata": {},
   "source": [
    "### Extract Execution Events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "fb9a1a7a-0e14-4275-85ee-053a781c2830",
   "metadata": {},
   "source": [
    "excluded_project_files = ['test', 'tests', 'setup.py', 'env', 'build', 'bin', 'docs', 'examples', \n",
    "                          'hacking', '.git', '.github', 'extras', 'profiling', 'plugin', 'gallery', \n",
    "                          'blib2to3', 'docker', 'contrib', 'changelogs', 'licenses', 'packaging']"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "9ba3df7e-141d-4919-a41a-ad23306aace9",
   "metadata": {},
   "source": [
    "excluded_tests = [\n",
    "    'tests/test_black.py::BlackTestCase::test_self',\n",
    "    'tests/test_black.py::BlackTestCase::test_pytree',\n",
    "    'tests/test_black.py::BlackTestCase::test_root_logger_not_used_directly',\n",
    "    'tests/test_black.py::BlackTestCase::test_tokenize',\n",
    "    'tests/test_black.py::BlackTestCase::test_pgen',\n",
    "    'tests/test_black.py::BlackTestCase::test_expression_diff_with_color',\n",
    "    'tests/test_black.py::BlackTestCase::test_expression_diff',\n",
    "    'tests/test_black.py::BlackTestCase::test_expression_ff',\n",
    "    'tests/test_black.py::BlackTestCase::test_expression',\n",
    "]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9b9d2f7a-bb4d-4872-9772-d1c877eb397f",
   "metadata": {},
   "source": [
    "def get_path(project: str, bug_id: int, passing: bool = True):\n",
    "    return os.path.join(events_output, project, str(bug_id), 'passing' if passing else 'failing')"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "07de2d2a-625c-4fdd-b2c2-58a200d3feeb",
   "metadata": {},
   "source": [
    "def instrument_and_install_project(project: str, bug_id: int, dst: str, verbose=False):\n",
    "    shutil.rmtree(dst, ignore_errors=True)\n",
    "    get_project(project, bug_id=bug_id, verbose=verbose)\n",
    "    config = generate_config(project, dst, excluded_project_files, get_path(project, bug_id, True), get_path(project, bug_id, False))\n",
    "    instrument(config_tmp)\n",
    "    env_on(project, verbose=verbose)\n",
    "    process = subprocess.run([bip_sdtools, '-w', get_dir(dst), '-s', sflkit_dir],\n",
    "                      stdout=None if verbose else subprocess.DEVNULL,\n",
    "                      stderr=subprocess.STDOUT if verbose else subprocess.DEVNULL, env=os.environ)\n",
    "    assert process.returncode == 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1572018c-d6af-4d60-8c0c-c0697a1111ae",
   "metadata": {},
   "source": [
    "instrument_and_install_project(test_project, test_bug_id, default_project)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3ec5843d-a850-4b31-a3b4-d274e924224d",
   "metadata": {},
   "source": [
    "def get_events_path_file(project: str):\n",
    "    return os.path.join(get_dir(project), events_path)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0de5fb30-3f9f-402b-8fb6-f1d5a42ab90c",
   "metadata": {},
   "source": [
    "class TestDiscard(Exception):\n",
    "    pass"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "2e47fe6f-32a5-4ac1-8281-20666296aa1f",
   "metadata": {},
   "source": [
    "def get_file(test_case: str, passing: bool = True):\n",
    "    return f'{\"p\" if passing else \"f\"}_{hashlib.md5(test_case.encode(\"utf8\")).hexdigest()}'"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "1b803d98-a5f1-4662-983e-6133b56faf0c",
   "metadata": {},
   "source": [
    "def get_events_path(project: str, bug_id: int, dst: str, test_case: str, passing: bool = True, verbose=False):\n",
    "    if not os.path.exists(events_output):\n",
    "        os.mkdir(events_output)\n",
    "    if not os.path.exists(os.path.join(events_output, project)):\n",
    "        os.mkdir(os.path.join(events_output, project))\n",
    "    if not os.path.exists(os.path.join(events_output, project, str(bug_id))):\n",
    "        os.mkdir(os.path.join(events_output, project, str(bug_id)))\n",
    "    if not os.path.exists(os.path.join(events_output, project, str(bug_id), 'passing')):\n",
    "        os.mkdir(os.path.join(events_output, project, str(bug_id), 'passing'))\n",
    "    if not os.path.exists(os.path.join(events_output, project, str(bug_id), 'failing')):\n",
    "        os.mkdir(os.path.join(events_output, project, str(bug_id), 'failing'))\n",
    "        \n",
    "    if os.path.exists(get_events_path_file(dst)):\n",
    "        os.remove(get_events_path_file(dst))\n",
    "        \n",
    "    test_result = run_test(dst, test_case, verbose=verbose)\n",
    "    \n",
    "    if test_result != passing:\n",
    "        raise TestDiscard(f'The result of {test_case} was not correct.')\n",
    "    if os.path.exists(get_events_path_file(dst)):\n",
    "        shutil.copy(get_events_path_file(dst), os.path.join(get_path(project, bug_id, passing), \n",
    "                                                            get_file(test_case, passing)))\n",
    "    else:\n",
    "        raise TestDiscard(f'Events path not found for {test_case}.')"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "df67911d-76c2-461e-bccc-6ace5a56aa2d",
   "metadata": {},
   "source": [
    "get_events_path(test_project, test_bug_id, default_project, test_passing[0], verbose=True)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5dbbe504-d1ae-49bf-8cde-4dd02099475e",
   "metadata": {},
   "source": [
    "def get_all_events_paths(project: str, bug_id: int, dst: str, passing: List[str], failing: bool = List[str], verbose=False):\n",
    "    p = 0\n",
    "    f = 0\n",
    "    d = 0\n",
    "    \n",
    "    for t in passing:\n",
    "        if t not in excluded_tests:\n",
    "            try:\n",
    "                get_events_path(project, bug_id, dst, t, verbose=verbose)\n",
    "                p += 1\n",
    "            except TestDiscard as e:\n",
    "                if verbose:\n",
    "                    print(e)\n",
    "                d += 1\n",
    "        else:\n",
    "            d += 1\n",
    "    \n",
    "    for t in failing:\n",
    "        try:\n",
    "            get_events_path(project, bug_id, dst, t, passing=False, verbose=verbose)\n",
    "            f += 1\n",
    "        except TestDiscard as e:\n",
    "            if verbose:\n",
    "                print(e)\n",
    "            d += 1\n",
    "                    \n",
    "    \n",
    "    print(f'{project:<40}{bug_id:<6}{p:<6}{f:<6}{d:<6}')\n",
    "    return p > 0 and f > 0"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "f723e399-98cd-46bb-afde-c9bb8a16b13c",
   "metadata": {},
   "source": [
    "assert get_all_events_paths(test_project, test_bug_id, default_project, test_passing, test_failing)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "5a676c98-2ca4-4927-9611-fa57068602fe",
   "metadata": {},
   "source": [
    "def extract_events_for_project(project: str, bug_id: int, dst: str, passing: List[str], failing: bool = List[str], verbose=False):\n",
    "    instrument_and_install_project(project, bug_id, dst, verbose=verbose)\n",
    "    return get_all_events_paths(project, bug_id, dst, passing, failing, verbose=verbose)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "565235c1-1ec9-4eed-97b8-19e6c425cba7",
   "metadata": {},
   "source": [
    "def extract_events(subject: Subject, dst: str, verbose=False):\n",
    "    return extract_events_for_project(subject.project, subject.bug_id, dst, subject.passing, subject.failing, verbose=verbose)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "c2e4376c-3673-4c43-95da-21bd200d4398",
   "metadata": {},
   "source": [
    "extract_events(test_subject, default_project)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "6524a8b9-2668-4b84-969e-6435a7405cb8",
   "metadata": {},
   "source": [
    "def get_all_events(subjects: List[Subject], continue_: bool = False):\n",
    "    for subject in subjects:\n",
    "        if not (continue_ and os.path.exists(os.path.join(events_output, subject.project, str(subject.bug_id)))):\n",
    "            extract_events(subject, default_project)\n",
    "        else:\n",
    "            p = len(os.listdir(get_path(subject.project, subject.bug_id, passing=True)))\n",
    "            f = len(os.listdir(get_path(subject.project, subject.bug_id, passing=False)))\n",
    "            print(f'{subject.project:<40}{subject.bug_id:<6}{p:<6}{f:<6}{len(subject.passing) - p + len(subject.failing) - f:<6}')"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "4ab19803-d881-40d6-a47e-993d061dee1e",
   "metadata": {},
   "source": [
    "if events_extraction:\n",
    "    get_all_events(subjects, continue_=continue_events_extraction)"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "39ec46c6-c422-4483-86b5-c2a8fe01993a",
   "metadata": {},
   "source": [
    "### Extract Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "1f6d0455-645d-4c5c-b5aa-0571ef5004bd",
   "metadata": {},
   "source": [
    "def get_results(project: str, bug_id: int, dst: str, verbose=False):\n",
    "    config = generate_config(project, dst, excluded_project_files, get_path(project, bug_id, True), get_path(project, bug_id, False))\n",
    "    return analyze(config_tmp)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "0fd204cf-2237-43fa-b32c-b91d9737e734",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "source": [
    "test_results = get_results(test_project, test_bug_id, default_project)\n",
    "test_results"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "e9048d55-5420-466e-a6ef-9f406d5caf0a",
   "metadata": {},
   "source": [
    "def get_suggestions(type_: AnalysisType, metric: str, results:dict) -> List[Suggestion]:\n",
    "    return results[type_.name][metric]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "583954fc-0455-4fcd-b613-f546ab403590",
   "metadata": {},
   "source": [
    "def get_k_suggestions(suggestions: List[Suggestion], k: int) -> Set[Suggestion]:\n",
    "    result = set()\n",
    "    highest = suggestions[0].suspiciousness\n",
    "    for suggestion in suggestions:\n",
    "        if len(result) >= k and suggestion.suspiciousness < highest:\n",
    "            break\n",
    "        result.update(suggestion.lines)\n",
    "    return result"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "48fe463f-3c0e-4bdc-b6ab-5b0ce673a99d",
   "metadata": {},
   "source": [
    "def get_k(type_: AnalysisType, metric: str, results: dict, k: int = 10) -> Set[Suggestion]:\n",
    "    return get_k_suggestions(get_suggestions(type_, metric, results), k)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "057dd3d6-6c2e-40be-8c22-ba7e4068f653",
   "metadata": {},
   "source": [
    "get_k(AnalysisType.LINE, 'Ochiai', test_results)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d804ad96-6b80-47d6-9e0c-69fffe452746",
   "metadata": {},
   "source": [
    "def precision_at_k(type_: AnalysisType, metric: str, results:dict, locations: List[Location], k: int = 10):\n",
    "    suggestions = get_k(type_, metric, results, k=k)\n",
    "    locations = set(locations)\n",
    "    return len(locations.intersection(suggestions)) / len(suggestions)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "1df5384f-c1d9-42f6-82ca-1b8397bccb18",
   "metadata": {},
   "source": [
    "for k in [1, 3, 5, 10]:\n",
    "    print(f'k={k:<2} : {precision_at_k(AnalysisType.LINE, \"Ochiai\", test_results, test_faulty_lines, k=k)}')"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "a864a29f-5abe-4281-a4a5-6030adcc9616",
   "metadata": {},
   "source": [
    "def recall_at_k(type_: AnalysisType, metric: str, results:dict, locations: List[Location], k: int = 10):\n",
    "    suggestions = get_k(type_, metric, results, k=k)\n",
    "    locations = set(locations)\n",
    "    return len(locations.intersection(suggestions)) / len(locations)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "01c61a73-40b8-4ed5-898e-479f5513a088",
   "metadata": {},
   "source": [
    "for k in [1, 3, 5, 10]:\n",
    "    print(f'k={k:<2} : {recall_at_k(AnalysisType.LINE, \"Ochiai\", test_results, test_faulty_lines, k=k)}')"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "88272995-0df3-4d5b-88d6-b950ee66995c",
   "metadata": {},
   "source": [
    "## Run Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "2fd51684-a490-4bdd-a1d7-2ce922eef263",
   "metadata": {},
   "source": [
    "def run_for_subject(subject: Subject, ks=[1, 3, 5, 10], extract=True, continue_extraction=True, verbose=False):\n",
    "    results = get_results(subject.project, subject.bug_id, default_project, verbose=verbose)\n",
    "    evaluation = dict()\n",
    "    for type_ in results:\n",
    "        evaluation[type_] = dict()\n",
    "        for metric in results[type_]:\n",
    "            evaluation[type_][metric] = dict()\n",
    "            for k in ks:\n",
    "                evaluation[type_][metric][f'p@{k}'] = precision_at_k(AnalysisType[type_], metric, results, subject.faulty_lines, k=k)\n",
    "                evaluation[type_][metric][f'r@{k}'] = recall_at_k(AnalysisType[type_], metric, results, subject.faulty_lines, k=k)\n",
    "    print(f'Evaluated {subject}')\n",
    "    return evaluation"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "ca6d5440-415b-4160-98a1-4c6b4b584b7c",
   "metadata": {},
   "source": [
    "run_for_subject(test_subject)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "74c7cb52-f309-4b97-bbc8-802df51a6425",
   "metadata": {},
   "source": [
    "def run_for_subjects(subjects: List[Subject], ks=[1, 3, 5, 10], extract=True, continue_extraction=True, verbose=False):\n",
    "    results = list()\n",
    "    f = len(subjects)\n",
    "    for subject in subjects:\n",
    "        results.append(run_for_subject(subject, ks=ks, extract=extract, continue_extraction=continue_extraction, verbose=verbose))\n",
    "    return results"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "0105a98d-c229-4846-b983-7c08e9528733",
   "metadata": {},
   "source": [
    "results = run_for_subjects(subjects, extract=extract_events, continue_extraction=continue_events_extraction)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "eac7036d-0a59-42b8-9562-e85eb40c02dd",
   "metadata": {},
   "source": [
    "def average(results):\n",
    "    evaluation = dict()\n",
    "    f = len(results)\n",
    "    for r in results:\n",
    "        for t in r:\n",
    "            if t not in evaluation:\n",
    "                evaluation[t] = dict()\n",
    "            for m in r[t]:\n",
    "                if m not in evaluation[t]:\n",
    "                    evaluation[t][m] = dict()\n",
    "                for pr in r[t][m]:\n",
    "                    if pr not in evaluation[t][m]:\n",
    "                        evaluation[t][m][pr] = 0\n",
    "                    evaluation[t][m][pr] += r[t][m][pr] / f\n",
    "    return evaluation"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58f98abf-b5be-4216-84ab-7d60242afcb3",
   "metadata": {},
   "source": [
    "evaluation = average(results)\n",
    "evaluation"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "711d8c91-47ec-4b9f-9edd-ad3b26fb016f",
   "metadata": {},
   "source": [
    "with open('results.json', 'w') as fp:\n",
    "    json.dump(evaluation, fp, indent = 4)"
   ],
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}