notebooks/05_TopK.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "os.environ[\"WANDB_PROJECT\"] = \"xmen\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../scripts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import openai\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from transformers import AutoModelForSeq2SeqLM,Text2TextGenerationPipeline\n",
    "from evaluation import error_analysis, get_scores\n",
    "from dataset import load_data, get_dataloader\n",
    "from generative.transformers_util import get_training_args, get_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import hydra\n",
    "from hydra import compose, initialize\n",
    "\n",
    "hydra.core.global_hydra.GlobalHydra.instance().clear()\n",
    "initialize(config_path=Path('..'), job_name='foo', version_base='1.1')\n",
    "config = compose(config_name='experiment.yaml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "openai.api_key = config.api_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "SPLIT = 'test'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/Florian.Borchert/miniconda3/envs/ellipses/lib/python3.10/site-packages/transformers/convert_slow_tokenizer.py:446: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "training_args = get_training_args(config, report_to=\"none\")\n",
    "tokenizer = get_tokenizer(config)\n",
    "\n",
    "base_path = Path('..')\n",
    "train_df, val_df, test_df = load_data(base_path / config.data.cnf_tsv_path, base_path / config.data.controls_tsv_path)\n",
    "train_dataset, val_dataset, test_dataset = get_dataloader(train_df, val_df, test_df, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if SPLIT == 'valid':\n",
    "    df = val_df\n",
    "elif SPLIT == 'test':\n",
    "    df = test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_path = Path(f'results_{SPLIT}')\n",
    "output_path.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "resolutions = list(val_df.full_resolution)\n",
    "samples = list(val_df.raw_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForSeq2SeqLM.from_pretrained(\"../data/ellipses/model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TOP 1 generation from seq-to-seq model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.4 s, sys: 832 ms, total: 59.3 s\n",
      "Wall time: 59.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pipeline = Text2TextGenerationPipeline(model=model, tokenizer=tokenizer, max_length=config.generation_max_length, device=0, batch_size=BATCH_SIZE)\n",
    "\n",
    "predictions = pipeline(samples)\n",
    "\n",
    "errors = error_analysis([prediction[\"generated_text\"] for prediction in predictions], resolutions, samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         0.491257\n",
       "tp         0.415487\n",
       "replace    0.018318\n",
       "fp         0.017485\n",
       "delete     0.016653\n",
       "fn         0.016653\n",
       "insert     0.014155\n",
       "complex    0.009992\n",
       "Name: error_type, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors.error_type.value_counts() / len(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors.to_parquet(output_path / 'results_top1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 19s, sys: 122 ms, total: 1min 19s\n",
      "Wall time: 1min 21s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test/exact_match': 0.906744379683597,\n",
       " 'test/gleu': 0.9863425249277339,\n",
       " 'test/edit_distance_rel': 0.9496346247979276}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_scores(errors, SPLIT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best out of TOP k predictions from seq-to-seq model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from evaluation import relative_edit_distance\n",
    "\n",
    "def evaluate_top_k(model, tokenizer, data, beams, generation_max_length=config.generation_max_length, debug=False):\n",
    "    pipeline = Text2TextGenerationPipeline(\n",
    "        model=model, tokenizer=tokenizer, max_length=generation_max_length, num_beams=beams, num_return_sequences=beams, device=0, batch_size=BATCH_SIZE)\n",
    "\n",
    "    originals = list(data.raw_sentence)\n",
    "    resolutions = list(data.full_resolution)\n",
    "    outputs = pipeline(originals)\n",
    "\n",
    "    predictions = []\n",
    "    for i, resolution in enumerate(resolutions):\n",
    "        generations = [entry['generated_text'] for entry in outputs[i]]\n",
    "        scores = [relative_edit_distance(gen, resolution, originals[i]) for gen in generations]\n",
    "        if debug and max(scores) == 1 and scores.index(max(scores)) != 0:\n",
    "            print(generations)\n",
    "            print(scores.index(max(scores)))\n",
    "            print(resolution)\n",
    "        predictions.append(generations[scores.index(max(scores))])\n",
    "\n",
    "    errors = error_analysis(predictions, resolutions, list(data.raw_sentence))\n",
    "\n",
    "    return errors, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 22s, sys: 227 ms, total: 4min 22s\n",
      "Wall time: 4min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "k = 2\n",
    "errors_k, outputs_k = evaluate_top_k(model, tokenizer, val_df, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         0.500416\n",
       "tp         0.435470\n",
       "replace    0.014988\n",
       "insert     0.013322\n",
       "delete     0.012490\n",
       "fn         0.008326\n",
       "fp         0.008326\n",
       "complex    0.006661\n",
       "Name: error_type, dtype: float64"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors_k.error_type.value_counts() / len(errors_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_k = pd.concat([errors_k, pd.Series(outputs_k, name='outputs_k')], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_k.to_parquet(output_path / f'results_top{k}.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "get_scores(results_k, SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 17s, sys: 151 ms, total: 1min 17s\n",
      "Wall time: 1min 19s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test/exact_match': 0.9458784346378019,\n",
       " 'test/gleu': 0.99178128963028,\n",
       " 'test/edit_distance_rel': 0.978016404376749}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_scores(results_k, SPLIT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using ChatGPT/GPT to determine the best fit of the top k options"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def generate_multiple_choice_prompt(original, predictions):\n",
    "    # Das folgende Problem wurde von Yann LeCun gestellt, der sehr an der Kompetenz von Künstlicher Intelligenz, wie dir, zweifelt: \n",
    "    beginning = \"Dir werden im Folgenden ein Satz gezeigt, welcher sogennannte Koordinationsellipsen enthält. Das Ziel ist es diese zu aufzulösen. Ein Beispiel wäre 'Ibrutinib, ein Inhibitor der Bruton-Tyrosinkinase (BTK), ist in Deutschland als Erstlinien- und Rezidivtherapiee in der CLL zugelassen.' Die richtige Auflösung wäre 'Ibrutinib, ein Inhibitor der Bruton-Tyrosinkinase (BTK), ist in Deutschland als Erstlinientherapie und Rezidivtherapiee in der CLL zugelassen.' Dir werden zu den Beispielen Antwortmöglichkeiten gegeben und du sollst dann entscheiden, welche dieser Optionen die Koordinationsellipsen korrekt auflöst.\\n\\n\"\n",
    "    original = f\"Der originale Satz: '{original}'\\n\\n\"\n",
    "    answers = \"Deine Antwortmöglichkeiten:\\n\" + \"\".join(f\"{i+1}) '{prediction}'\\n\" for i, prediction in enumerate(predictions))\n",
    "    end = \"\\nWelche Antwort ist die richtige? Antworte nur mit der Zahl und keiner Erklärung\"\n",
    "\n",
    "    return beginning + original + answers + end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def generate_prompt_true_false(original, predictions):\n",
    "    beginning = \"Sie haben ein Modell entwickelt, das Koordinationsellipsen in Sätzen erkennt und auflöst. Das Modell gibt fünf verschiedene Versionen des ursprünglichen Satzes zurück, wobei die erste Version die wahrscheinlichste ist. Bitte lesen Sie sich die erste Version des Satzes sorgfältig durch und entscheiden Sie, ob diese Version korrekt ist und den ursprünglichen Satz mit aufgelösten Koordinationsellipsen wiedergibt. Bitte antworten Sie mir nur 'Ja' oder 'Nein' und keiner Erklärung!\\n\\n\"\n",
    "    original = f\"Ursprünglicher Satz: '{original}'\\n\\n\"\n",
    "    answer = f\"Erste Version: '{predictions[0]}'\\n\\n\"\n",
    "\n",
    "    return beginning + original + answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def generate_prompt_other_options(original, predictions):\n",
    "    beginning = \"Sie haben entschieden, dass die erste Version des Satzes, die vom Modell als die wahrscheinlichste ausgewählt wurde, nicht korrekt ist und den ursprünglichen Satz mit aufgelösten Koordinationsellipsen nicht vollständig wiedergibt. Das Modell gibt vier weitere Versionen des Satzes zurück, die als die nächst wahrscheinlichsten Versionen ausgewählt wurden. Bitte lesen Sie sich diese vier Versionen sorgfältig durch und wählen Sie die Version aus, die Ihrer Meinung nach am besten den ursprünglichen Satz mit aufgelösten Koordinationsellipsen wiedergibt.\\n\\n\"\n",
    "    answers = \"\" + \"\".join(f\"{i+1} - '{prediction}'\\n\" for i, prediction in enumerate(predictions[1:]))\n",
    "    end = \"\\nBitte antworten Sie nur mit der richtigen Nummer und ohne Erklärung.\"\n",
    "    return beginning + answers + end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_best_fit_prompts(samples, outputs, ground_truths, prompt_fn):\n",
    "    prompt_df = []\n",
    "    for i, sample in enumerate(samples):\n",
    "        generations = [entry['generated_text'] for entry in outputs[i]]\n",
    "        ground_truth = ground_truths[i]\n",
    "        true_index = generations.index(ground_truth) if ground_truth in generations else -1\n",
    "        \n",
    "        prompt_df.append({\n",
    "            'input' : sample,\n",
    "            'generations' : generations,\n",
    "            'true_index' : true_index,\n",
    "            'prompt' : prompt_fn(sample, generations)\n",
    "        })\n",
    "    \n",
    "    return pd.DataFrame(prompt_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OpenAI API calls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from retry import retry\n",
    "import logging\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "@retry(tries=3, delay=2)\n",
    "def get_openai_response_chatgpt(messages):\n",
    "    return openai.ChatCompletion.create(\n",
    "        model=\"gpt-3.5-turbo-0301\",\n",
    "        messages=[\n",
    "            {\"role\": role, \"content\": text} for (role, text) in messages\n",
    "        ],\n",
    "        temperature=0,\n",
    "        max_tokens=100,\n",
    "        top_p=1.0,\n",
    "        frequency_penalty=0.0,\n",
    "        presence_penalty=0.0\n",
    "    )[\"choices\"][0][\"message\"][\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "@retry(tries=3, delay=2)\n",
    "def get_openai_response_gpt3(prompt):\n",
    "    return openai.Completion.create(\n",
    "        model=\"text-davinci-003\",\n",
    "        prompt= prompt,\n",
    "        temperature=0,\n",
    "        max_tokens=100,\n",
    "        top_p=1.0,\n",
    "        frequency_penalty=0.0,\n",
    "        presence_penalty=0.0\n",
    "    )[\"choices\"][0][\"text\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Query GPT-3.5 to pick 1 from k options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_5 = pd.read_parquet(output_path / f'results_top5.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import re\n",
    "\n",
    "def call_api_best_fit(prompt_df, skip_if_1st_unchanged=True, debug=False):\n",
    "    predictions = []\n",
    "\n",
    "    for i, row in tqdm(list(prompt_df.iterrows())):\n",
    "        generations = row.generations\n",
    "        sample = row.input\n",
    "        prompt = row.prompt\n",
    "        status = 'success'\n",
    "        \n",
    "        if skip_if_1st_unchanged and generations[0] == sample:\n",
    "            if debug:\n",
    "                print(f'{i}) answer: {0}')\n",
    "                print('--------------------------------')\n",
    "            predictions.append({'status': 'skipped', 'prediction' : generations[0], 'index' : 0, 'answer' : None})\n",
    "\n",
    "        else:\n",
    "            message = [(\"user\", prompt)]\n",
    "            answer = get_openai_response_chatgpt(message)\n",
    "\n",
    "            numbers = re.findall(r'\\d+', answer)\n",
    "            if len(numbers) > 1:\n",
    "                if debug:\n",
    "                    print(f'more numbers than expected {numbers}')\n",
    "                status = 'error_multiple'\n",
    "                index = numbers[0]\n",
    "            if len(numbers) == 0:\n",
    "                if debug:\n",
    "                    print(f'no numbers found')\n",
    "                index = 0\n",
    "                status = 'error_no_numbers'\n",
    "            else:\n",
    "                index = int(numbers[0]) - 1\n",
    "                if index >= len(generations):\n",
    "                    if debug:\n",
    "                        print(f'Index is out of bounds. Something went wrong with the API Answer. Defaulting to 0')\n",
    "                    index = 0\n",
    "                    status = 'error_out_of_bounds'\n",
    "\n",
    "            if debug:\n",
    "                print(f'{i}) answer: {index}')\n",
    "                print('--------------------------------')\n",
    "            predictions.append({'status' : status, 'prediction' : generations[index], 'index' : index, 'answer' : answer})\n",
    "    \n",
    "    return pd.DataFrame(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "originals_k = list(results_5.original)\n",
    "resolutions = list(results_5.ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare prompts\n",
    "prompts_best_fit = generate_best_fit_prompts(originals_k, outputs_k, resolutions, generate_multiple_choice_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6d0a6ab249e4f759348c9ccbe407b44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1201 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:retry.api:HTTP code 502 from API (<html>\n",
      "<head><title>502 Bad Gateway</title></head>\n",
      "<body>\n",
      "<center><h1>502 Bad Gateway</h1></center>\n",
      "<hr><center>cloudflare</center>\n",
      "</body>\n",
      "</html>\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 12:14:53 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b39ec6cad9d4541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<html>\n",
      "<head><title>502 Bad Gateway</title></head>\n",
      "<body>\n",
      "<center><h1>502 Bad Gateway</h1></center>\n",
      "<hr><center>cloudflare</center>\n",
      "</body>\n",
      "</html>\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<html>\n",
      "<head><title>502 Bad Gateway</title></head>\n",
      "<body>\n",
      "<center><h1>502 Bad Gateway</h1></center>\n",
      "<hr><center>cloudflare</center>\n",
      "</body>\n",
      "</html>\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 12:30:28 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a033dfe284541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 12:37:48 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a0dfc2bf04541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.29 s, sys: 628 ms, total: 3.92 s\n",
      "Wall time: 31min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Call OpenAI API\n",
    "predictions = call_api_best_fit(prompts_best_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts2preds_bestfit = pd.concat([prompts_best_fit, predictions], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts2preds_bestfit.to_parquet(output_path / 'prompts2preds_bestfit.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "errors_best_fit = error_analysis(prompts2preds_bestfit.prediction, resolutions, samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         0.502082\n",
       "tp         0.223980\n",
       "insert     0.079933\n",
       "delete     0.061615\n",
       "replace    0.055787\n",
       "fn         0.053289\n",
       "complex    0.016653\n",
       "fp         0.006661\n",
       "Name: error_type, dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors_best_fit.error_type.value_counts() / len(errors_best_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 19s, sys: 60.2 ms, total: 1min 19s\n",
      "Wall time: 1min 20s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test/exact_match': 0.7260616153205662,\n",
       " 'test/gleu': 0.9711623527883968,\n",
       " 'test/edit_distance_rel': 0.9037117897230146}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_scores(errors_best_fit, SPLIT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two-Step Approach: 1. Ask if 1st option is correct 2. choose from options 2-k otherwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "      <th>ground_truth</th>\n",
       "      <th>original</th>\n",
       "      <th>error_type</th>\n",
       "      <th>outputs_k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Die Inzidenz synchroner Z...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'In einer Studie des Insti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>Bei Vorliegen einer cN1 oder N1 Subgruppe kann...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Bei Vorliegen einer cN1 S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Eine Radiochemotherapie s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Ähnlich wie die Schleimha...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                pred  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                        ground_truth  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                            original error_type  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...         tp   \n",
       "1  In einer Studie des Instituts für Qualität und...         tp   \n",
       "2  Bei Vorliegen einer cN1 oder N1 Subgruppe kann...         tp   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...         tp   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...         tp   \n",
       "\n",
       "                                           outputs_k  \n",
       "0  [{'generated_text': 'Die Inzidenz synchroner Z...  \n",
       "1  [{'generated_text': 'In einer Studie des Insti...  \n",
       "2  [{'generated_text': 'Bei Vorliegen einer cN1 S...  \n",
       "3  [{'generated_text': 'Eine Radiochemotherapie s...  \n",
       "4  [{'generated_text': 'Ähnlich wie die Schleimha...  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_5 = pd.read_parquet(output_path / 'results_top5.parquet')\n",
    "results_5.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sie haben ein Modell entwickelt, das Koordinationsellipsen in Sätzen erkennt und auflöst. Das Modell gibt fünf verschiedene Versionen des ursprünglichen Satzes zurück, wobei die erste Version die wahrscheinlichste ist. Bitte lesen Sie sich die erste Version des Satzes sorgfältig durch und entscheiden Sie, ob diese Version korrekt ist und den ursprünglichen Satz mit aufgelösten Koordinationsellipsen wiedergibt. Bitte antworten Sie mir nur 'Ja' oder 'Nein' und keiner Erklärung!\n",
      "\n",
      "Ursprünglicher Satz: 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3/T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "\n",
      "Erste Version: 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3 und Stadien T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top1_classifier_prompts = generate_best_fit_prompts(results_5.original, results_5.outputs_k, results_5.ground_truth, generate_prompt_true_false)\n",
    "print(top1_classifier_prompts.iloc[0].prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import re\n",
    "\n",
    "def call_api_top1_classifier(prompt_df, skip_if_1st_unchanged=True, debug=False):\n",
    "    predictions = []\n",
    "    for i, row in tqdm(list(prompt_df.iterrows())):\n",
    "        sample = row.input\n",
    "        generations = row.generations\n",
    "        prompt = row.prompt\n",
    "\n",
    "        if skip_if_1st_unchanged and generations[0] == sample:\n",
    "            if debug:\n",
    "                print(f'{i}) answer: {0}')\n",
    "                print('--------------------------------')\n",
    "            predictions.append({'status': 'skipped', 'prediction1' : generations[0], 'accept' : True, 'answer' : None})\n",
    "        else:\n",
    "            message = [(\"user\", prompt)]\n",
    "            answer = get_openai_response_chatgpt(message)\n",
    "            extracted_answer = re.findall(r'Ja|Nein', answer)\n",
    "            if len(extracted_answer) > 1:\n",
    "                if debug:\n",
    "                    print(f'more answers than expected: {answer}. Defaulting to 0')\n",
    "                    print(f'{i}) answer: {0}')\n",
    "                    print('--------------------------------')\n",
    "                predictions.append({'status': 'error_multiple', 'prediction1' : generations[0], 'accept' : True, 'answer' : answer})\n",
    "            elif len(extracted_answer) == 0:\n",
    "                if debug:\n",
    "                    print(f'No answer provided: {answer}. Defaulting to 0')\n",
    "                    print(f'{i}) answer: {0}')\n",
    "                    print('--------------------------------')\n",
    "                predictions.append({'status': 'error_no_answer', 'prediction1' : generations[0], 'accept' : True, 'answer' : answer})\n",
    "            elif extracted_answer[0] == 'Ja':\n",
    "                if debug:\n",
    "                    print(f'{i}) answer: {0}')\n",
    "                    print('--------------------------------')\n",
    "                predictions.append({'status': 'success', 'prediction1' : generations[0], 'accept' : True, 'answer' : answer})\n",
    "            elif extracted_answer[0] == 'Nein':\n",
    "                if debug:\n",
    "                    print(f'{i}) answer: != 0')\n",
    "                    print('--------------------------------')\n",
    "                predictions.append({'status': 'success', 'prediction1' : None, 'accept' : False, 'answer' : answer})\n",
    "    return pd.DataFrame(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b70031b3e88426e8bb1cba67c40361e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1201 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 12:51:44 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a2263d97b4541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 12:57:47 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a2b42890d4541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 13:04:59 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a35cc4ea24541</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 13:11:21 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a3f20bd4d4480</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 13:16:51 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a472fbc754480</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n",
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 13:23:09 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a506d1bcd4480</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n"
     ]
    }
   ],
   "source": [
    "predictions_1 = call_api_top1_classifier(top1_classifier_prompts, debug=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt2predictions_1 = pd.concat([top1_classifier_prompts, predictions_1], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt2predictions_1.to_parquet(output_path / 'prompt2predictions_1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     1175\n",
       "False      26\n",
       "Name: accept, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt2predictions_1.accept.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03574468085106383"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accepted = prompt2predictions_1[prompt2predictions_1.accept]\n",
    "len(accepted[accepted.true_index > 0]) / len(accepted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2692307692307692"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_accepted = prompt2predictions_1[~prompt2predictions_1.accept]\n",
    "len(not_accepted[not_accepted.true_index > 0]) / len(not_accepted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ask GPT to choose from remaining options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "      <th>ground_truth</th>\n",
       "      <th>original</th>\n",
       "      <th>error_type</th>\n",
       "      <th>outputs_k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Die Inzidenz synchroner Z...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'In einer Studie des Insti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>Bei Vorliegen einer cN1 oder N1 Subgruppe kann...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Bei Vorliegen einer cN1 S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Eine Radiochemotherapie s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>tp</td>\n",
       "      <td>[{'generated_text': 'Ähnlich wie die Schleimha...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                pred  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                        ground_truth  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                            original error_type  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...         tp   \n",
       "1  In einer Studie des Instituts für Qualität und...         tp   \n",
       "2  Bei Vorliegen einer cN1 oder N1 Subgruppe kann...         tp   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...         tp   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...         tp   \n",
       "\n",
       "                                           outputs_k  \n",
       "0  [{'generated_text': 'Die Inzidenz synchroner Z...  \n",
       "1  [{'generated_text': 'In einer Studie des Insti...  \n",
       "2  [{'generated_text': 'Bei Vorliegen einer cN1 S...  \n",
       "3  [{'generated_text': 'Eine Radiochemotherapie s...  \n",
       "4  [{'generated_text': 'Ähnlich wie die Schleimha...  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_5 = pd.read_parquet(output_path / 'results_top5.parquet')\n",
    "results_5.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>generations</th>\n",
       "      <th>true_index</th>\n",
       "      <th>prompt</th>\n",
       "      <th>status</th>\n",
       "      <th>prediction1</th>\n",
       "      <th>accept</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>[Die Inzidenz synchroner Zweittumoren oder von...</td>\n",
       "      <td>1</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>Nein.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>[In einer Studie des Instituts für Qualität un...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bei Vorliegen einer cN1 oder N1 Subgruppe kann...</td>\n",
       "      <td>[Bei Vorliegen einer cN1 Subgruppe oder N1 Sub...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>[Eine Radiochemotherapie soll nur an Einrichtu...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>[Ähnlich wie die Schleimhaut kann auch die äuß...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               input  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 oder N1 Subgruppe kann...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                         generations  true_index  \\\n",
       "0  [Die Inzidenz synchroner Zweittumoren oder von...           1   \n",
       "1  [In einer Studie des Instituts für Qualität un...           0   \n",
       "2  [Bei Vorliegen einer cN1 Subgruppe oder N1 Sub...           0   \n",
       "3  [Eine Radiochemotherapie soll nur an Einrichtu...           0   \n",
       "4  [Ähnlich wie die Schleimhaut kann auch die äuß...           0   \n",
       "\n",
       "                                              prompt   status  \\\n",
       "0  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "1  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "2  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "3  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "4  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "\n",
       "                                         prediction1  accept answer  \n",
       "0                                               None   False  Nein.  \n",
       "1  In einer Studie des Instituts für Qualität und...    True    Ja.  \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...    True    Ja.  \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...    True    Ja.  \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...    True    Ja.  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt2predictions_1 = pd.read_parquet(output_path / 'prompt2predictions_1.parquet')\n",
    "prompt2predictions_1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sie haben entschieden, dass die erste Version des Satzes, die vom Modell als die wahrscheinlichste ausgewählt wurde, nicht korrekt ist und den ursprünglichen Satz mit aufgelösten Koordinationsellipsen nicht vollständig wiedergibt. Das Modell gibt vier weitere Versionen des Satzes zurück, die als die nächst wahrscheinlichsten Versionen ausgewählt wurden. Bitte lesen Sie sich diese vier Versionen sorgfältig durch und wählen Sie die Version aus, die Ihrer Meinung nach am besten den ursprünglichen Satz mit aufgelösten Koordinationsellipsen wiedergibt.\n",
      "\n",
      "1 - 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3/Stadien T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "2 - 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3/Stadien T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "3 - 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3-Stadien T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "4 - 'Die Inzidenz synchroner Zweittumoren oder von Fernmetastasen liegt bei Karzinomen der Mundhöhle, abhängig von der Größe des Primärtumors, zwischen 4% und 33%, wobei die Stadien T3 oder Stadien T4 und Patienten mit Lymphknotenbefall in Level IV besonders häufig betroffen sind [REF], [REF], [REF].'\n",
      "\n",
      "Bitte antworten Sie nur mit der richtigen Nummer und ohne Erklärung.\n"
     ]
    }
   ],
   "source": [
    "other_options_prompts = generate_best_fit_prompts(results_5.original, results_5.outputs_k, results_5.ground_truth, generate_prompt_other_options)\n",
    "print(other_options_prompts.iloc[0].prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>generations</th>\n",
       "      <th>true_index</th>\n",
       "      <th>prompt</th>\n",
       "      <th>status</th>\n",
       "      <th>prediction1</th>\n",
       "      <th>accept</th>\n",
       "      <th>answer</th>\n",
       "      <th>prompt2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Die Inzidenz synchroner Zweittumoren oder von ...</td>\n",
       "      <td>[Die Inzidenz synchroner Zweittumoren oder von...</td>\n",
       "      <td>1</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>Nein.</td>\n",
       "      <td>Sie haben entschieden, dass die erste Version ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>[In einer Studie des Instituts für Qualität un...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>In einer Studie des Instituts für Qualität und...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "      <td>Sie haben entschieden, dass die erste Version ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bei Vorliegen einer cN1 oder N1 Subgruppe kann...</td>\n",
       "      <td>[Bei Vorliegen einer cN1 Subgruppe oder N1 Sub...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "      <td>Sie haben entschieden, dass die erste Version ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>[Eine Radiochemotherapie soll nur an Einrichtu...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Eine Radiochemotherapie soll nur an Einrichtun...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "      <td>Sie haben entschieden, dass die erste Version ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>[Ähnlich wie die Schleimhaut kann auch die äuß...</td>\n",
       "      <td>0</td>\n",
       "      <td>Sie haben ein Modell entwickelt, das Koordinat...</td>\n",
       "      <td>success</td>\n",
       "      <td>Ähnlich wie die Schleimhaut kann auch die äuße...</td>\n",
       "      <td>True</td>\n",
       "      <td>Ja.</td>\n",
       "      <td>Sie haben entschieden, dass die erste Version ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               input  \\\n",
       "0  Die Inzidenz synchroner Zweittumoren oder von ...   \n",
       "1  In einer Studie des Instituts für Qualität und...   \n",
       "2  Bei Vorliegen einer cN1 oder N1 Subgruppe kann...   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...   \n",
       "\n",
       "                                         generations  true_index  \\\n",
       "0  [Die Inzidenz synchroner Zweittumoren oder von...           1   \n",
       "1  [In einer Studie des Instituts für Qualität un...           0   \n",
       "2  [Bei Vorliegen einer cN1 Subgruppe oder N1 Sub...           0   \n",
       "3  [Eine Radiochemotherapie soll nur an Einrichtu...           0   \n",
       "4  [Ähnlich wie die Schleimhaut kann auch die äuß...           0   \n",
       "\n",
       "                                              prompt   status  \\\n",
       "0  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "1  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "2  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "3  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "4  Sie haben ein Modell entwickelt, das Koordinat...  success   \n",
       "\n",
       "                                         prediction1  accept answer  \\\n",
       "0                                               None   False  Nein.   \n",
       "1  In einer Studie des Instituts für Qualität und...    True    Ja.   \n",
       "2  Bei Vorliegen einer cN1 Subgruppe oder N1 Subg...    True    Ja.   \n",
       "3  Eine Radiochemotherapie soll nur an Einrichtun...    True    Ja.   \n",
       "4  Ähnlich wie die Schleimhaut kann auch die äuße...    True    Ja.   \n",
       "\n",
       "                                             prompt2  \n",
       "0  Sie haben entschieden, dass die erste Version ...  \n",
       "1  Sie haben entschieden, dass die erste Version ...  \n",
       "2  Sie haben entschieden, dass die erste Version ...  \n",
       "3  Sie haben entschieden, dass die erste Version ...  \n",
       "4  Sie haben entschieden, dass die erste Version ...  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt2predictions_2 = pd.concat([prompt2predictions_1, other_options_prompts.prompt.rename('prompt2')], axis=1)\n",
    "prompt2predictions_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def call_api_otheroptions_classifier(prompt_df, debug=False):\n",
    "    predictions = []\n",
    "    for i, row in tqdm(list(prompt_df.iterrows())):\n",
    "        if row.accept:\n",
    "            if debug:\n",
    "                print(f'{i} was accepted previously.')\n",
    "            predictions.append({'status2': 'accept_1', 'prediction2' : row.prediction1, 'answer2' : None})\n",
    "        else:\n",
    "            prompt1 = row.prompt\n",
    "            answer = row.answer\n",
    "            prompt2 = row.prompt2\n",
    "\n",
    "            messages = [(\"user\", prompt1), (\"assistant\", answer), (\"user\", prompt2)]\n",
    "\n",
    "            answer2 = get_openai_response_chatgpt(messages)\n",
    "\n",
    "            numbers = re.findall(r'\\d+', answer2)\n",
    "            if len(numbers) > 1:\n",
    "                if debug:\n",
    "                    print(f'more numbers than expected {numbers}')\n",
    "                predictions.append({'status2': 'error_multiple', 'prediction2' : row.generations[1], 'answer2' : answer2})\n",
    "            if len(numbers) == 0:\n",
    "                if debug:\n",
    "                    print(f'no numbers found')\n",
    "                predictions.append({'status2': 'no_numbers', 'prediction2' : row.generations[1], 'answer2' : answer2})\n",
    "            else:\n",
    "                index = int(numbers[0])\n",
    "                if index > 4:\n",
    "                    if debug:\n",
    "                        print(f'Index is out of bounds. Something went wrong with the API Answer. Defaulting to 0')\n",
    "                    predictions.append({'status2': 'no_numbers', 'prediction2' : row.generations[1], 'answer2' : answer2})\n",
    "                else:\n",
    "                    predictions.append({'status2': 'success', 'prediction2' : row.generations[index], 'answer2' : answer2})\n",
    "    return pd.DataFrame(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ce6dedbf8b64272be42859077f4314b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1201 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:retry.api:HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-06 13:47:20 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Berlin</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b3a73d9bc032685</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">141.89.221.208</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "), retrying in 2 seconds...\n"
     ]
    }
   ],
   "source": [
    "predictions_2 = call_api_otheroptions_classifier(prompt2predictions_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "result2 = pd.concat([prompt2predictions_2, predictions_2], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_2 = error_analysis(result2.prediction2, val_df.full_resolution, result2.input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         593\n",
       "tp         487\n",
       "replace     24\n",
       "delete      24\n",
       "insert      23\n",
       "fn          21\n",
       "fp          18\n",
       "complex     11\n",
       "Name: error_type, dtype: int64"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors_2.error_type.value_counts()# / len(errors_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 18s, sys: 75.6 ms, total: 1min 19s\n",
      "Wall time: 1min 21s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'eval/exact_match': 0.8984179850124896,\n",
       " 'eval/gleu': 0.9864061103437068,\n",
       " 'eval/edit_distance_rel': 0.9502187577476662}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_scores(errors_2, SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Baseline TOP 1 only\n",
    "errors_top1 = pd.read_parquet(output_path / 'results_top1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         590\n",
       "tp         499\n",
       "replace     22\n",
       "fp          21\n",
       "delete      20\n",
       "fn          20\n",
       "insert      17\n",
       "complex     12\n",
       "Name: error_type, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors_top1.error_type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 18s, sys: 83.9 ms, total: 1min 18s\n",
      "Wall time: 1min 20s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test/exact_match': 0.906744379683597,\n",
       " 'test/gleu': 0.9863425249277339,\n",
       " 'test/edit_distance_rel': 0.9496346247979276}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_scores(errors_top1, SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tn         590\n",
       "tp         497\n",
       "delete      24\n",
       "replace     22\n",
       "fp          21\n",
       "fn          20\n",
       "insert      16\n",
       "complex     11\n",
       "Name: error_type, dtype: int64"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors_temp = error_analysis(result2.generations.map(lambda l: l[0]), val_df.full_resolution, result2.input)\n",
    "errors_temp.error_type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ellipses]",
   "language": "python",
   "name": "conda-env-ellipses-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}