결과물한눈에보기.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a5792089-ea2b-434a-afe2-7aa50bb89c3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# %pip install tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3669d41-ee41-4961-82f8-fb778b12a541",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e9d99ca-1175-4d75-b9b2-04f119960ad2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "25a7e405-1368-46aa-ba51-0bc7a98962df",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "852c83ff-f39c-41a4-b6f9-05f22ad06e9b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# results/korean_origin_bench/20b/00_shot.json\n",
    "def _get_metric_name(v):\n",
    "    metrics = ['f1', 'macro_f1', 'acc_norm', 'acc']\n",
    "    for m in metrics:\n",
    "        if v.get(m):\n",
    "            return {\n",
    "                'metric': m,\n",
    "                'value': v[m],\n",
    "            }\n",
    "\n",
    "def get_df_klue(path, model_name=''):\n",
    "    data = []\n",
    "    for i in ['0', '5', '10', '50']:\n",
    "        shot = f'{path}/{i}_shot.json'\n",
    "        try:\n",
    "            data.append(\n",
    "                {\n",
    "                    f\"{k} ({_get_metric_name(v)['metric']})\": _get_metric_name(v)['value']\n",
    "                    for k, v in json.load(open(shot))['results'].items()\n",
    "                }\n",
    "            )\n",
    "        except FileNotFoundError:\n",
    "            pass\n",
    "    df = pd.DataFrame(data, index=[0, 5, 10, 50][:len(data)]).T\n",
    "    print(df.to_markdown())\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f5b870a-58e3-4a21-b860-cc406f1eed8f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kohatespeech (macro_f1)          | 0.278224 | 0.378693 | 0.370702 | 0.509343 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.556898 | 0.581788 | 0.667511 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.248404 | 0.484745 | 0.473659 | 0.461714 |\n",
      "| korunsmile (f1)                  | 0.327145 | 0.329163 | 0.347889 | 0.395522 |\n",
      "| nsmc (acc)                       | 0.6442   | 0.87702  | 0.89982  | 0.90984  |\n",
      "| pawsx_ko (acc)                   | 0.5355   | 0.5455   | 0.5435   | 0.5255   |\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>5</th>\n",
       "      <th>10</th>\n",
       "      <th>50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kohatespeech (macro_f1)</th>\n",
       "      <td>0.278224</td>\n",
       "      <td>0.378693</td>\n",
       "      <td>0.370702</td>\n",
       "      <td>0.509343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kohatespeech_apeach (macro_f1)</th>\n",
       "      <td>0.337667</td>\n",
       "      <td>0.556898</td>\n",
       "      <td>0.581788</td>\n",
       "      <td>0.667511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kohatespeech_gen_bias (macro_f1)</th>\n",
       "      <td>0.248404</td>\n",
       "      <td>0.484745</td>\n",
       "      <td>0.473659</td>\n",
       "      <td>0.461714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>korunsmile (f1)</th>\n",
       "      <td>0.327145</td>\n",
       "      <td>0.329163</td>\n",
       "      <td>0.347889</td>\n",
       "      <td>0.395522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nsmc (acc)</th>\n",
       "      <td>0.644200</td>\n",
       "      <td>0.877020</td>\n",
       "      <td>0.899820</td>\n",
       "      <td>0.909840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pawsx_ko (acc)</th>\n",
       "      <td>0.535500</td>\n",
       "      <td>0.545500</td>\n",
       "      <td>0.543500</td>\n",
       "      <td>0.525500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        0         5         10        50\n",
       "kohatespeech (macro_f1)           0.278224  0.378693  0.370702  0.509343\n",
       "kohatespeech_apeach (macro_f1)    0.337667  0.556898  0.581788  0.667511\n",
       "kohatespeech_gen_bias (macro_f1)  0.248404  0.484745  0.473659  0.461714\n",
       "korunsmile (f1)                   0.327145  0.329163  0.347889  0.395522\n",
       "nsmc (acc)                        0.644200  0.877020  0.899820  0.909840\n",
       "pawsx_ko (acc)                    0.535500  0.545500  0.543500  0.525500"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_df_klue('results/klue_etc_bench/home/jovyan/beomi/llama2-koen-13b/60b', 'llama2-koen-13b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5d226502-7541-427b-8b36-bec5489ca296",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['results/all/42MARU/GenAI-llama2-ko-en-platypus',\n",
       " 'results/all/EleutherAI/polyglot-ko-12.8b',\n",
       " 'results/all/EleutherAI/polyglot-ko-5.8b',\n",
       " 'results/all/HumanF-MarkrAI/pub-llama-13B-v3',\n",
       " 'results/all/KT-AI/midm-bitext-S-7B-inst-v1',\n",
       " 'results/all/beomi/Yi-Ko-6B',\n",
       " 'results/all/beomi/Yi-Ko-6B-20B',\n",
       " 'results/all/beomi/llama-2-ko-7b',\n",
       " 'results/all/beomi/llama-2-ko-7b-emb-dev',\n",
       " 'results/all/beomi/llama-2-ko-7b-emb-dev-7B',\n",
       " 'results/all/beomi/open-llama-2-ko-7b',\n",
       " 'results/all/beomi/open-llama-2-ko-7b-dev',\n",
       " 'results/all/beomi/open-llama-2-ko-7b-dev-v1',\n",
       " 'results/all/hyunseoki/ko-en-llama2-13b',\n",
       " 'results/all/jyoung105/KoR-Orca-Platypus-13B-neft',\n",
       " 'results/all/kyujinpy/KoR-Orca-Platypus-13B']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "various_models = sorted(glob('results/all/*/*'))\n",
    "various_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a93c00e7-941d-4149-9efe-bd22e66a039e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results/all/42MARU/GenAI-llama2-ko-en-platypus\n",
      "|                                  |        0 |        5 |       10 |\n",
      "|:---------------------------------|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.75496  | 0.813542 | 0.826657 |\n",
      "| kobest_copa (macro_f1)           | 0.789858 | 0.8109   | 0.812864 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.46552  | 0.481476 | 0.487711 |\n",
      "| kobest_sentineg (macro_f1)       | 0.599451 | 0.942065 | 0.952136 |\n",
      "| kohatespeech (macro_f1)          | 0.409181 | 0.368872 | 0.407597 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.340951 | 0.599693 | 0.63309  |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.137929 | 0.500808 | 0.47299  |\n",
      "| korunsmile (f1)                  | 0.4154   | 0.343849 | 0.385809 |\n",
      "| nsmc (acc)                       | 0.58484  | 0.8477   | 0.86622  |\n",
      "| pawsx_ko (acc)                   | 0.5405   | 0.531    | 0.535    |\n",
      "\n",
      "results/all/EleutherAI/polyglot-ko-12.8b\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.39744  | 0.550444 | 0.571613 | 0.581169 |\n",
      "| kobest_copa (macro_f1)           | 0.779573 | 0.803799 | 0.796624 | 0.824853 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.493165 | 0.499706 | 0.485373 | 0.502159 |\n",
      "| kobest_sentineg (macro_f1)       | 0.502426 | 0.962201 | 0.977329 | 0.979849 |\n",
      "| kohatespeech (macro_f1)          | 0.264347 | 0.370575 | 0.284216 | 0.299345 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.380782 | 0.503566 | 0.476384 | 0.445471 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.169721 | 0.469281 | 0.485368 | 0.457998 |\n",
      "| korunsmile (f1)                  | 0.396283 | 0.334776 | 0.32845  | 0.292387 |\n",
      "| nsmc (acc)                       | 0.56508  | 0.79774  | 0.8291   | 0.87034  |\n",
      "| pawsx_ko (acc)                   | 0.522    | 0.509    | 0.498    | 0.5505   |\n",
      "\n",
      "results/all/EleutherAI/polyglot-ko-5.8b\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.501258 | 0.554344 | 0.542351 | 0.554364 |\n",
      "| kobest_copa (macro_f1)           | 0.758768 | 0.770305 | 0.771382 | 0.792617 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.493639 | 0.493747 | 0.4992   | 0.507357 |\n",
      "| kobest_sentineg (macro_f1)       | 0.414397 | 0.949614 | 0.967241 | 0.964725 |\n",
      "| kohatespeech (macro_f1)          | 0.267213 | 0.347344 | 0.305576 | 0.303377 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.488004 | 0.447996 | 0.329979 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.464119 | 0.461714 | 0.461714 |\n",
      "| korunsmile (f1)                  | 0.410939 | 0.321295 | 0.326014 | 0.283732 |\n",
      "| nsmc (acc)                       | 0.55824  | 0.74962  | 0.7911   | 0.83588  |\n",
      "| pawsx_ko (acc)                   | 0.4865   | 0.523    | 0.517    | 0.535    |\n",
      "\n",
      "results/all/HumanF-MarkrAI/pub-llama-13B-v3\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.334282 | 0.545224 | 0.536021 | 0.543549 |\n",
      "| kobest_copa (macro_f1)           | 0.755683 | 0.774692 | 0.778591 | 0.795791 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.446513 | 0.455871 | 0.455861 | 0.466838 |\n",
      "| kobest_sentineg (macro_f1)       | 0.752343 | 0.92695  | 0.949596 | 0.949622 |\n",
      "| kohatespeech (macro_f1)          | 0.424856 | 0.365288 | 0.394254 | 0.364301 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.578401 | 0.637884 | 0.723222 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.475542 | 0.505903 | 0.502546 |\n",
      "| korunsmile (f1)                  | 0.423588 | 0.358856 | 0.395571 | 0.425031 |\n",
      "| nsmc (acc)                       | 0.65294  | 0.83548  | 0.85876  | 0.87318  |\n",
      "| pawsx_ko (acc)                   | 0.532    | 0.544    | 0.535    | 0.5495   |\n",
      "\n",
      "results/all/KT-AI/midm-bitext-S-7B-inst-v1\n",
      "|                                  |        0 |        5 |       10 |\n",
      "|:---------------------------------|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.842593 | 0.771203 | 0.758757 |\n",
      "| kobest_copa (macro_f1)           | 0.782635 | 0.837907 | 0.836863 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.488192 | 0.495704 | 0.484098 |\n",
      "| kobest_sentineg (macro_f1)       | 0.45601  | 0.977325 | 0.964734 |\n",
      "| kohatespeech (macro_f1)          | 0.303697 | 0.408802 | 0.382379 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.4573   | 0.811319 | 0.813246 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.156223 | 0.539454 | 0.545435 |\n",
      "| korunsmile (f1)                  | 0.374452 | 0.333345 | 0.320071 |\n",
      "| nsmc (acc)                       | 0.54434  | 0.86494  | 0.866    |\n",
      "| pawsx_ko (acc)                   | 0.521    | 0.544    | 0.546    |\n",
      "\n",
      "results/all/beomi/Yi-Ko-6B\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.705806 | 0.79905  | 0.814299 | 0.81704  |\n",
      "| kobest_copa (macro_f1)           | 0.775604 | 0.808899 | 0.816866 | 0.842943 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.500876 | 0.498673 | 0.493507 | 0.492183 |\n",
      "| kobest_sentineg (macro_f1)       | 0.404371 | 0.967254 | 0.982368 | 0.974811 |\n",
      "| kohatespeech (macro_f1)          | 0.353428 | 0.351804 | 0.402423 | 0.503764 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.498679 | 0.471962 | 0.608401 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.484745 | 0.474475 | 0.461714 |\n",
      "| korunsmile (f1)                  | 0.382804 | 0.349344 | 0.391383 | 0.432875 |\n",
      "| nsmc (acc)                       | 0.55064  | 0.8801   | 0.89866  | 0.9071   |\n",
      "| pawsx_ko (acc)                   | 0.5145   | 0.54     | 0.538    | 0.5165   |\n",
      "\n",
      "results/all/beomi/Yi-Ko-6B-20B\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.478334 | 0.723701 | 0.747499 | 0.764627 |\n",
      "| kobest_copa (macro_f1)           | 0.747596 | 0.794827 | 0.798853 | 0.814933 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.490449 | 0.491227 | 0.489294 | 0.507597 |\n",
      "| kobest_sentineg (macro_f1)       | 0.567971 | 0.972291 | 0.969772 | 0.972291 |\n",
      "| kohatespeech (macro_f1)          | 0.323056 | 0.376289 | 0.397651 | 0.497185 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.53343  | 0.554911 | 0.609341 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.471234 | 0.474475 | 0.486704 |\n",
      "| korunsmile (f1)                  | 0.421594 | 0.369275 | 0.386737 | 0.412408 |\n",
      "| nsmc (acc)                       | 0.5925   | 0.86962  | 0.8926   | 0.90436  |\n",
      "| pawsx_ko (acc)                   | 0.5085   | 0.5435   | 0.529    | 0.545    |\n",
      "\n",
      "results/all/beomi/llama-2-ko-7b\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.612147 | 0.682832 | 0.713392 | 0.71622  |\n",
      "| kobest_copa (macro_f1)           | 0.759784 | 0.799843 | 0.807907 | 0.829976 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.447951 | 0.460632 | 0.464623 | 0.458628 |\n",
      "| kobest_sentineg (macro_f1)       | 0.3517   | 0.969773 | 0.977329 | 0.97481  |\n",
      "| kohatespeech (macro_f1)          | 0.314636 | 0.383336 | 0.357491 | 0.366585 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.346127 | 0.567627 | 0.583391 | 0.629269 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.204651 | 0.509189 | 0.471078 | 0.451119 |\n",
      "| korunsmile (f1)                  | 0.290663 | 0.306208 | 0.304279 | 0.343946 |\n",
      "| nsmc (acc)                       | 0.57942  | 0.84242  | 0.87368  | 0.8939   |\n",
      "| pawsx_ko (acc)                   | 0.538    | 0.52     | 0.5275   | 0.5195   |\n",
      "\n",
      "results/all/beomi/llama-2-ko-7b-emb-dev\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.35205  | 0.518172 | 0.633135 | 0.644859 |\n",
      "| kobest_copa (macro_f1)           | 0.711538 | 0.741701 | 0.747708 | 0.770879 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.412284 | 0.437617 | 0.410904 | 0.428184 |\n",
      "| kobest_sentineg (macro_f1)       | 0.372007 | 0.949619 | 0.957178 | 0.967254 |\n",
      "| kohatespeech (macro_f1)          | 0.318217 | 0.396189 | 0.395318 | 0.465953 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.526961 | 0.568284 | 0.621547 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.489657 | 0.484745 | 0.45612  |\n",
      "| korunsmile (f1)                  | 0.302557 | 0.346101 | 0.362893 | 0.275587 |\n",
      "| nsmc (acc)                       | 0.53584  | 0.8287   | 0.85362  | 0.87214  |\n",
      "| pawsx_ko (acc)                   | 0.5035   | 0.503    | 0.512    | 0.457    |\n",
      "\n",
      "results/all/beomi/llama-2-ko-7b-emb-dev-7B\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.337452 | 0.570559 | 0.585608 | 0.567486 |\n",
      "| kobest_copa (macro_f1)           | 0.665517 | 0.69096  | 0.686659 | 0.698672 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.383174 | 0.379488 | 0.380637 | 0.383733 |\n",
      "| kobest_sentineg (macro_f1)       | 0.367966 | 0.91937  | 0.96473  | 0.947103 |\n",
      "| kohatespeech (macro_f1)          | 0.300627 | 0.362646 | 0.364923 | 0.287482 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.486543 | 0.452716 | 0.335808 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.489203 | 0.461098 | 0.461714 |\n",
      "| korunsmile (f1)                  | 0.35822  | 0.367208 | 0.381937 | 0.376735 |\n",
      "| nsmc (acc)                       | 0.50714  | 0.80386  | 0.83964  | 0.86138  |\n",
      "| pawsx_ko (acc)                   | 0.539    | 0.4915   | 0.5225   | 0.521    |\n",
      "\n",
      "results/all/beomi/open-llama-2-ko-7b\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.341852 | 0.645625 | 0.631874 | 0.626114 |\n",
      "| kobest_copa (macro_f1)           | 0.76727  | 0.787425 | 0.786739 | 0.806706 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.478635 | 0.480596 | 0.481216 | 0.478697 |\n",
      "| kobest_sentineg (macro_f1)       | 0.3517   | 0.959691 | 0.979848 | 0.979848 |\n",
      "| kohatespeech (macro_f1)          | 0.318428 | 0.359593 | 0.36305  | 0.386928 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.454278 | 0.382714 | 0.420392 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.127236 | 0.479041 | 0.484745 | 0.460481 |\n",
      "| korunsmile (f1)                  | 0.414869 | 0.362479 | 0.383989 | 0.301188 |\n",
      "| nsmc (acc)                       | 0.55512  | 0.85376  | 0.8617   | 0.85884  |\n",
      "| pawsx_ko (acc)                   | 0.519    | 0.5285   | 0.5285   | 0.53     |\n",
      "\n",
      "results/all/beomi/open-llama-2-ko-7b-dev\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.336491 | 0.65339  | 0.665316 | 0.687771 |\n",
      "| kobest_copa (macro_f1)           | 0.789515 | 0.789515 | 0.789629 | 0.818752 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.469814 | 0.474781 | 0.479316 | 0.479225 |\n",
      "| kobest_sentineg (macro_f1)       | 0.357169 | 0.947098 | 0.96473  | 0.977329 |\n",
      "| kohatespeech (macro_f1)          | 0.363024 | 0.362087 | 0.34152  | 0.353998 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.338867 | 0.482564 | 0.429394 | 0.504437 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.135272 | 0.475359 | 0.494577 | 0.489707 |\n",
      "| korunsmile (f1)                  | 0.388456 | 0.368589 | 0.369784 | 0.23569  |\n",
      "| nsmc (acc)                       | 0.5541   | 0.82568  | 0.86266  | 0.87248  |\n",
      "| pawsx_ko (acc)                   | 0.5425   | 0.5255   | 0.528    | 0.5      |\n",
      "\n",
      "results/all/beomi/open-llama-2-ko-7b-dev-v1\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.41297  | 0.617865 | 0.65946  | 0.609196 |\n",
      "| kobest_copa (macro_f1)           | 0.783688 | 0.798807 | 0.804672 | 0.8168   |\n",
      "| kobest_hellaswag (macro_f1)      | 0.458644 | 0.469164 | 0.474449 | 0.478576 |\n",
      "| kobest_sentineg (macro_f1)       | 0.399297 | 0.954653 | 0.964734 | 0.97733  |\n",
      "| kohatespeech (macro_f1)          | 0.380455 | 0.366437 | 0.366917 | 0.310756 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.540037 | 0.569105 | 0.577291 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.461353 | 0.512783 | 0.527541 |\n",
      "| korunsmile (f1)                  | 0.39677  | 0.361002 | 0.363335 | 0.228674 |\n",
      "| nsmc (acc)                       | 0.53268  | 0.8581   | 0.87594  | 0.87642  |\n",
      "| pawsx_ko (acc)                   | 0.5345   | 0.5055   | 0.5175   | 0.514    |\n",
      "\n",
      "results/all/hyunseoki/ko-en-llama2-13b\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.595474 | 0.730298 | 0.740454 | 0.735644 |\n",
      "| kobest_copa (macro_f1)           | 0.776812 | 0.788846 | 0.804878 | 0.811973 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.470698 | 0.483454 | 0.475882 | 0.488026 |\n",
      "| kobest_sentineg (macro_f1)       | 0.522626 | 0.949622 | 0.95466  | 0.969773 |\n",
      "| kohatespeech (macro_f1)          | 0.392963 | 0.369294 | 0.36653  | 0.342853 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.342078 | 0.575979 | 0.586214 | 0.589353 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.138892 | 0.497935 | 0.469296 | 0.551062 |\n",
      "| korunsmile (f1)                  | 0.432898 | 0.346927 | 0.388364 | 0.436292 |\n",
      "| nsmc (acc)                       | 0.5992   | 0.84662  | 0.8648   | 0.87848  |\n",
      "| pawsx_ko (acc)                   | 0.4975   | 0.5275   | 0.531    | 0.5225   |\n",
      "\n",
      "results/all/jyoung105/KoR-Orca-Platypus-13B-neft\n",
      "|                                  |        0 |        5 |       10 |       50 |\n",
      "|:---------------------------------|---------:|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.655843 | 0.354271 | 0.363429 | 0.356151 |\n",
      "| kobest_copa (macro_f1)           | 0.754733 | 0.768648 | 0.782922 | 0.79992  |\n",
      "| kobest_hellaswag (macro_f1)      | 0.455897 | 0.449564 | 0.457505 | 0.442365 |\n",
      "| kobest_sentineg (macro_f1)       | 0.532725 | 0.949614 | 0.95214  | 0.962216 |\n",
      "| kohatespeech (macro_f1)          | 0.394561 | 0.360054 | 0.360721 | 0.320941 |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.344554 | 0.384063 | 0.36931  | 0.359277 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.136255 | 0.398311 | 0.333205 | 0.364247 |\n",
      "| korunsmile (f1)                  | 0.408954 | 0.342518 | 0.374663 | 0.382229 |\n",
      "| nsmc (acc)                       | 0.59698  | 0.85924  | 0.87258  | 0.88072  |\n",
      "| pawsx_ko (acc)                   | 0.5225   | 0.5515   | 0.5435   | 0.5515   |\n",
      "\n",
      "results/all/kyujinpy/KoR-Orca-Platypus-13B\n",
      "|                                  |        0 |        5 |       10 |\n",
      "|:---------------------------------|---------:|---------:|---------:|\n",
      "| kobest_boolq (macro_f1)          | 0.334282 | 0.5338   | 0.531394 |\n",
      "| kobest_copa (macro_f1)           | 0.755683 | 0.773707 | 0.778591 |\n",
      "| kobest_hellaswag (macro_f1)      | 0.446513 | 0.455871 | 0.457967 |\n",
      "| kobest_sentineg (macro_f1)       | 0.752343 | 0.92695  | 0.949596 |\n",
      "| kohatespeech (macro_f1)          | 0.422185 | 0.365288 | 0.3971   |\n",
      "| kohatespeech_apeach (macro_f1)   | 0.337667 | 0.571251 | 0.622383 |\n",
      "| kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.484719 | 0.507788 |\n",
      "| korunsmile (f1)                  | 0.423391 | 0.358939 | 0.396573 |\n",
      "| nsmc (acc)                       | 0.65288  | 0.8356   | 0.85846  |\n",
      "| pawsx_ko (acc)                   | 0.534    | 0.5445   | 0.5345   |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in various_models:\n",
    "    print(model)\n",
    "    get_df_klue(model)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "071411b4-1497-48d0-8440-2abda75dd499",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# results/all/beomi/llama-2-ko-7b-emb-dev\n",
    "# |                                  |        0 |        5 |       10 |       50 |\n",
    "# |:---------------------------------|---------:|---------:|---------:|---------:|\n",
    "# | kobest_boolq (macro_f1)          | 0.337452 | 0.570559 | 0.585608 | 0.567486 |\n",
    "# | kobest_copa (macro_f1)           | 0.665517 | 0.69096  | 0.686659 | 0.698672 |\n",
    "# | kobest_hellaswag (macro_f1)      | 0.383174 | 0.379488 | 0.380637 | 0.383733 |\n",
    "# | kobest_sentineg (macro_f1)       | 0.367966 | 0.91937  | 0.96473  | 0.947103 |\n",
    "# | kohatespeech (macro_f1)          | 0.300627 | 0.362646 | 0.364923 | 0.287482 |\n",
    "# | kohatespeech_apeach (macro_f1)   | 0.337667 | 0.486543 | 0.452716 | 0.335808 |\n",
    "# | kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.489203 | 0.461098 | 0.461714 |\n",
    "# | korunsmile (f1)                  | 0.35822  | 0.367208 | 0.381937 | 0.376735 |\n",
    "# | nsmc (acc)                       | 0.50714  | 0.80386  | 0.83964  | 0.86138  |\n",
    "# | pawsx_ko (acc)                   | 0.539    | 0.4915   | 0.5225   | 0.521    |\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "821b7c25-8f9e-4598-be7c-66df019331ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3048c2e-d84e-4f00-ba8b-ef5e7ccb8b70",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}