Name_Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:51:55.932209Z",
     "start_time": "2023-08-06T08:51:51.332708Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "from rdkit import Chem\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import openai\n",
    "import datetime\n",
    "\n",
    "\n",
    "def molToCanonical(smiles):\n",
    "    mol = Chem.MolFromSmiles(smiles)\n",
    "    canonical_smiles = Chem.MolToSmiles(mol)\n",
    "    return canonical_smiles\n",
    "\n",
    "from steamship import Steamship\n",
    "# Set up the OpenAI API client\n",
    "openai.api_key = 'Your API Key'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:51:56.023886Z",
     "start_time": "2023-08-06T08:51:55.939218Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Name Prediction/pubchem.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:51:56.045635Z",
     "start_time": "2023-08-06T08:51:56.032003Z"
    },
    "run_control": {
     "marked": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = df[~df['iupac'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:51:56.072154Z",
     "start_time": "2023-08-06T08:51:56.053925Z"
    }
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(df, test_size=530, random_state=42)\n",
    "\n",
    "valid, test = train_test_split(test, test_size=500, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:08.102813Z",
     "start_time": "2023-08-06T08:49:08.078949Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_scaffold_similar_molecules(target_smiles, molecule_scaffold_list, molecule_smiles_list, n=5):\n",
    "    target_mol = Chem.MolFromSmiles(target_smiles)\n",
    "    target_scaffold = MurckoScaffold.GetScaffoldForMol(target_mol)\n",
    "    target_fp = rdMolDescriptors.GetMorganFingerprint(target_scaffold, 2)\n",
    "\n",
    "    similarities = []\n",
    "\n",
    "    for idx, scaffold_fp in enumerate(molecule_scaffold_list):\n",
    "        try:\n",
    "            tanimoto_similarity = DataStructs.TanimotoSimilarity(target_fp, scaffold_fp)\n",
    "            similarities.append((idx, tanimoto_similarity))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue\n",
    "\n",
    "    similarities.sort(key=lambda x: x[1], reverse=True)\n",
    "    top_5_similar_molecules = similarities[:n]\n",
    "\n",
    "    return [molecule_smiles_list[i[0]] for i in top_5_similar_molecules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:09.192173Z",
     "start_time": "2023-08-06T08:49:08.325898Z"
    }
   },
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "from rdkit.Chem.Scaffolds import MurckoScaffold\n",
    "from rdkit.Chem import DataStructs\n",
    "from rdkit.Chem import rdMolDescriptors\n",
    "from tqdm import tqdm\n",
    "\n",
    "def get_scaffold_fp(x):\n",
    "    try:\n",
    "        mol = Chem.MolFromSmiles(x)\n",
    "        scaffold = MurckoScaffold.GetScaffoldForMol(mol)\n",
    "        scaffold_fp = rdMolDescriptors.GetMorganFingerprint(scaffold, 2)\n",
    "        return scaffold_fp\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:32.562996Z",
     "start_time": "2023-08-06T08:49:09.200768Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[16:49:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:11] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:11] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:16] Explicit valence for atom # 39 Cl, 3, is greater than permitted\n",
      "[16:49:19] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:20] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:21] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:21] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:21] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:22] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:22] Explicit valence for atom # 2 Cl, 3, is greater than permitted\n",
      "[16:49:23] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:23] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:23] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:23] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:25] Explicit valence for atom # 7 Br, 3, is greater than permitted\n",
      "[16:49:26] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:28] Explicit valence for atom # 25 Br, 3, is greater than permitted\n",
      "[16:49:28] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:49:28] Explicit valence for atom # 2 Br, 3, is greater than permitted\n",
      "[16:49:29] Explicit valence for atom # 18 Br, 3, is greater than permitted\n"
     ]
    }
   ],
   "source": [
    "train['scaffold_fp'] = train['smiles'].apply(lambda x: get_scaffold_fp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:32.577915Z",
     "start_time": "2023-08-06T08:49:32.570067Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:32.625857Z",
     "start_time": "2023-08-06T08:49:32.595362Z"
    }
   },
   "outputs": [],
   "source": [
    "train = train[~train['scaffold_fp'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:49:32.650680Z",
     "start_time": "2023-08-06T08:49:32.634247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9356"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:30.223123Z",
     "start_time": "2023-04-30T22:08:30.219491Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_smiles2iupac(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular SMILES, your task is to predict the IUPAC name using your experienced chemical IUPAC name knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided.\\n\"\n",
    "    \n",
    "    for example in examples:\n",
    "        prompt += f\"Molecular SMILES: {example[0]}\\nMolecular IUPAC name: {example[1]}\\n\"\n",
    "    prompt += f\"Molecular SMILES: {input_text}\\nMolecular IUPAC name:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:30.384798Z",
     "start_time": "2023-04-30T22:08:30.381242Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_iupac2smiles(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular IUPAC name, your task is to predict the molecular SMILES using your experienced chemical IUPAC name knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided. You should only reply with molecular SMILES string notations to represent the IUPAC name. The SMILES must be valid and chemically reasonable. \\n\"\n",
    "    for example in examples:\n",
    "        prompt += f\"Molecular IUPAC name: {example[0]}\\nMolecular SMILES: {example[1]}\\n\"\n",
    "    prompt += f\"Molecular IUPAC name: {input_text}\\nMolecular SMILES:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:30.543756Z",
     "start_time": "2023-04-30T22:08:30.540085Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_smiles2formula(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular SMILES, your task is to predict the molecular formula using your experienced chemical molecular formula knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided.\\n\"\n",
    "    \n",
    "    for example in examples:\n",
    "        prompt += f\"Molecular SMILES: {example[0]}\\nMolecular formula: {example[1]}\\n\"\n",
    "    prompt += f\"Molecular SMILES: {input_text}\\nMolecular formula:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:30.709195Z",
     "start_time": "2023-04-30T22:08:30.705599Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_iupac2formula(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular formula, your task is to predict the molecular SMILES using your experienced chemical molecular formula knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided. You should only reply with molecular SMILES string notations to represent the molecular formula. The SMILES must be valid and chemically reasonable.\\n\"\n",
    "    \n",
    "    for example in examples:\n",
    "        prompt += f\"Molecular formula: {example[0]}\\nMolecular SMILES: {example[1]}\\n\"\n",
    "    prompt += f\"Molecular formula: {input_text}\\nMolecular SMILES:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:33.737437Z",
     "start_time": "2023-04-30T22:08:33.729790Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "def generate_response_by_davinci(prompt, model_engine = 'text-davinci-003'):\n",
    "    time.sleep(2)\n",
    "    completion = openai.Completion.create(\n",
    "      engine=model_engine,\n",
    "      prompt=prompt,\n",
    "      temperature=0.2,\n",
    "      max_tokens=256,\n",
    "#       top_p=1.0,\n",
    "      frequency_penalty=0.0,\n",
    "      presence_penalty=0.0, \n",
    "      n=5,\n",
    "    )\n",
    "#     message = completion.choices[0]['text'].strip()\n",
    "    \n",
    "    message = completion.choices\n",
    "    message = [i['text'].strip() for i in message]\n",
    "    return message\n",
    "\n",
    "def generate_response_by_gpt35(prompt, model_engine = \"gpt-3.5-turbo\"):\n",
    "    time.sleep(2)\n",
    "    completion = openai.ChatCompletion.create(\n",
    "        model=model_engine, temperature=0.2, n=5, \n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "    )\n",
    "#     message = completion.choices[0].message.content.strip()\n",
    "\n",
    "    message = completion.choices\n",
    "    message = [i.message.content.strip() for i in message]\n",
    "    return message\n",
    "\n",
    "def generate_response_by_gpt4(prompt):\n",
    "    time.sleep(2)\n",
    "    # Create a Steamship client\n",
    "    # NOTE: When developing a package, just use `self.client`\n",
    "    client = Steamship(workspace=\"gpt-4-67j\")\n",
    "\n",
    "    # Create an instance of this generator\n",
    "    generator = client.use_plugin('gpt-4', config={\"temperature\":0.2, \"n\": 5})\n",
    "\n",
    "    # Generate text\n",
    "    task = generator.generate(text=prompt)\n",
    "    # Wait for completion of the task.\n",
    "    task.wait()\n",
    "    # Print the output\n",
    "#     message = task.output.blocks[0].text.strip()\n",
    "    message = task.output.blocks\n",
    "    message = [i.text.strip() for i in message]\n",
    "    return message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:35.254996Z",
     "start_time": "2023-04-30T22:08:35.244597Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CID</th>\n",
       "      <th>smiles</th>\n",
       "      <th>iupac</th>\n",
       "      <th>formula</th>\n",
       "      <th>mol_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4909</th>\n",
       "      <td>110709753</td>\n",
       "      <td>CC(C)(C)C(=O)CN1CCN(CC1)C(=O)C2=CC(=CC=C2)Cl</td>\n",
       "      <td>1-[4-(3-chlorobenzoyl)piperazin-1-yl]-3,3-dime...</td>\n",
       "      <td>C17H23ClN2O2</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558</th>\n",
       "      <td>65924153</td>\n",
       "      <td>CC1CCCC(C1)(CN)CN2CCN(CC2)C3=NC=CS3</td>\n",
       "      <td>[3-methyl-1-[[4-(1,3-thiazol-2-yl)piperazin-1-...</td>\n",
       "      <td>C16H28N4S</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7806</th>\n",
       "      <td>142384555</td>\n",
       "      <td>CCC(C)(CC)C=CC(=O)O</td>\n",
       "      <td>(E)-4-ethyl-4-methylhex-2-enoic acid</td>\n",
       "      <td>C9H16O2</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            CID                                        smiles  \\\n",
       "4909  110709753  CC(C)(C)C(=O)CN1CCN(CC1)C(=O)C2=CC(=CC=C2)Cl   \n",
       "7558   65924153           CC1CCCC(C1)(CN)CN2CCN(CC2)C3=NC=CS3   \n",
       "7806  142384555                           CCC(C)(CC)C=CC(=O)O   \n",
       "\n",
       "                                                  iupac       formula  \\\n",
       "4909  1-[4-(3-chlorobenzoyl)piperazin-1-yl]-3,3-dime...  C17H23ClN2O2   \n",
       "7558  [3-methyl-1-[[4-(1,3-thiazol-2-yl)piperazin-1-...     C16H28N4S   \n",
       "7806               (E)-4-ethyl-4-methylhex-2-enoic acid       C9H16O2   \n",
       "\n",
       "      mol_length  \n",
       "4909          44  \n",
       "7558          35  \n",
       "7806          19  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:35.784543Z",
     "start_time": "2023-04-30T22:08:35.781005Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_input_output_columns_by_task(task):\n",
    "    if task == 'smiles2iupac':\n",
    "        return \"smiles\", \"iupac\"\n",
    "    elif task == 'smiles2formula':\n",
    "        return \"smiles\", \"formula\"\n",
    "    elif task == 'iupac2smiles':\n",
    "        return \"iupac\", \"smiles\"\n",
    "    elif task == 'formula2smiles':\n",
    "        return \"formula\", 'smiles'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:37.127798Z",
     "start_time": "2023-04-30T22:08:37.123897Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt(reactant, examples, task):\n",
    "    if task == 'smiles2iupac':\n",
    "        return create_prompt_smiles2iupac(reactant, examples)\n",
    "    elif task == 'smiles2formula':\n",
    "        return create_prompt_smiles2formula(reactant, examples)\n",
    "    elif task == 'iupac2smiles':\n",
    "        return create_prompt_iupac2smiles(reactant, examples)\n",
    "    elif task == 'formula2smiles':\n",
    "        return create_prompt_formula2smiles(reactant, examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:38.708916Z",
     "start_time": "2023-04-30T22:08:38.700935Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['python', 'java']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import difflib\n",
    "\n",
    "def similarity_ratio(s1, s2):\n",
    "    # Calculate the similarity ratio between the two strings\n",
    "    ratio = difflib.SequenceMatcher(None, s1, s2).ratio()\n",
    "    \n",
    "    # Return the similarity ratio\n",
    "    return ratio\n",
    "\n",
    "# Example usage:\n",
    "s1 = \"python\"\n",
    "s2 = \"java\"\n",
    "ratio = similarity_ratio(s1, s2)\n",
    "print(ratio)  # Output: 0.0\n",
    "\n",
    "\n",
    "def top_n_similar_strings(query, candidates, n=5):\n",
    "    # Calculate the Levenshtein distance between the query and each candidate\n",
    "    distances = [(c, similarity_ratio(query, c)) for c in candidates]\n",
    "    \n",
    "    # Sort the candidates by their Levenshtein distance to the query\n",
    "    sorted_distances = sorted(distances, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    # Get the top n candidates with the smallest Levenshtein distance\n",
    "    top_candidates = [d[0] for d in sorted_distances[:n]]\n",
    "    \n",
    "    # Return the top n candidates\n",
    "    return top_candidates\n",
    "top_n_similar_strings(\"pathon\", ['python', 'java'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:53:17.834127Z",
     "start_time": "2023-08-06T08:53:17.826030Z"
    }
   },
   "outputs": [],
   "source": [
    "def molToCanonical(smiles):\n",
    "    try:\n",
    "        mol = Chem.MolFromSmiles(smiles)\n",
    "        canonical_smiles = Chem.MolToSmiles(mol)\n",
    "        return canonical_smiles\n",
    "    except:\n",
    "        return smiles\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:41.593282Z",
     "start_time": "2023-04-30T22:08:41.576241Z"
    }
   },
   "outputs": [],
   "source": [
    "valid['smiles'] = valid['smiles'].apply(lambda x: molToCanonical(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:41.991888Z",
     "start_time": "2023-04-30T22:08:41.773856Z"
    }
   },
   "outputs": [],
   "source": [
    "train['smiles'] = train['smiles'].apply(lambda x: molToCanonical(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T08:53:23.679038Z",
     "start_time": "2023-08-06T08:53:23.159466Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[16:53:23] WARNING: not removing hydrogen atom without neighbors\n",
      "[16:53:23] Explicit valence for atom # 9 Br, 3, is greater than permitted\n"
     ]
    }
   ],
   "source": [
    "test['smiles'] = test['smiles'].apply(lambda x: molToCanonical(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T00:36:56.962357Z",
     "start_time": "2023-04-26T00:23:17.976664Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name Prediction/results/valid_smiles2iupac_gpt35_SMILES_20_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [03:26,  6.87s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2iupac', 'gpt35', 'SMILES', 20, 'Fixed_ICL', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]\n",
      "45\n",
      "Name Prediction/results/valid_smiles2formula_davinci003_SMILES_20_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [05:28, 10.96s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2formula', 'davinci003', 'SMILES', 20, 'Fixed_ICL', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]\n",
      "46\n",
      "Name Prediction/results/valid_smiles2formula_gpt4_SMILES_20_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [03:08,  6.27s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2formula', 'gpt4', 'SMILES', 20, 'Fixed_ICL', 0.03333333333333334, 0.03333333333333333, 0.06666666666666667, 0.0, 0.0, 0.06666666666666667]]\n",
      "47\n",
      "Name Prediction/results/valid_smiles2formula_gpt35_SMILES_20_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [01:35,  3.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2formula', 'gpt35', 'SMILES', 20, 'Fixed_ICL', 0.12666666666666665, 0.13333333333333333, 0.13333333333333333, 0.13333333333333333, 0.13333333333333333, 0.1]]\n",
      "48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "Models = ['davinci003', 'gpt4', 'gpt35']\n",
    "ICL_Sample_methods = ['Scaffold_SIM', 'Fixed_ICL']\n",
    "ICL_Samples = [5, 20]\n",
    "NAME_Format = ['SMILES']\n",
    "TASK_Types = ['iupac2smiles', 'formula2smiles', 'smiles2iupac', 'smiles2formula']\n",
    "detail_save_folder = 'Name Prediction/results/'\n",
    "\n",
    "params_idx = 0\n",
    "\n",
    "# 5， 26， 40,  44\n",
    "\n",
    "for sample_method in ICL_Sample_methods:\n",
    "    for sample_num in ICL_Samples:\n",
    "        for name in NAME_Format:\n",
    "            for task in TASK_Types:\n",
    "                for model in Models:\n",
    "                    \n",
    "                    details_results = []\n",
    "                    performance_results = []\n",
    "                    \n",
    "                    if params_idx < 44:\n",
    "                        params_idx +=1\n",
    "                        continue\n",
    "                    \n",
    "                    detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "                    log_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.log'.format(task, model, name, sample_num, sample_method)\n",
    "                    performance_file = detail_save_folder + 'valid_performance_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "                    print(detail_predict_file)\n",
    "\n",
    "                    # append new date\n",
    "                    # Get the current date and time\n",
    "                    now = datetime.datetime.now()\n",
    "                    # Convert the date and time to a string\n",
    "                    date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "                    with open(log_file, \"w+\") as file:\n",
    "                        file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "                    # init var\n",
    "                    predicted_products = []\n",
    "                    \n",
    "                    input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "                    # query\n",
    "                    for idx, row in tqdm(valid.iterrows()):\n",
    "                        reactant = row[input_col]\n",
    "                        product = row[output_col]\n",
    "\n",
    "                        # ICL examples\n",
    "                        if sample_method == 'Fixed_ICL':\n",
    "                            chunk = train.sample(sample_num, random_state=42)\n",
    "                        elif sample_method == 'Scaffold_SIM':\n",
    "                            if input_col == 'smiles':\n",
    "                                sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train['smiles']), n=sample_num)\n",
    "                            else:\n",
    "                                # similarity by leven\n",
    "                                sim = top_n_similar_strings(reactant, list(train[input_col]), n=sample_num)\n",
    "                            chunk = train[train[input_col].isin(sim)]\n",
    "                            \n",
    "                        examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "                        # build prompt and save\n",
    "                        prompt = create_prompt(reactant, examples, task)\n",
    "                        \n",
    "                        with open(log_file, \"a\") as file:\n",
    "                            file.write(prompt + \"\\n\")\n",
    "                            file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "                        # different model \n",
    "                        if model == 'davinci003':\n",
    "                            predicted_product = generate_response_by_davinci(prompt)\n",
    "                        elif model == 'gpt35':\n",
    "                            predicted_product = generate_response_by_gpt35(prompt)\n",
    "                        elif model == 'gpt4':\n",
    "                            predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "                        predicted_products.append(predicted_product)\n",
    "                        details_results.append([reactant] + [product] + predicted_product)\n",
    "\n",
    "                    # evaluate\n",
    "                    acc_list = []\n",
    "                    for repeat in range(5):\n",
    "                        tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "                        correct = 0\n",
    "                        all_sample_num = len(valid)\n",
    "                        for idx, gt in enumerate(list(valid[output_col])):\n",
    "                            pred = tpredicted_products[idx]\n",
    "                            if task in ['iupac2smiles', 'formula2smiles']:\n",
    "                                try:\n",
    "                                    mol = Chem.MolFromSmiles(pred)\n",
    "                                    pred = Chem.MolToSmiles(mol)\n",
    "                                except Exception as e:\n",
    "                                    continue\n",
    "                                    \n",
    "                            if gt == pred:\n",
    "                                correct += 1\n",
    "                        acc = correct / all_sample_num\n",
    "                        acc_list.append(acc)\n",
    "\n",
    "                    # save to file\n",
    "                    performance_results.append([task, model, name, sample_num, sample_method, np.mean(acc_list)] + acc_list)\n",
    "                    print(performance_results)\n",
    "                    \n",
    "                    # performance save based on the task\n",
    "                    tem = pd.DataFrame(performance_results, columns=['task', 'model', 'name', 'sample_num', 'sample_method', 'avg_metric'] + ['metric_{}'.format(i) for i in range(5)])\n",
    "                    tem.to_csv(performance_file, index=False)\n",
    "                    \n",
    "                    details_df = pd.DataFrame(details_results, columns=[input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "                    details_df.to_csv(detail_predict_file, index=False)\n",
    "                    \n",
    "                    params_idx += 1\n",
    "                    print(params_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T13:42:13.021844Z",
     "start_time": "2023-04-26T13:42:12.726836Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_12729/4069735069.py:10: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n"
     ]
    }
   ],
   "source": [
    "all_performance = pd.DataFrame()\n",
    "for sample_method in ICL_Sample_methods:\n",
    "    for sample_num in ICL_Samples:\n",
    "        for name in NAME_Format:\n",
    "            for task in TASK_Types:\n",
    "                for model in Models:\n",
    "                    \n",
    "                    performance_file = detail_save_folder + 'valid_performance_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "                    tem = pd.read_csv(performance_file)\n",
    "                    all_performance = all_performance.append(tem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T13:50:34.621785Z",
     "start_time": "2023-04-26T13:50:34.617310Z"
    }
   },
   "outputs": [],
   "source": [
    "all_performance = all_performance.drop_duplicates(subset=['task', 'model', 'name', 'sample_num', 'sample_method'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T13:50:35.008879Z",
     "start_time": "2023-04-26T13:50:35.001918Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "iupac2smiles      12\n",
       "formula2smiles    12\n",
       "smiles2iupac      12\n",
       "smiles2formula    12\n",
       "Name: task, dtype: int64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_performance['task'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T13:52:53.281138Z",
     "start_time": "2023-04-26T13:52:53.272286Z"
    }
   },
   "outputs": [],
   "source": [
    "all_performance.to_csv(detail_save_folder + \"results_summary.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:33:57.659092Z",
     "start_time": "2023-05-18T03:33:57.656333Z"
    }
   },
   "outputs": [],
   "source": [
    "cols = ['metric_{}'.format(i) for i in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:33:59.732140Z",
     "start_time": "2023-05-18T03:33:59.648209Z"
    }
   },
   "outputs": [],
   "source": [
    "import statistics\n",
    "all_performance['std'] = all_performance[cols].apply(lambda row: statistics.stdev(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:34:41.690933Z",
     "start_time": "2023-05-18T03:34:41.681926Z"
    }
   },
   "outputs": [],
   "source": [
    "all_performance['metric'] = all_performance.apply(lambda row: \"{:.3f} $\\pm$ {:.3f}\".format(row['avg_metric'], row['std']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:53:35.380234Z",
     "start_time": "2023-05-18T03:53:35.360817Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>avg_metric</th>\n",
       "      <th>metric_0</th>\n",
       "      <th>metric_1</th>\n",
       "      <th>metric_2</th>\n",
       "      <th>metric_3</th>\n",
       "      <th>metric_4</th>\n",
       "      <th>std</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>smiles2formula</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.086</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.035777</td>\n",
       "      <td>0.086 $\\pm$ 0.036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>smiles2formula</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.005477</td>\n",
       "      <td>0.006 $\\pm$ 0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>smiles2formula</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.052</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.004472</td>\n",
       "      <td>0.052 $\\pm$ 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>smiles2formula</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.058</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.014832</td>\n",
       "      <td>0.058 $\\pm$ 0.015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>smiles2formula</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.070</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.031623</td>\n",
       "      <td>0.070 $\\pm$ 0.032</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             task       model    name  sample_num sample_method  avg_metric  \\\n",
       "0  smiles2formula        gpt4  SMILES          20  Scaffold_SIM       0.086   \n",
       "0  smiles2formula  davinci003  SMILES          20  Scaffold_SIM       0.006   \n",
       "0  smiles2formula       gpt35  SMILES          20  Scaffold_SIM       0.052   \n",
       "0  smiles2formula        gpt4  SMILES           5  Scaffold_SIM       0.058   \n",
       "0  smiles2formula        gpt4  SMILES          20     Fixed_ICL       0.070   \n",
       "\n",
       "   metric_0  metric_1  metric_2  metric_3  metric_4       std  \\\n",
       "0      0.06      0.12      0.09      0.04      0.12  0.035777   \n",
       "0      0.01      0.00      0.00      0.01      0.01  0.005477   \n",
       "0      0.05      0.06      0.05      0.05      0.05  0.004472   \n",
       "0      0.08      0.04      0.06      0.05      0.06  0.014832   \n",
       "0      0.12      0.05      0.08      0.06      0.04  0.031623   \n",
       "\n",
       "              metric  \n",
       "0  0.086 $\\pm$ 0.036  \n",
       "0  0.006 $\\pm$ 0.005  \n",
       "0  0.052 $\\pm$ 0.004  \n",
       "0  0.058 $\\pm$ 0.015  \n",
       "0  0.070 $\\pm$ 0.032  "
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_performance[all_performance['task'] == 'smiles2formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:50.707176Z",
     "start_time": "2023-04-30T22:08:50.703078Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test = test.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:50.907531Z",
     "start_time": "2023-04-30T22:08:50.904888Z"
    }
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:08:51.161026Z",
     "start_time": "2023-04-30T22:08:51.158415Z"
    }
   },
   "outputs": [],
   "source": [
    "# fix \n",
    "# real-time evaluation past 50 samples\n",
    "# rate limit , sleep 2min, rerun "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-07T18:01:35.221785Z",
     "start_time": "2023-05-07T18:01:35.201461Z"
    }
   },
   "outputs": [],
   "source": [
    "params = [\n",
    "    ('Scaffold_SIM', 20, 'SMILES', 'gpt4'),\n",
    "    ('Scaffold_SIM', 20, 'SMILES', 'davinci003'),\n",
    "    ('Scaffold_SIM', 20, 'SMILES', 'gpt35'),\n",
    "    \n",
    "    ('Scaffold_SIM', 5, 'SMILES', 'gpt4'),\n",
    "    ('Fixed_ICL', 20, 'SMILES', 'gpt4'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:14:00.737866Z",
     "start_time": "2023-04-30T22:14:00.734915Z"
    }
   },
   "outputs": [],
   "source": [
    "test = test.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T01:12:42.780032Z",
     "start_time": "2023-05-01T00:54:18.000389Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name Prediction/results_test/valid_iupac2smiles_gpt4_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_iupac2smiles_gpt4_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_iupac2smiles_davinci003_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_iupac2smiles_davinci003_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_iupac2smiles_gpt35_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_iupac2smiles_gpt35_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_iupac2smiles_gpt4_SMILES_5_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_iupac2smiles_gpt4_SMILES_5_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_iupac2smiles_gpt4_SMILES_20_Fixed_ICL.csv\n",
      "Name Prediction/results_test/valid_performance_iupac2smiles_gpt4_SMILES_20_Fixed_ICL.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_formula2smiles_gpt4_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_formula2smiles_gpt4_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_formula2smiles_davinci003_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_formula2smiles_davinci003_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_formula2smiles_gpt35_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_formula2smiles_gpt35_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_formula2smiles_gpt4_SMILES_5_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_formula2smiles_gpt4_SMILES_5_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_formula2smiles_gpt4_SMILES_20_Fixed_ICL.csv\n",
      "Name Prediction/results_test/valid_performance_formula2smiles_gpt4_SMILES_20_Fixed_ICL.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2iupac_gpt4_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2iupac_gpt4_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2iupac_davinci003_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2iupac_davinci003_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2iupac_gpt35_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2iupac_gpt35_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2iupac_gpt4_SMILES_5_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2iupac_gpt4_SMILES_5_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2iupac_gpt4_SMILES_20_Fixed_ICL.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2iupac_gpt4_SMILES_20_Fixed_ICL.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2formula_gpt4_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2formula_gpt4_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2formula_davinci003_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2formula_davinci003_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2formula_gpt35_SMILES_20_Scaffold_SIM.csv\n",
      "Name Prediction/results_test/valid_performance_smiles2formula_gpt35_SMILES_20_Scaffold_SIM.csv exits, continue to next params\n",
      "Name Prediction/results_test/valid_smiles2formula_gpt4_SMILES_5_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [04:01,  5.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [08:27,  5.07s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.057999999999999996\n",
      "[['smiles2formula', 'gpt4', 'SMILES', 5, 'Scaffold_SIM', 0.057999999999999996, 0.08, 0.04, 0.06, 0.05, 0.06]]\n",
      "Name Prediction/results_test/valid_smiles2formula_gpt4_SMILES_20_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [04:46,  6.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [09:57,  5.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.06999999999999999\n",
      "[['smiles2formula', 'gpt4', 'SMILES', 20, 'Fixed_ICL', 0.06999999999999999, 0.12, 0.05, 0.08, 0.06, 0.04]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "detail_save_folder = 'Name Prediction/results_test/'\n",
    "TASK_Types = ['iupac2smiles', 'formula2smiles', 'smiles2iupac', 'smiles2formula']\n",
    "\n",
    "for task in TASK_Types:\n",
    "    for sample_method, sample_num, name, model in params:\n",
    "        \n",
    "        details_results = []\n",
    "        performance_results = []\n",
    "\n",
    "        detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        log_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.log'.format(task, model, name, sample_num, sample_method)\n",
    "        performance_file = detail_save_folder + 'valid_performance_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        print(detail_predict_file)\n",
    "        \n",
    "        \n",
    "        if os.path.exists(performance_file):\n",
    "            print(\"{} exits, continue to next params\".format(performance_file))\n",
    "            continue\n",
    "        \n",
    "\n",
    "        # append new date\n",
    "        # Get the current date and time\n",
    "        now = datetime.datetime.now()\n",
    "        # Convert the date and time to a string\n",
    "        date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "        with open(log_file, \"a\") as file:\n",
    "            file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "        # init var\n",
    "        predicted_products = []\n",
    "        \n",
    "        # restore previous data and skip \n",
    "        previous_index = []\n",
    "        if os.path.exists(detail_predict_file):\n",
    "            previous = pd.read_csv(detail_predict_file)\n",
    "            previous_index = list(previous['index'])\n",
    "\n",
    "            # previous data => details_results \n",
    "            details_results = previous.values.tolist()\n",
    "            predicted_products = previous[['pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5']].values.tolist()\n",
    "\n",
    "        input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "        # query\n",
    "        for idx, row in tqdm(test.iterrows()):\n",
    "            reactant = row[input_col]\n",
    "            product = row[output_col]\n",
    "            index = row['index']\n",
    "        \n",
    "            if index in previous_index:\n",
    "                continue\n",
    "\n",
    "            # ICL examples\n",
    "            if sample_method == 'Fixed_ICL':\n",
    "                chunk = train.sample(sample_num, random_state=42)\n",
    "            elif sample_method == 'Scaffold_SIM':\n",
    "                if input_col == 'smiles':\n",
    "                    sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train['smiles']), n=sample_num)\n",
    "                else:\n",
    "                    # similarity by leven\n",
    "                    sim = top_n_similar_strings(reactant, list(train[input_col]), n=sample_num)\n",
    "                chunk = train[train[input_col].isin(sim)]\n",
    "\n",
    "            examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "            # build prompt and save\n",
    "            prompt = create_prompt(reactant, examples, task)\n",
    "\n",
    "            with open(log_file, \"a\") as file:\n",
    "                file.write(prompt + \"\\n\")\n",
    "                file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "            # different model \n",
    "            if model == 'davinci003':\n",
    "                predicted_product = generate_response_by_davinci(prompt)\n",
    "            elif model == 'gpt35':\n",
    "                predicted_product = generate_response_by_gpt35(prompt)\n",
    "            elif model == 'gpt4':\n",
    "                predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "            predicted_products.append(predicted_product)\n",
    "            details_results.append([index, reactant] + [product] + predicted_product)\n",
    "            \n",
    "            details_df = pd.DataFrame(details_results, columns=['index', input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "            details_df.to_csv(detail_predict_file, index=False)\n",
    "            \n",
    "            # evaluate\n",
    "            if len(predicted_products) % 50 == 0:         \n",
    "                # evaluate\n",
    "                acc_list = []\n",
    "                for repeat in range(5):\n",
    "                    tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "                    correct = 0\n",
    "                    all_sample_num = len(predicted_products)\n",
    "                    for idx, gt in enumerate(list(test.head(all_sample_num)[output_col])):\n",
    "                        pred = tpredicted_products[idx]\n",
    "                        if task in ['iupac2smiles', 'formula2smiles']:\n",
    "                            try:\n",
    "                                mol = Chem.MolFromSmiles(pred)\n",
    "                                pred = Chem.MolToSmiles(mol)\n",
    "                            except Exception as e:\n",
    "                                continue\n",
    "\n",
    "                        if gt == pred:\n",
    "                            correct += 1\n",
    "                    acc = correct / all_sample_num\n",
    "                    acc_list.append(acc)\n",
    "                print(np.mean(acc_list))\n",
    "            \n",
    "        # evaluate\n",
    "        acc_list = []\n",
    "        for repeat in range(5):\n",
    "            tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "            correct = 0\n",
    "            all_sample_num = len(test)\n",
    "            for idx, gt in enumerate(list(test[output_col])):\n",
    "                pred = tpredicted_products[idx]\n",
    "                if task in ['iupac2smiles', 'formula2smiles']:\n",
    "                    try:\n",
    "                        mol = Chem.MolFromSmiles(pred)\n",
    "                        pred = Chem.MolToSmiles(mol)\n",
    "                    except Exception as e:\n",
    "                        continue\n",
    "\n",
    "                if gt == pred:\n",
    "                    correct += 1\n",
    "            acc = correct / all_sample_num\n",
    "            acc_list.append(acc)\n",
    "\n",
    "        # save to file\n",
    "        performance_results.append([task, model, name, sample_num, sample_method, np.mean(acc_list)] + acc_list)\n",
    "        print(performance_results)\n",
    "\n",
    "        # performance save based on the task\n",
    "        tem = pd.DataFrame(performance_results, columns=['task', 'model', 'name', 'sample_num', 'sample_method', 'avg_metric'] + ['metric_{}'.format(i) for i in range(5)])\n",
    "        tem.to_csv(performance_file, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-07T18:01:44.089922Z",
     "start_time": "2023-05-07T18:01:44.080668Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "detail_save_folder = 'Name Prediction/results_test/'\n",
    "TASK_Types = ['iupac2smiles', 'formula2smiles', 'smiles2iupac', 'smiles2formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-07T18:01:44.682912Z",
     "start_time": "2023-05-07T18:01:44.458119Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10384/3496328388.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  all_performance = all_performance.append(tem)\n"
     ]
    }
   ],
   "source": [
    "all_performance = pd.DataFrame()\n",
    "for task in TASK_Types:\n",
    "    for sample_method, sample_num, name, model in params: \n",
    "        performance_file = detail_save_folder + 'valid_performance_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        tem = pd.read_csv(performance_file)\n",
    "        all_performance = all_performance.append(tem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:32:39.598775Z",
     "start_time": "2023-05-18T03:32:39.562542Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>avg_metric</th>\n",
       "      <th>metric_0</th>\n",
       "      <th>metric_1</th>\n",
       "      <th>metric_2</th>\n",
       "      <th>metric_3</th>\n",
       "      <th>metric_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iupac2smiles</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iupac2smiles</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iupac2smiles</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iupac2smiles</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iupac2smiles</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>20</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           task       model    name  sample_num sample_method  avg_metric  \\\n",
       "0  iupac2smiles        gpt4  SMILES          20  Scaffold_SIM       0.012   \n",
       "0  iupac2smiles  davinci003  SMILES          20  Scaffold_SIM       0.000   \n",
       "0  iupac2smiles       gpt35  SMILES          20  Scaffold_SIM       0.010   \n",
       "0  iupac2smiles        gpt4  SMILES           5  Scaffold_SIM       0.014   \n",
       "0  iupac2smiles        gpt4  SMILES          20     Fixed_ICL       0.010   \n",
       "\n",
       "   metric_0  metric_1  metric_2  metric_3  metric_4  \n",
       "0      0.01      0.01      0.01      0.02      0.01  \n",
       "0      0.00      0.00      0.00      0.00      0.00  \n",
       "0      0.01      0.01      0.01      0.01      0.01  \n",
       "0      0.01      0.02      0.02      0.00      0.02  \n",
       "0      0.00      0.01      0.01      0.01      0.02  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_performance[all_performance['task'] == 'iupac2smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:34:11.609223Z",
     "start_time": "2023-04-30T22:34:11.594849Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_similar_strings_sim(query, candidates, n=5):\n",
    "    # Calculate the Levenshtein distance between the query and each candidate\n",
    "    distances = [(c, similarity_ratio(query, c)) for c in candidates]\n",
    "    \n",
    "    # Sort the candidates by their Levenshtein distance to the query\n",
    "    sorted_distances = sorted(distances, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    # Get the top n candidates with the smallest Levenshtein distance\n",
    "    top_candidates = [d[0] for d in sorted_distances[:n]]\n",
    "    \n",
    "    # Return the top n candidates\n",
    "    return top_candidates, [d[1] for d in sorted_distances[:n]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:37:11.602626Z",
     "start_time": "2023-04-30T22:37:11.596474Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C28H35ClN6O5'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.iloc[0]['formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:37:14.639186Z",
     "start_time": "2023-04-30T22:37:14.612989Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "a, b = top_n_similar_strings_sim(test.iloc[0]['formula'], list(train['formula']), n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T22:37:17.572010Z",
     "start_time": "2023-04-30T22:37:17.558850Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CID</th>\n",
       "      <th>smiles</th>\n",
       "      <th>iupac</th>\n",
       "      <th>formula</th>\n",
       "      <th>mol_length</th>\n",
       "      <th>scaffold_fp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9086</th>\n",
       "      <td>18784961</td>\n",
       "      <td>CCCCC(CNC(Cc1ccc(O)cc1)C(=O)O)NC(=O)OC(C)(C)C</td>\n",
       "      <td>3-(4-hydroxyphenyl)-2-[2-[(2-methylpropan-2-yl...</td>\n",
       "      <td>C20H32N2O5</td>\n",
       "      <td>48</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2243</th>\n",
       "      <td>41120438</td>\n",
       "      <td>O=C(COC(=O)CCNC1=NS(=O)(=O)c2ccccc21)Nc1cccc(C...</td>\n",
       "      <td>[2-(3-chloroanilino)-2-oxoethyl] 3-[(1,1-dioxo...</td>\n",
       "      <td>C18H16ClN3O5S</td>\n",
       "      <td>58</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2523</th>\n",
       "      <td>73750156</td>\n",
       "      <td>Cc1ccc(C=NNc2nnc(C)n2N)o1.Cl</td>\n",
       "      <td>5-methyl-3-N-[(5-methylfuran-2-yl)methylidenea...</td>\n",
       "      <td>C9H13ClN6O</td>\n",
       "      <td>32</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3823</th>\n",
       "      <td>35517793</td>\n",
       "      <td>Cc1ccc(OCC(=O)OCC(=O)NCCN2CC(C)OC(C)C2)cc1C</td>\n",
       "      <td>[2-[2-[(2S,6S)-2,6-dimethylmorpholin-4-yl]ethy...</td>\n",
       "      <td>C20H30N2O5</td>\n",
       "      <td>48</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3959</th>\n",
       "      <td>3778798</td>\n",
       "      <td>CC(C)Oc1cc(NC(=O)c2sccc2Oc2ccc([N+](=O)[O-])cc...</td>\n",
       "      <td>N-(2,4-dichloro-5-propan-2-yloxyphenyl)-3-(2-f...</td>\n",
       "      <td>C20H15Cl2FN2O5S</td>\n",
       "      <td>73</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>135797389</td>\n",
       "      <td>CCCCCCCCCCCCCCCC(=NCc1cccc(OC)c1)C1=C(O)C(CO)O...</td>\n",
       "      <td>3-hydroxy-2-(hydroxymethyl)-4-[N-[(3-methoxyph...</td>\n",
       "      <td>C29H45NO5</td>\n",
       "      <td>53</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>108532930</td>\n",
       "      <td>Cc1ccc(C(=O)CCC(=O)N2CCN(C(=O)CCNC(=O)OC(C)(C)...</td>\n",
       "      <td>tert-butyl N-[3-[4-[4-(4-methylphenyl)-4-oxobu...</td>\n",
       "      <td>C23H33N3O5</td>\n",
       "      <td>58</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4704</th>\n",
       "      <td>38308997</td>\n",
       "      <td>CCCn1c(=O)[nH]c(=O)c2c(C(=O)OC(C)C(=O)Nc3c(CC)...</td>\n",
       "      <td>[(2S)-1-(2,6-diethylanilino)-1-oxopropan-2-yl]...</td>\n",
       "      <td>C27H34N4O5</td>\n",
       "      <td>70</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5794</th>\n",
       "      <td>76749614</td>\n",
       "      <td>Cc1cc(C)n(C(=O)N(C)CCCc2[nH]c(C=C3C(=O)Nc4ccc(...</td>\n",
       "      <td>N-[3-[5-[[5-[(3-chlorophenyl)-methylsulfamoyl]...</td>\n",
       "      <td>C37H43ClN8O5S</td>\n",
       "      <td>109</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9150</th>\n",
       "      <td>11721092</td>\n",
       "      <td>C=CCOC(=O)C=C[N+](CC)(CC)CC.O=[N+]([O-])c1ccc(...</td>\n",
       "      <td>(2-chloro-4-nitrophenyl)-[(2-chloro-4-nitrophe...</td>\n",
       "      <td>C24H28Cl2N6O6</td>\n",
       "      <td>93</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7602</th>\n",
       "      <td>88392014</td>\n",
       "      <td>CCCCCCCCC1(c2ccc(CCC=CCl)c(F)c2F)CCCCC1</td>\n",
       "      <td>1-[(E)-4-chlorobut-3-enyl]-2,3-difluoro-4-(1-o...</td>\n",
       "      <td>C24H35ClF2</td>\n",
       "      <td>44</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559</th>\n",
       "      <td>36927420</td>\n",
       "      <td>Cc1cccc(C(=O)NCC(=O)Nc2cc(S(=O)(=O)Nc3cccc(Cl)...</td>\n",
       "      <td>N-[2-[5-[(3-chlorophenyl)sulfamoyl]-2-hydroxya...</td>\n",
       "      <td>C22H20ClN3O5S</td>\n",
       "      <td>68</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1541</th>\n",
       "      <td>150034501</td>\n",
       "      <td>[N-]=[N+]=NN1CCC2(CC1)OCc1cc(Cl)ccc12</td>\n",
       "      <td>1'-azido-6-chlorospiro[1H-2-benzofuran-3,4'-pi...</td>\n",
       "      <td>C12H13ClN4O</td>\n",
       "      <td>42</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5646</th>\n",
       "      <td>23627869</td>\n",
       "      <td>CC(=O)Cc1ccc(OCCCCOc2ccc3c4ccccc4n(CCC(=O)O)c3...</td>\n",
       "      <td>3-[2-[4-[4-(2-oxopropyl)phenoxy]butoxy]carbazo...</td>\n",
       "      <td>C28H29NO5</td>\n",
       "      <td>61</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3702</th>\n",
       "      <td>74142094</td>\n",
       "      <td>CCC12C=CCCCCOC(=O)C1C1C(=O)N(C(C)CO)C3C(=O)N(c...</td>\n",
       "      <td>18-(2-chlorophenyl)-3-ethyl-15-(1-hydroxypropa...</td>\n",
       "      <td>C28H33ClN2O6</td>\n",
       "      <td>68</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3378</th>\n",
       "      <td>20129220</td>\n",
       "      <td>COC(=O)CN1Cc2cc(OCC(=O)N(C(C)O)C3CCCCC3)ccc2N=C1N</td>\n",
       "      <td>methyl 2-[2-amino-6-[2-[cyclohexyl(1-hydroxyet...</td>\n",
       "      <td>C21H30N4O5</td>\n",
       "      <td>56</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>99024545</td>\n",
       "      <td>COc1ccc(OC)c(-c2cc(C(=O)N3CCCC3Cc3ccccc3Cl)no2)c1</td>\n",
       "      <td>[(2S)-2-[(2-chlorophenyl)methyl]pyrrolidin-1-y...</td>\n",
       "      <td>C23H23ClN2O4</td>\n",
       "      <td>56</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9169</th>\n",
       "      <td>44248662</td>\n",
       "      <td>O=C(c1cc(Cl)c(O)cc1O)N1CCCC1c1ccc(CN2CCOCC2)cc1</td>\n",
       "      <td>(5-chloro-2,4-dihydroxyphenyl)-[2-[4-(morpholi...</td>\n",
       "      <td>C22H25ClN2O4</td>\n",
       "      <td>57</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8137</th>\n",
       "      <td>70010117</td>\n",
       "      <td>CCOC(=O)c1cnc2c(C(=O)OC)cc(C#N)cc2c1NCc1ccc(OC...</td>\n",
       "      <td>3-O-ethyl 8-O-methyl 4-[(3-chloro-4-methoxyphe...</td>\n",
       "      <td>C23H20ClN3O5</td>\n",
       "      <td>62</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4384</th>\n",
       "      <td>58638459</td>\n",
       "      <td>CC#CCn1c(N2CCCCC2)nc2nc(N(C)CCc3ccccc3)n(C)c(=...</td>\n",
       "      <td>7-but-2-ynyl-1-methyl-2-[methyl(2-phenylethyl)...</td>\n",
       "      <td>C24H30N6O</td>\n",
       "      <td>55</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            CID                                             smiles  \\\n",
       "9086   18784961      CCCCC(CNC(Cc1ccc(O)cc1)C(=O)O)NC(=O)OC(C)(C)C   \n",
       "2243   41120438  O=C(COC(=O)CCNC1=NS(=O)(=O)c2ccccc21)Nc1cccc(C...   \n",
       "2523   73750156                       Cc1ccc(C=NNc2nnc(C)n2N)o1.Cl   \n",
       "3823   35517793        Cc1ccc(OCC(=O)OCC(=O)NCCN2CC(C)OC(C)C2)cc1C   \n",
       "3959    3778798  CC(C)Oc1cc(NC(=O)c2sccc2Oc2ccc([N+](=O)[O-])cc...   \n",
       "318   135797389  CCCCCCCCCCCCCCCC(=NCc1cccc(OC)c1)C1=C(O)C(CO)O...   \n",
       "587   108532930  Cc1ccc(C(=O)CCC(=O)N2CCN(C(=O)CCNC(=O)OC(C)(C)...   \n",
       "4704   38308997  CCCn1c(=O)[nH]c(=O)c2c(C(=O)OC(C)C(=O)Nc3c(CC)...   \n",
       "5794   76749614  Cc1cc(C)n(C(=O)N(C)CCCc2[nH]c(C=C3C(=O)Nc4ccc(...   \n",
       "9150   11721092  C=CCOC(=O)C=C[N+](CC)(CC)CC.O=[N+]([O-])c1ccc(...   \n",
       "7602   88392014            CCCCCCCCC1(c2ccc(CCC=CCl)c(F)c2F)CCCCC1   \n",
       "1559   36927420  Cc1cccc(C(=O)NCC(=O)Nc2cc(S(=O)(=O)Nc3cccc(Cl)...   \n",
       "1541  150034501              [N-]=[N+]=NN1CCC2(CC1)OCc1cc(Cl)ccc12   \n",
       "5646   23627869  CC(=O)Cc1ccc(OCCCCOc2ccc3c4ccccc4n(CCC(=O)O)c3...   \n",
       "3702   74142094  CCC12C=CCCCCOC(=O)C1C1C(=O)N(C(C)CO)C3C(=O)N(c...   \n",
       "3378   20129220  COC(=O)CN1Cc2cc(OCC(=O)N(C(C)O)C3CCCCC3)ccc2N=C1N   \n",
       "636    99024545  COc1ccc(OC)c(-c2cc(C(=O)N3CCCC3Cc3ccccc3Cl)no2)c1   \n",
       "9169   44248662    O=C(c1cc(Cl)c(O)cc1O)N1CCCC1c1ccc(CN2CCOCC2)cc1   \n",
       "8137   70010117  CCOC(=O)c1cnc2c(C(=O)OC)cc(C#N)cc2c1NCc1ccc(OC...   \n",
       "4384   58638459  CC#CCn1c(N2CCCCC2)nc2nc(N(C)CCc3ccccc3)n(C)c(=...   \n",
       "\n",
       "                                                  iupac          formula  \\\n",
       "9086  3-(4-hydroxyphenyl)-2-[2-[(2-methylpropan-2-yl...       C20H32N2O5   \n",
       "2243  [2-(3-chloroanilino)-2-oxoethyl] 3-[(1,1-dioxo...    C18H16ClN3O5S   \n",
       "2523  5-methyl-3-N-[(5-methylfuran-2-yl)methylidenea...       C9H13ClN6O   \n",
       "3823  [2-[2-[(2S,6S)-2,6-dimethylmorpholin-4-yl]ethy...       C20H30N2O5   \n",
       "3959  N-(2,4-dichloro-5-propan-2-yloxyphenyl)-3-(2-f...  C20H15Cl2FN2O5S   \n",
       "318   3-hydroxy-2-(hydroxymethyl)-4-[N-[(3-methoxyph...        C29H45NO5   \n",
       "587   tert-butyl N-[3-[4-[4-(4-methylphenyl)-4-oxobu...       C23H33N3O5   \n",
       "4704  [(2S)-1-(2,6-diethylanilino)-1-oxopropan-2-yl]...       C27H34N4O5   \n",
       "5794  N-[3-[5-[[5-[(3-chlorophenyl)-methylsulfamoyl]...    C37H43ClN8O5S   \n",
       "9150  (2-chloro-4-nitrophenyl)-[(2-chloro-4-nitrophe...    C24H28Cl2N6O6   \n",
       "7602  1-[(E)-4-chlorobut-3-enyl]-2,3-difluoro-4-(1-o...       C24H35ClF2   \n",
       "1559  N-[2-[5-[(3-chlorophenyl)sulfamoyl]-2-hydroxya...    C22H20ClN3O5S   \n",
       "1541  1'-azido-6-chlorospiro[1H-2-benzofuran-3,4'-pi...      C12H13ClN4O   \n",
       "5646  3-[2-[4-[4-(2-oxopropyl)phenoxy]butoxy]carbazo...        C28H29NO5   \n",
       "3702  18-(2-chlorophenyl)-3-ethyl-15-(1-hydroxypropa...     C28H33ClN2O6   \n",
       "3378  methyl 2-[2-amino-6-[2-[cyclohexyl(1-hydroxyet...       C21H30N4O5   \n",
       "636   [(2S)-2-[(2-chlorophenyl)methyl]pyrrolidin-1-y...     C23H23ClN2O4   \n",
       "9169  (5-chloro-2,4-dihydroxyphenyl)-[2-[4-(morpholi...     C22H25ClN2O4   \n",
       "8137  3-O-ethyl 8-O-methyl 4-[(3-chloro-4-methoxyphe...     C23H20ClN3O5   \n",
       "4384  7-but-2-ynyl-1-methyl-2-[methyl(2-phenylethyl)...        C24H30N6O   \n",
       "\n",
       "      mol_length                                        scaffold_fp  \n",
       "9086          48  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "2243          58  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "2523          32  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "3823          48  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "3959          73  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "318           53  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "587           58  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "4704          70  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "5794         109  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "9150          93  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "7602          44  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "1559          68  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "1541          42  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "5646          61  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "3702          68  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "3378          56  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "636           56  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "9169          57  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "8137          62  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "4384          55  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[train['formula'].isin(a)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T01:40:56.676998Z",
     "start_time": "2023-05-01T01:40:56.630645Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_scaffold_similar_molecules_sim(target_smiles, molecule_scaffold_list, molecule_smiles_list, n=5):\n",
    "    target_mol = Chem.MolFromSmiles(target_smiles)\n",
    "    target_scaffold = MurckoScaffold.GetScaffoldForMol(target_mol)\n",
    "    target_fp = rdMolDescriptors.GetMorganFingerprint(target_scaffold, 2)\n",
    "\n",
    "    similarities = []\n",
    "\n",
    "    for idx, scaffold_fp in enumerate(molecule_scaffold_list):\n",
    "        try:\n",
    "            tanimoto_similarity = DataStructs.TanimotoSimilarity(target_fp, scaffold_fp)\n",
    "            similarities.append((idx, tanimoto_similarity))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue\n",
    "\n",
    "    similarities.sort(key=lambda x: x[1], reverse=True)\n",
    "    top_5_similar_molecules = similarities[:n]\n",
    "\n",
    "    return [molecule_smiles_list[i[0]] for i in top_5_similar_molecules], [i[1] for i in top_5_similar_molecules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T01:43:34.460500Z",
     "start_time": "2023-05-01T01:43:34.455377Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('CC(C)(C)OC(=O)NC1CCN(C(=O)CN2CCOCC2C(=O)Nc2cc(Cl)cc3c2[nH]c2cnccc23)CC1',)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.iloc[0]['smiles'],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T01:41:35.963207Z",
     "start_time": "2023-05-01T01:41:35.949658Z"
    },
    "run_control": {
     "marked": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "a, b = top_n_scaffold_similar_molecules_sim(test.iloc[0]['smiles'], list(train['scaffold_fp']), list(train['smiles']), n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T01:41:36.435836Z",
     "start_time": "2023-05-01T01:41:36.341329Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CID</th>\n",
       "      <th>smiles</th>\n",
       "      <th>iupac</th>\n",
       "      <th>formula</th>\n",
       "      <th>mol_length</th>\n",
       "      <th>scaffold_fp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>110266602</td>\n",
       "      <td>CN(C)c1ncc(-c2ccccc2F)c(C2CCCN(C(=O)CCN3CCOCC3...</td>\n",
       "      <td>1-[3-[2-(dimethylamino)-5-(2-fluorophenyl)pyri...</td>\n",
       "      <td>C24H32FN5O2</td>\n",
       "      <td>57</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1170</th>\n",
       "      <td>75092174</td>\n",
       "      <td>CC(CCN1CCCCC1)C(=O)NC1NNC(c2cnc3ccccc3c2)C1F</td>\n",
       "      <td>N-(4-fluoro-5-quinolin-3-ylpyrazolidin-3-yl)-2...</td>\n",
       "      <td>C22H30FN5O</td>\n",
       "      <td>51</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3067</th>\n",
       "      <td>27495086</td>\n",
       "      <td>O=C(Nc1cccnc1)c1cn(CC2CCCN(C(=O)C3CC3)C2)c2ccc...</td>\n",
       "      <td>1-[[(3R)-1-(cyclopropanecarbonyl)piperidin-3-y...</td>\n",
       "      <td>C24H26N4O2</td>\n",
       "      <td>59</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5397</th>\n",
       "      <td>99071111</td>\n",
       "      <td>COC(=O)N1CCCN(C(=O)C(CCSC)NC(=O)c2ccccc2Cl)CC1</td>\n",
       "      <td>methyl 4-[(2R)-2-[(2-chlorobenzoyl)amino]-4-me...</td>\n",
       "      <td>C19H26ClN3O4S</td>\n",
       "      <td>49</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6536</th>\n",
       "      <td>32535358</td>\n",
       "      <td>Cc1ccc(OCC(=O)N2CCN(C(=O)c3ccoc3)CC2)c2c1C(C)C...</td>\n",
       "      <td>(3R)-7-[2-[4-(furan-3-carbonyl)piperazin-1-yl]...</td>\n",
       "      <td>C22H24N2O5</td>\n",
       "      <td>57</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951</th>\n",
       "      <td>77903191</td>\n",
       "      <td>CN(C(=O)C1CCCN1C(=O)C(N)Cc1ccc(O)cc1)C(Cc1cccc...</td>\n",
       "      <td>1-[2-[[1-[2-amino-3-(4-hydroxyphenyl)propanoyl...</td>\n",
       "      <td>C29H36N4O6</td>\n",
       "      <td>73</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>26670304</td>\n",
       "      <td>CC1c2ccsc2CCN1CC(=O)Nc1cc(S(=O)(=O)N2CCCCC2)cc...</td>\n",
       "      <td>N-(2-chloro-5-piperidin-1-ylsulfonylphenyl)-2-...</td>\n",
       "      <td>C21H26ClN3O3S2</td>\n",
       "      <td>59</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4404</th>\n",
       "      <td>100652034</td>\n",
       "      <td>Cc1cc(Br)cc(C(=O)N2CCCC(O)(C(N)=O)C2)c1</td>\n",
       "      <td>(3R)-1-(3-bromo-5-methylbenzoyl)-3-hydroxypipe...</td>\n",
       "      <td>C14H17BrN2O3</td>\n",
       "      <td>42</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>32181817</td>\n",
       "      <td>CCS(=O)(=O)N1CCC(NC(=O)c2cccc3c4c([nH]c23)CCCC...</td>\n",
       "      <td>N-(1-ethylsulfonylpiperidin-4-yl)-5,6,7,8,9,10...</td>\n",
       "      <td>C21H29N3O3S</td>\n",
       "      <td>51</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680</th>\n",
       "      <td>159041195</td>\n",
       "      <td>CC(=O)N(C)C1C(C)CN(c2ccncc2Cc2ncc3ccc(-c4c(F)c...</td>\n",
       "      <td>N-[(3R,4S,5S)-3-amino-1-[3-[[2-[2,6-difluoro-4...</td>\n",
       "      <td>C30H35F2N7O2</td>\n",
       "      <td>82</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6508</th>\n",
       "      <td>19574181</td>\n",
       "      <td>CC(C)(C(=O)N1CCN(Cc2cccnc2)CC1)n1cc(Br)cn1</td>\n",
       "      <td>2-(4-bromopyrazol-1-yl)-2-methyl-1-[4-(pyridin...</td>\n",
       "      <td>C17H22BrN5O</td>\n",
       "      <td>47</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>109095944</td>\n",
       "      <td>COCCCNC(=O)c1cccc(C(=O)Nc2ccccc2N2CCOCC2)n1</td>\n",
       "      <td>2-N-(3-methoxypropyl)-6-N-(2-morpholin-4-ylphe...</td>\n",
       "      <td>C21H26N4O4</td>\n",
       "      <td>49</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7065</th>\n",
       "      <td>55776537</td>\n",
       "      <td>Cc1cccc(NC(=O)CN2C(=O)c3cc(Cl)c(Cl)cc3C2=O)c1C...</td>\n",
       "      <td>2-(5,6-dichloro-1,3-dioxoisoindol-2-yl)-N-[3-m...</td>\n",
       "      <td>C22H19Cl2N3O4</td>\n",
       "      <td>65</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8116</th>\n",
       "      <td>126788735</td>\n",
       "      <td>CC(c1ccsc1)N1CCN(C(=O)Nc2cccc3ccn(C)c23)CC1</td>\n",
       "      <td>N-(1-methylindol-7-yl)-4-(1-thiophen-3-ylethyl...</td>\n",
       "      <td>C20H24N4OS</td>\n",
       "      <td>49</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9169</th>\n",
       "      <td>44248662</td>\n",
       "      <td>O=C(c1cc(Cl)c(O)cc1O)N1CCCC1c1ccc(CN2CCOCC2)cc1</td>\n",
       "      <td>(5-chloro-2,4-dihydroxyphenyl)-[2-[4-(morpholi...</td>\n",
       "      <td>C22H25ClN2O4</td>\n",
       "      <td>57</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8045</th>\n",
       "      <td>39772913</td>\n",
       "      <td>COc1ccc(F)cc1CN1CCN(c2nc(CN3CCCCC3)nc3sc(C)c(C...</td>\n",
       "      <td>4-[4-[(5-fluoro-2-methoxyphenyl)methyl]piperaz...</td>\n",
       "      <td>C26H34FN5OS</td>\n",
       "      <td>63</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>124726720</td>\n",
       "      <td>O=C(NCC(c1c(F)cccc1Cl)N1CCCC1)C1COCCN1CC(F)(F)F</td>\n",
       "      <td>(3S)-N-[(2S)-2-(2-chloro-6-fluorophenyl)-2-pyr...</td>\n",
       "      <td>C19H24ClF4N3O2</td>\n",
       "      <td>52</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4533</th>\n",
       "      <td>156811924</td>\n",
       "      <td>N#CCC1CN(c2cc(C(=O)Nc3cc(O)cc4ccccc34)nc(NCCN3...</td>\n",
       "      <td>6-[(3S)-3-(cyanomethyl)piperazin-1-yl]-N-(3-hy...</td>\n",
       "      <td>C27H32N8O3</td>\n",
       "      <td>70</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6324</th>\n",
       "      <td>20959955</td>\n",
       "      <td>COc1cc(CNC(=O)c2cnc(N3CCCCC3)nc2C)cc(OC)c1OC</td>\n",
       "      <td>4-methyl-2-piperidin-1-yl-N-[(3,4,5-trimethoxy...</td>\n",
       "      <td>C21H28N4O4</td>\n",
       "      <td>52</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6713</th>\n",
       "      <td>42425046</td>\n",
       "      <td>COC(=O)C1CCN(C(=O)CSc2nnc(C[NH+]3CCCCC3)n2-c2c...</td>\n",
       "      <td>methyl 1-[2-[[4-phenyl-5-(piperidin-1-ium-1-yl...</td>\n",
       "      <td>C23H32N5O3S+</td>\n",
       "      <td>59</td>\n",
       "      <td>&lt;rdkit.DataStructs.cDataStructs.UIntSparseIntV...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            CID                                             smiles  \\\n",
       "980   110266602  CN(C)c1ncc(-c2ccccc2F)c(C2CCCN(C(=O)CCN3CCOCC3...   \n",
       "1170   75092174       CC(CCN1CCCCC1)C(=O)NC1NNC(c2cnc3ccccc3c2)C1F   \n",
       "3067   27495086  O=C(Nc1cccnc1)c1cn(CC2CCCN(C(=O)C3CC3)C2)c2ccc...   \n",
       "5397   99071111     COC(=O)N1CCCN(C(=O)C(CCSC)NC(=O)c2ccccc2Cl)CC1   \n",
       "6536   32535358  Cc1ccc(OCC(=O)N2CCN(C(=O)c3ccoc3)CC2)c2c1C(C)C...   \n",
       "1951   77903191  CN(C(=O)C1CCCN1C(=O)C(N)Cc1ccc(O)cc1)C(Cc1cccc...   \n",
       "1178   26670304  CC1c2ccsc2CCN1CC(=O)Nc1cc(S(=O)(=O)N2CCCCC2)cc...   \n",
       "4404  100652034            Cc1cc(Br)cc(C(=O)N2CCCC(O)(C(N)=O)C2)c1   \n",
       "288    32181817  CCS(=O)(=O)N1CCC(NC(=O)c2cccc3c4c([nH]c23)CCCC...   \n",
       "680   159041195  CC(=O)N(C)C1C(C)CN(c2ccncc2Cc2ncc3ccc(-c4c(F)c...   \n",
       "6508   19574181         CC(C)(C(=O)N1CCN(Cc2cccnc2)CC1)n1cc(Br)cn1   \n",
       "247   109095944        COCCCNC(=O)c1cccc(C(=O)Nc2ccccc2N2CCOCC2)n1   \n",
       "7065   55776537  Cc1cccc(NC(=O)CN2C(=O)c3cc(Cl)c(Cl)cc3C2=O)c1C...   \n",
       "8116  126788735        CC(c1ccsc1)N1CCN(C(=O)Nc2cccc3ccn(C)c23)CC1   \n",
       "9169   44248662    O=C(c1cc(Cl)c(O)cc1O)N1CCCC1c1ccc(CN2CCOCC2)cc1   \n",
       "8045   39772913  COc1ccc(F)cc1CN1CCN(c2nc(CN3CCCCC3)nc3sc(C)c(C...   \n",
       "520   124726720    O=C(NCC(c1c(F)cccc1Cl)N1CCCC1)C1COCCN1CC(F)(F)F   \n",
       "4533  156811924  N#CCC1CN(c2cc(C(=O)Nc3cc(O)cc4ccccc34)nc(NCCN3...   \n",
       "6324   20959955       COc1cc(CNC(=O)c2cnc(N3CCCCC3)nc2C)cc(OC)c1OC   \n",
       "6713   42425046  COC(=O)C1CCN(C(=O)CSc2nnc(C[NH+]3CCCCC3)n2-c2c...   \n",
       "\n",
       "                                                  iupac         formula  \\\n",
       "980   1-[3-[2-(dimethylamino)-5-(2-fluorophenyl)pyri...     C24H32FN5O2   \n",
       "1170  N-(4-fluoro-5-quinolin-3-ylpyrazolidin-3-yl)-2...      C22H30FN5O   \n",
       "3067  1-[[(3R)-1-(cyclopropanecarbonyl)piperidin-3-y...      C24H26N4O2   \n",
       "5397  methyl 4-[(2R)-2-[(2-chlorobenzoyl)amino]-4-me...   C19H26ClN3O4S   \n",
       "6536  (3R)-7-[2-[4-(furan-3-carbonyl)piperazin-1-yl]...      C22H24N2O5   \n",
       "1951  1-[2-[[1-[2-amino-3-(4-hydroxyphenyl)propanoyl...      C29H36N4O6   \n",
       "1178  N-(2-chloro-5-piperidin-1-ylsulfonylphenyl)-2-...  C21H26ClN3O3S2   \n",
       "4404  (3R)-1-(3-bromo-5-methylbenzoyl)-3-hydroxypipe...    C14H17BrN2O3   \n",
       "288   N-(1-ethylsulfonylpiperidin-4-yl)-5,6,7,8,9,10...     C21H29N3O3S   \n",
       "680   N-[(3R,4S,5S)-3-amino-1-[3-[[2-[2,6-difluoro-4...    C30H35F2N7O2   \n",
       "6508  2-(4-bromopyrazol-1-yl)-2-methyl-1-[4-(pyridin...     C17H22BrN5O   \n",
       "247   2-N-(3-methoxypropyl)-6-N-(2-morpholin-4-ylphe...      C21H26N4O4   \n",
       "7065  2-(5,6-dichloro-1,3-dioxoisoindol-2-yl)-N-[3-m...   C22H19Cl2N3O4   \n",
       "8116  N-(1-methylindol-7-yl)-4-(1-thiophen-3-ylethyl...      C20H24N4OS   \n",
       "9169  (5-chloro-2,4-dihydroxyphenyl)-[2-[4-(morpholi...    C22H25ClN2O4   \n",
       "8045  4-[4-[(5-fluoro-2-methoxyphenyl)methyl]piperaz...     C26H34FN5OS   \n",
       "520   (3S)-N-[(2S)-2-(2-chloro-6-fluorophenyl)-2-pyr...  C19H24ClF4N3O2   \n",
       "4533  6-[(3S)-3-(cyanomethyl)piperazin-1-yl]-N-(3-hy...      C27H32N8O3   \n",
       "6324  4-methyl-2-piperidin-1-yl-N-[(3,4,5-trimethoxy...      C21H28N4O4   \n",
       "6713  methyl 1-[2-[[4-phenyl-5-(piperidin-1-ium-1-yl...    C23H32N5O3S+   \n",
       "\n",
       "      mol_length                                        scaffold_fp  \n",
       "980           57  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "1170          51  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "3067          59  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "5397          49  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "6536          57  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "1951          73  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "1178          59  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "4404          42  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "288           51  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "680           82  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "6508          47  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "247           49  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "7065          65  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "8116          49  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "9169          57  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "8045          63  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "520           52  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "4533          70  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "6324          52  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  \n",
       "6713          59  <rdkit.DataStructs.cDataStructs.UIntSparseIntV...  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[train['smiles'].isin(a)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zero-shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T13:37:56.620456Z",
     "start_time": "2023-05-01T13:37:56.589228Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_zero_shot_prompt(input_text, task):\n",
    "    if task == 'smiles2iupac':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular SMILES: {}, predict the molecular IUPAC name using your experienced chemical molecular SMILES and IUPAC name knowledge. No explanations and other information. \\\n",
    "Only return the molecular IUPAC name.\".format(input_text)\n",
    "        \n",
    "    elif task == 'smiles2formula':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular SMILES: {}, predict the chemical molecular formula using your experienced chemical molecular SMILES and formula knowledge. No explanations and other information. \\\n",
    "Only return the molecular formula.\".format(input_text)\n",
    "        \n",
    "    elif task == 'iupac2smiles':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular IUPAC name: {}, predict the molecular SMILES using your experienced chemical molecular IUPAC name and SMILES knowledge. No explanations and other information. \\\n",
    "Only return the molecular SMILES.\".format(input_text)\n",
    "        \n",
    "    elif task == 'formula2smiles':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular formula: {}, predict the molecular SMILES using your experienced chemical molecular formula and SMILES knowledge. No explanations and other information. \\\n",
    "Only return the molecular SMILES.\".format(input_text)\n",
    "        \n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T13:37:57.174687Z",
     "start_time": "2023-05-01T13:37:57.168805Z"
    }
   },
   "outputs": [],
   "source": [
    "model = 'gpt4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T14:53:59.680800Z",
     "start_time": "2023-05-01T14:17:20.673041Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name Prediction/results_test/iupac2smiles_gpt4_zero_shot.csv\n",
      "Name Prediction/results_test/iupac2smiles_gpt4_zero_shot_performance.csv exits, continue to next params\n",
      "Name Prediction/results_test/formula2smiles_gpt4_zero_shot.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [04:28,  2.68s/it]\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 13 14 15 16 17 18 19 21 22\n",
      "[10:21:49] SMILES Parse Error: extra close parentheses while parsing: C1=C(NC(=O)NS(=O)(=O)C2=C(C=CC=C2)F)C(=O)C2=CC(NNC3=NC4=C(C=O)N3C3=CCCCC3)=C(CC3=CC(=CC=C3)F)C=C4)=CC=C12\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C1=C(NC(=O)NS(=O)(=O)C2=C(C=CC=C2)F)C(=O)C2=CC(NNC3=NC4=C(C=O)N3C3=CCCCC3)=C(CC3=CC(=CC=C3)F)C=C4)=CC=C12' for input: 'C1=C(NC(=O)NS(=O)(=O)C2=C(C=CC=C2)F)C(=O)C2=CC(NNC3=NC4=C(C=O)N3C3=CCCCC3)=C(CC3=CC(=CC=C3)F)C=C4)=CC=C12'\n",
      "[10:21:49] Explicit valence for atom # 24 S, 7, is greater than permitted\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C16H27F3N2O3\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C16H27F3N2O3' for input: 'C16H27F3N2O3'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C54H58FN3O10\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C54H58FN3O10' for input: 'C54H58FN3O10'\n",
      "[10:21:49] Explicit valence for atom # 25 N, 4, is greater than permitted\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 106 107 108 109 110\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C12H22Si\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C12H22Si' for input: 'C12H22Si'\n",
      "[10:21:49] SMILES Parse Error: extra open parentheses for input: 'C1CCCC1CCCC(N(C(C2CCN(CC2)C(C3=C(OC4=C(C=C5C6=CC(=C(O)C=C6C(C(C(=O)O5)=O)(C)Cl)C7=C3C8=CC(F)=C(OC3CCC3)C8=C7)=C4)C(O9C3C(C(=O)C(=O)N3C)O9)C)=O)C)O4C1C(C(=O)C(=O)N1C)O4'\n",
      "[10:21:49] Explicit valence for atom # 10 N, 4, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 3 4 5 6 7 8 9 11 15\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 6 7 8 9 10 12 13 14 15 16 17 18 19 34 35 36 37\n",
      "[10:21:49] Explicit valence for atom # 28 Cl, 2, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 10 11 12 14 15 16 17 18 19\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'BrC1=CC=C(S(N(CC=C2)=O)(=O)=O)C(Br)=C1C3=NCC=S3'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1cc(ccc1C)c2c[nH]c3nc(N)nc(o2)c34N(C)C'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C17H20O5\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C17H20O5' for input: 'C17H20O5'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] Explicit valence for atom # 7 C, 5, is greater than permitted\n",
      "[10:21:49] non-ring atom 11 marked aromatic\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc2cc3cc4cc5cc2c1-c1cc2cc(ccc(c34)c21)-c1cc2cc(ccc(c45)c21)'\n",
      "[10:21:49] SMILES Parse Error: extra close parentheses while parsing: CC1=C(C(=O)C2=CCCCC2=O)C(C=CC3=CCCC(C(=CC(=O)C4=CC=CC=C4)=CC(=O)N(=S)S)=CN3)=NC(=S)S)=C1\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'CC1=C(C(=O)C2=CCCCC2=O)C(C=CC3=CCCC(C(=CC(=O)C4=CC=CC=C4)=CC(=O)N(=S)S)=CN3)=NC(=S)S)=C1' for input: 'CC1=C(C(=O)C2=CCCCC2=O)C(C=CC3=CCCC(C(=CC(=O)C4=CC=CC=C4)=CC(=O)N(=S)S)=CN3)=NC(=S)S)=C1'\n",
      "[10:21:49] Explicit valence for atom # 18 Br, 2, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 7 8 9 10 11 13 15 17 18\n",
      "[10:21:49] Explicit valence for atom # 24 F, 2, is greater than permitted\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: [C]1([C@H](C[C@@H]([C@@](C1)(C(=O)O)O)O)O)[S](=O)(=O)C2=C(N=C(N=C2)N)N.K\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES '[C]1([C@H](C[C@@H]([C@@](C1)(C(=O)O)O)O)O)[S](=O)(=O)C2=C(N=C(N=C2)N)N.K' for input: '[C]1([C@H](C[C@@H]([C@@](C1)(C(=O)O)O)O)O)[S](=O)(=O)C2=C(N=C(N=C2)N)N.K'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc(cc1)c2c3ccccc3nc4c5ccccc5nc2c6cc(ccc6C)S(=O)(=O)O'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C(CCCC)(CCCC)Si\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C(CCCC)(CCCC)Si' for input: 'C(CCCC)(CCCC)Si'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C13H20N4O\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C13H20N4O' for input: 'C13H20N4O'\n",
      "[10:21:49] Explicit valence for atom # 12 C, 5, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 10 11 12 13 14 15 16 17 18\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'CCN(CC)C(=O)C1=CC=C(C=C1)SCC2=C(N=C(N=C2Cl)C)C.OCC2=CC=CC=C2O1'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 2 4 14 15 17\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 23 24 25 26 27 28 29 30 31\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: The\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'The' for input: 'The'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc2c(c1)c1cc3c(cc2)n(cc4cc(no3)c2ccccc2)cc1c1ccccc1'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc2c(c1)c3cnc(nc3n2c4ccncc)c5cccc(c5)N'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 22 23 25\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: Co(C(C)(C)C)N(CC(=O)[O-])OCC([O-])=O.\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Co(C(C)(C)C)N(CC(=O)[O-])OCC([O-])=O.' for input: 'Co(C(C)(C)C)N(CC(=O)[O-])OCC([O-])=O.'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'Cc1cc2c(cc1Nc3ccnc(n3)-c4nc(-c5ccc(cc5)S(=O)(=O)N6CCN(CC6)C)n5ncnc45)C(Cl)(Cl)Cl'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 11 12 13 14 15\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C28H35ClN6O5\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C28H35ClN6O5' for input: 'C28H35ClN6O5'\n",
      "[10:21:49] Explicit valence for atom # 17 N, 4, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 22 23 25\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C1CCCCN1C(=O)C(C(CC(=O)[O-])N2CCCCC2)(C3=CC=C(C=C3)S(=O)(=O)[O-])/[NH+]=C(/C)K\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C1CCCCN1C(=O)C(C(CC(=O)[O-])N2CCCCC2)(C3=CC=C(C=C3)S(=O)(=O)[O-])/[NH+]=C(/C)K' for input: 'C1CCCCN1C(=O)C(C(CC(=O)[O-])N2CCCCC2)(C3=CC=C(C=C3)S(=O)(=O)[O-])/[NH+]=C(/C)K'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 2 3 4 5 24\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'O=C(NCCCCCC(=O)NCCCCCC(=O)NCCC1=CC=C(F)C=C1)C2=CC(=C[C@]=C(C54OC3O[C@H](C(=C[C@@]3([H])[C@@H](C5)C=C4)[C@]([H])56O)([H])C)[C@]([H])(C(=C2O[H])C)C([H])([H])[C@@H]6O)C([H])(C)C'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C12H22Si\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C12H22Si' for input: 'C12H22Si'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: O=C(Nc1cccc2ccccc12)C3=CC([N+](O-)=O)=CC=C3\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'O=C(Nc1cccc2ccccc12)C3=CC([N+](O-)=O)=CC=C3' for input: 'O=C(Nc1cccc2ccccc12)C3=CC([N+](O-)=O)=CC=C3'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: This\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'This' for input: 'This'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: Sorry,\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Sorry,' for input: 'Sorry,'\n",
      "[10:21:49] Explicit valence for atom # 28 Cl, 2, is greater than permitted\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: c1ccc(cc1)C(c2ccccc2)n3ccc4c(n3)cccc4>O\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'c1ccc(cc1)C(c2ccccc2)n3ccc4c(n3)cccc4>O' for input: 'c1ccc(cc1)C(c2ccccc2)n3ccc4c(n3)cccc4>O'\n",
      "[10:21:49] Explicit valence for atom # 1 O, 3, is greater than permitted\n",
      "[10:21:49] Explicit valence for atom # 4 C, 5, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 7 8 9 10 12 13 14\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: Cc1cc(N(C(=O)C)C(F)(F)F)c(C)n(C)c1-c1ccc(N2CCN([C@H](C(C(O)=O)(c2ambac2)=O)CC2)CC2)cc1\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Cc1cc(N(C(=O)C)C(F)(F)F)c(C)n(C)c1-c1ccc(N2CCN([C@H](C(C(O)=O)(c2ambac2)=O)CC2)CC2)cc1' for input: 'Cc1cc(N(C(=O)C)C(F)(F)F)c(C)n(C)c1-c1ccc(N2CCN([C@H](C(C(O)=O)(c2ambac2)=O)CC2)CC2)cc1'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: The\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'The' for input: 'The'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'C1(c2ccccc2)C(=O)N(CCN)C(=O)C(=C3C=NC45CCCCC4N3CCCC5)CCN'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C16H24N5O12P\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C16H24N5O12P' for input: 'C16H24N5O12P'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: c1cc(c(cc1c2cn(CCn2)c3c(cc(cc3)n4c(cn(CC)c5c4cc(cc5)))CC(N)=O)n6c(nc(n6)c7ccc(cc7))C)n8c(n(CCn9c8c(cc(c9)n(CCCCCC(c(%10)c(c(C)cn%10))C)c%11cn(CCn%11)c%12c(cc(cc%12)n%13c(cn(CC)c%14c%13cc(cc%14)))CC(N)=O)C)c%15nc(n(c%15)c%16ccc(cc%16))C)C)n(con%17)c%17cc(CC(N)=O)cc2%18c%14cc(cc%18)\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'c1cc(c(cc1c2cn(CCn2)c3c(cc(cc3)n4c(cn(CC)c5c4cc(cc5)))CC(N)=O)n6c(nc(n6)c7ccc(cc7))C)n8c(n(CCn9c8c(cc(c9)n(CCCCCC(c(%10)c(c(C)cn%10))C)c%11cn(CCn%11)c%12c(cc(cc%12)n%13c(cn(CC)c%14c%13cc(cc%14)))CC(N)=O)C)c%15nc(n(c%15)c%16ccc(cc%16))C)C)n(con%17)c%17cc(CC(N)=O)cc2%18c%14cc(cc%18)' for input: 'c1cc(c(cc1c2cn(CCn2)c3c(cc(cc3)n4c(cn(CC)c5c4cc(cc5)))CC(N)=O)n6c(nc(n6)c7ccc(cc7))C)n8c(n(CCn9c8c(cc(c9)n(CCCCCC(c(%10)c(c(C)cn%10))C)c%11cn(CCn%11)c%12c(cc(cc%12)n%13c(cn(CC)c%14c%13cc(cc%14)))CC(N)=O)C)c%15nc(n(c%15)c%16ccc(cc%16))C)C)n(con%17)c%17cc(CC(N)=O)cc2%18c%14cc(cc%18)'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: The\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'The' for input: 'The'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: The\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'The' for input: 'The'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 16 17 18 19 20 21 23 24 26 27 40\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'Cc1ccc(S(=O)(=O)c2ccc(NC(=O)c3cc4c(N(C(C4)C(=O)c4ccc(C)cc4)c4ccccc4)C(=O)c3ccc(N)c3)cc2)cc1'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 12 13 15\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C54H58FN3O10\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C54H58FN3O10' for input: 'C54H58FN3O10'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 19 20 21\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C(C(C)(C)C)C(C(C)(C)C)Si(C(C)(C)C)C(C(C)(C)C)C\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C(C(C)(C)C)C(C(C)(C)C)Si(C(C)(C)C)C(C(C)(C)C)C' for input: 'C(C(C)(C)C)C(C(C)(C)C)Si(C(C)(C)C)C(C(C)(C)C)C'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] SMILES Parse Error: extra close parentheses while parsing: c1ccc(cc1)c2cc(c(cc2)c3ccc4cc(c(cc4c3)c5ccc6cc(c(cc6c5)c7ccc8cc(c(cc8c7)c9cccn(c9)c%10ccc(cc%10)c%11cccnc%11)c%12cccn(c%12)c%13ccc(cc%13)c%14cccnc%14)c%15cccnc%15)c%16cccnc%16)C(=O)O)C(=O)O\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'c1ccc(cc1)c2cc(c(cc2)c3ccc4cc(c(cc4c3)c5ccc6cc(c(cc6c5)c7ccc8cc(c(cc8c7)c9cccn(c9)c%10ccc(cc%10)c%11cccnc%11)c%12cccn(c%12)c%13ccc(cc%13)c%14cccnc%14)c%15cccnc%15)c%16cccnc%16)C(=O)O)C(=O)O' for input: 'c1ccc(cc1)c2cc(c(cc2)c3ccc4cc(c(cc4c3)c5ccc6cc(c(cc6c5)c7ccc8cc(c(cc8c7)c9cccn(c9)c%10ccc(cc%10)c%11cccnc%11)c%12cccn(c%12)c%13ccc(cc%13)c%14cccnc%14)c%15cccnc%15)c%16cccnc%16)C(=O)O)C(=O)O'\n",
      "[10:21:49] Explicit valence for atom # 19 Cl, 2, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 7 8 9 10 11\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'C1CCCCC12CCCCC1CCC1C2NCCN1CCN1C2CCCCC12'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: There\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'There' for input: 'There'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc2c(c1)c1ccc3c(c1c2)n2c4ccccc4c5c6ccccc6n(c5c45cc[n+](C)c3c45)S3(=O)=O'\n",
      "[10:21:49] SMILES Parse Error: unclosed ring for input: 'c1ccc2c(c1)ccc1c2ccc2c1ccc1c2ccc2c1ccc1c2ccc2c1ccc1c2ccc2c1ccc1c2ccc2c1ccc1c2ccc2c1ccc-2c1ccc2c3ccc2c2ccc1cc1ccc2cc3'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 1 2 3 4 5 6 29 30 31 32 34\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 11 12 13 14 16\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 6 7 8 9 11 12 13 14 15\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 14 15 16 17 18 19 20 21 22 23 25\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 10 11 14 19 20 21 22\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: Oh,\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Oh,' for input: 'Oh,'\n",
      "[10:21:49] SMILES Parse Error: extra close parentheses while parsing: Cc1ccccc1c2[nH]c3c(c(=O)n(Cc4ccc(cc4)F)c5ccccc5)cn3F)c6ccc(cc2)n6\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Cc1ccccc1c2[nH]c3c(c(=O)n(Cc4ccc(cc4)F)c5ccccc5)cn3F)c6ccc(cc2)n6' for input: 'Cc1ccccc1c2[nH]c3c(c(=O)n(Cc4ccc(cc4)F)c5ccccc5)cn3F)c6ccc(cc2)n6'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C1(=C(C(=C(C(=C1[N+](=O)[O-])[N+](=O)[O-])[N+](=O)[O-])C#N)[O-].K[S](=O)(=O)O\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C1(=C(C(=C(C(=C1[N+](=O)[O-])[N+](=O)[O-])[N+](=O)[O-])C#N)[O-].K[S](=O)(=O)O' for input: 'C1(=C(C(=C(C(=C1[N+](=O)[O-])[N+](=O)[O-])[N+](=O)[O-])C#N)[O-].K[S](=O)(=O)O'\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 6 9 10 14 16\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: CC(C)C[C@H](C(=O)N[C@@H](CCCCN)C(=O)N[C@@H](Cc1cnc[nH]1)C(=O)N[C@@H](CC(C)C)C(=O)N[C@@H](Cc1ccccc1)C(=O)NCC(=O)N[C@@H](CCC(=O)N)C(=O)N[C@@H](Cc1c[nH]cn1)C(=O)N[C@@H](CCCNC(=N)N)C(=O)N[C@@H](Cc1ccccc1)C(=O)O)NC(=O)CNC(=O)[C@H](C(=O)N[C@@H](CD)C(=O)O)SCCSC\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'CC(C)C[C@H](C(=O)N[C@@H](CCCCN)C(=O)N[C@@H](Cc1cnc[nH]1)C(=O)N[C@@H](CC(C)C)C(=O)N[C@@H](Cc1ccccc1)C(=O)NCC(=O)N[C@@H](CCC(=O)N)C(=O)N[C@@H](Cc1c[nH]cn1)C(=O)N[C@@H](CCCNC(=N)N)C(=O)N[C@@H](Cc1ccccc1)C(=O)O)NC(=O)CNC(=O)[C@H](C(=O)N[C@@H](CD)C(=O)O)SCCSC' for input: 'CC(C)C[C@H](C(=O)N[C@@H](CCCCN)C(=O)N[C@@H](Cc1cnc[nH]1)C(=O)N[C@@H](CC(C)C)C(=O)N[C@@H](Cc1ccccc1)C(=O)NCC(=O)N[C@@H](CCC(=O)N)C(=O)N[C@@H](Cc1c[nH]cn1)C(=O)N[C@@H](CCCNC(=N)N)C(=O)N[C@@H](Cc1ccccc1)C(=O)O)NC(=O)CNC(=O)[C@H](C(=O)N[C@@H](CD)C(=O)O)SCCSC'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: CC(C)CCC(C)CCC(C)(C)Si\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'CC(C)CCC(C)CCC(C)(C)Si' for input: 'CC(C)CCC(C)CCC(C)(C)Si'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C13H20N4O\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C13H20N4O' for input: 'C13H20N4O'\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C1=CC=C(C=C1)C2=CC=C(C=C2)C3=CC(=O)C(=C(C3=O)OC)C4=CC=CC=C4Cl.\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C1=CC=C(C=C1)C2=CC=C(C=C2)C3=CC(=O)C(=C(C3=O)OC)C4=CC=CC=C4Cl.' for input: 'C1=CC=C(C=C1)C2=CC=C(C=C2)C3=CC(=O)C(=C(C3=O)OC)C4=CC=CC=C4Cl.'\n",
      "[10:21:49] SMILES Parse Error: extra close parentheses while parsing: CC(C(=O)N1CCCCC1C2=CC=CC(NS(=O)(=O)C(C)C)=N2)=C(C(F)(F)F)C=C3)=C3C=C(OC)C=C3\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'CC(C(=O)N1CCCCC1C2=CC=CC(NS(=O)(=O)C(C)C)=N2)=C(C(F)(F)F)C=C3)=C3C=C(OC)C=C3' for input: 'CC(C(=O)N1CCCCC1C2=CC=CC(NS(=O)(=O)C(C)C)=N2)=C(C(F)(F)F)C=C3)=C3C=C(OC)C=C3'\n",
      "[10:21:49] Explicit valence for atom # 7 O, 3, is greater than permitted\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 3 4 5 6 8 9 10\n",
      "[10:21:49] Can't kekulize mol.  Unkekulized atoms: 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: Can't\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'Can't' for input: 'Can't'\n",
      "[10:21:49] Explicit valence for atom # 21 N, 4, is greater than permitted\n",
      "[10:21:49] SMILES Parse Error: syntax error while parsing: C(C(CC(=O)O)N(CCN(CC(=O)O)C(C)C)c1ccccc1)C(=O)[O-].Co+[O-]\n",
      "[10:21:49] SMILES Parse Error: Failed parsing SMILES 'C(C(CC(=O)O)N(CCN(CC(=O)O)C(C)C)c1ccccc1)C(=O)[O-].Co+[O-]' for input: 'C(C(CC(=O)O)N(CCN(CC(=O)O)C(C)C)c1ccccc1)C(=O)[O-].Co+[O-]'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['formula2smiles', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]\n",
      "Name Prediction/results_test/smiles2iupac_gpt4_zero_shot.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [21:24, 12.85s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2iupac', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]\n",
      "Name Prediction/results_test/smiles2formula_gpt4_zero_shot.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [10:45,  6.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['smiles2formula', 0.048, 0.02, 0.05, 0.08, 0.05, 0.04]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "detail_save_folder = 'Name Prediction/results_test/'\n",
    "TASK_Types = ['iupac2smiles', 'formula2smiles', 'smiles2iupac', 'smiles2formula']\n",
    "\n",
    "for task in TASK_Types:\n",
    "\n",
    "    details_results = []\n",
    "    performance_results = []\n",
    "\n",
    "    detail_predict_file = detail_save_folder + '{}_{}_zero_shot.csv'.format(task, model)\n",
    "    log_file = detail_save_folder + '{}_{}_zero_shot.log'.format(task, model)\n",
    "    performance_file = detail_save_folder + '{}_{}_zero_shot_performance.csv'.format(task, model)\n",
    "    print(detail_predict_file)\n",
    "\n",
    "\n",
    "    if os.path.exists(performance_file):\n",
    "        print(\"{} exits, continue to next params\".format(performance_file))\n",
    "        continue\n",
    "\n",
    "\n",
    "    # append new date\n",
    "    # Get the current date and time\n",
    "    now = datetime.datetime.now()\n",
    "    # Convert the date and time to a string\n",
    "    date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    with open(log_file, \"a\") as file:\n",
    "        file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "    # init var\n",
    "    predicted_products = []\n",
    "\n",
    "    # restore previous data and skip \n",
    "    previous_index = []\n",
    "    if os.path.exists(detail_predict_file):\n",
    "        previous = pd.read_csv(detail_predict_file)\n",
    "        previous_index = list(previous['index'])\n",
    "\n",
    "        # previous data => details_results \n",
    "        details_results = previous.values.tolist()\n",
    "        predicted_products = previous[['pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5']].values.tolist()\n",
    "\n",
    "    input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "    # query\n",
    "    for idx, row in tqdm(test.iterrows()):\n",
    "        reactant = row[input_col]\n",
    "        product = row[output_col]\n",
    "        index = row['index']\n",
    "\n",
    "        if index in previous_index:\n",
    "            continue\n",
    "\n",
    "    \n",
    "        prompt = create_zero_shot_prompt(reactant, task)\n",
    "        with open(log_file, \"a\") as file:\n",
    "            file.write(prompt + \"\\n\")\n",
    "            file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "        # different model \n",
    "        if model == 'davinci003':\n",
    "            predicted_product = generate_response_by_davinci(prompt)\n",
    "        elif model == 'gpt35':\n",
    "            predicted_product = generate_response_by_gpt35(prompt)\n",
    "        elif model == 'gpt4':\n",
    "            predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "        predicted_products.append(predicted_product)\n",
    "        details_results.append([index, reactant] + [product] + predicted_product)\n",
    "\n",
    "        details_df = pd.DataFrame(details_results, columns=['index', input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "        details_df.to_csv(detail_predict_file, index=False)\n",
    "\n",
    "    # evaluate\n",
    "    acc_list = []\n",
    "    for repeat in range(5):\n",
    "        tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "        correct = 0\n",
    "        all_sample_num = len(test)\n",
    "        for idx, gt in enumerate(list(test[output_col])):\n",
    "            pred = tpredicted_products[idx]\n",
    "            if task in ['iupac2smiles', 'formula2smiles']:\n",
    "                try:\n",
    "                    mol = Chem.MolFromSmiles(pred)\n",
    "                    pred = Chem.MolToSmiles(mol)\n",
    "                except Exception as e:\n",
    "                    continue\n",
    "\n",
    "            if gt == pred:\n",
    "                correct += 1\n",
    "        acc = correct / all_sample_num\n",
    "        acc_list.append(acc)\n",
    "\n",
    "    # save to file\n",
    "    performance_results.append([task, np.mean(acc_list)] + acc_list)\n",
    "    print(performance_results)\n",
    "\n",
    "    # performance save based on the task\n",
    "    tem = pd.DataFrame(performance_results, columns=['task', 'avg_metric'] + ['metric_{}'.format(i) for i in range(5)])\n",
    "    tem.to_csv(performance_file, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T02:53:01.721553Z",
     "start_time": "2023-05-18T02:53:01.717538Z"
    }
   },
   "outputs": [],
   "source": [
    "import statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:31:06.471674Z",
     "start_time": "2023-05-18T03:31:06.463096Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tem = pd.read_csv(detail_save_folder + \"formula2smiles_gpt4_zero_shot_performance.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:31:06.672642Z",
     "start_time": "2023-05-18T03:31:06.669678Z"
    }
   },
   "outputs": [],
   "source": [
    "metric_cols = ['metric_{}'.format(i) for i in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:31:06.846859Z",
     "start_time": "2023-05-18T03:31:06.843253Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['metric_0', 'metric_1', 'metric_2', 'metric_3', 'metric_4']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:31:07.032547Z",
     "start_time": "2023-05-18T03:31:07.026264Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statistics.stdev(list(tem.iloc[0][metric_cols]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-18T03:31:07.213082Z",
     "start_time": "2023-05-18T03:31:07.202402Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>avg_metric</th>\n",
       "      <th>metric_0</th>\n",
       "      <th>metric_1</th>\n",
       "      <th>metric_2</th>\n",
       "      <th>metric_3</th>\n",
       "      <th>metric_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>formula2smiles</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             task  avg_metric  metric_0  metric_1  metric_2  metric_3  \\\n",
       "0  formula2smiles         0.0       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   metric_4  \n",
       "0       0.0  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T13:49:45.421432Z",
     "start_time": "2023-05-20T13:49:45.207341Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Name Prediction/results_test/'"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detail_save_folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:08:09.951872Z",
     "start_time": "2023-05-20T14:08:09.936312Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(detail_save_folder + \"valid_formula2smiles_gpt4_SMILES_20_Scaffold_SIM.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:08:11.960299Z",
     "start_time": "2023-05-20T14:08:11.955922Z"
    }
   },
   "outputs": [],
   "source": [
    "idx = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:08:12.440586Z",
     "start_time": "2023-05-20T14:08:12.424097Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>formula</th>\n",
       "      <th>smiles</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>pred_4</th>\n",
       "      <th>pred_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2928</td>\n",
       "      <td>C28H35ClN6O5</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1CCN(C(=O)CN2CCOCC2C(=O)Nc2cc(...</td>\n",
       "      <td>CCc1cc(C(=O)N2CCN(C)CC2)c(NNC(=O)COC(=O)C23CCN...</td>\n",
       "      <td>CC(=O)Nc1cc(CC(=O)COC(=O)C2(C)CCCC(=O)N(CCC)n3...</td>\n",
       "      <td>COc1cc(NS(=O)(=O)c2ccccc2NCCOc2cccc(Cl)c2)ccc1...</td>\n",
       "      <td>COCCN1C(=O)c2cc(Cl)c(OC)c(OCc3ccc(N4CCC=N4)cc3...</td>\n",
       "      <td>COc1ccc(CN2CCN(CCNC(=O)c3cc(NCc4ccccc4Cl)nc([N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5735</td>\n",
       "      <td>C21H26N2O5S</td>\n",
       "      <td>Cc1nc2cc(C(O)CC(=O)N3C4CC5CCC4(CS3(=O)=O)C5(C)...</td>\n",
       "      <td>CCOC(=O)C1=C(NC(=O)CSc2ncccc2)C(O)CC(C)C1OC</td>\n",
       "      <td>COc1ccc(S(=O)(=O)N2CCN(C(=O)c3ccccc3)CCC2)c2c1...</td>\n",
       "      <td>COc1ccc(C(=O)Nc2cccc(OS(=O)(=O)N3CCCC3)c2)cc1OC</td>\n",
       "      <td>CCSCC(NC(=O)OC(=O)C1=CC=C(C)N=C1)C(=O)N1Cc2ccc...</td>\n",
       "      <td>CCOc1ccc(NC(=O)CCC(=O)NNC2CCCS2)cc1OCC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1624</td>\n",
       "      <td>C8H10N4O2</td>\n",
       "      <td>Cc1noc(CCn2cc[nH]c2=O)n1</td>\n",
       "      <td>CCn1cnc(=O)c(=O)n1Cc1cccnc1</td>\n",
       "      <td>Nc1nc(O)ccc1NC(=O)Nc1ccc[nH]1</td>\n",
       "      <td>NNc1ccc(N)cc1OC(=O)C=N</td>\n",
       "      <td>COc1ccnc(OC)c1N1C(=O)C(=O)N=N1</td>\n",
       "      <td>Cc1nc(NC(=O)C=N)c(N)nc1O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3047</td>\n",
       "      <td>C8H13NO3</td>\n",
       "      <td>CCOC(=O)C(C(C)=O)=C(C)N</td>\n",
       "      <td>CC(=O)N1CCCC1C(=O)O</td>\n",
       "      <td>CC(=O)N1CCCC1C(=O)O</td>\n",
       "      <td>CC(=O)NCC(C)C(O)C(=O)O</td>\n",
       "      <td>CC1(N)CCCC(C(=O)O)C1</td>\n",
       "      <td>CC(C(=O)O)N1CCCCCC1=O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8848</td>\n",
       "      <td>C22H17N3O3</td>\n",
       "      <td>CC(=O)Nc1cccc(-c2nc(-c3ccc(Oc4ccccc4)cc3)no2)c1</td>\n",
       "      <td>Cc1ccc2c(c1)C(=O)N(C)C3CCC(N3c2)c1ccc(=O)n(C)n1</td>\n",
       "      <td>O=C(Nc1cccc(Nc2ccncc2)c1)c1cc(-c2ccccn2)ccc1O</td>\n",
       "      <td>O=C(Nc1cccc2c1cccc2)C1=C(C=CC=N1)c1cc(C)nc2c1c...</td>\n",
       "      <td>O=C(Nc1cccc2ccccc12)c1cccnc1-c1cnc2c(=O)[nH]cn...</td>\n",
       "      <td>O=C(Nc1ccccc1)c1cc2c(c(N3CCOCC3)c1C=O)CCCC2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index       formula                                             smiles  \\\n",
       "0   2928  C28H35ClN6O5  CC(C)(C)OC(=O)NC1CCN(C(=O)CN2CCOCC2C(=O)Nc2cc(...   \n",
       "1   5735   C21H26N2O5S  Cc1nc2cc(C(O)CC(=O)N3C4CC5CCC4(CS3(=O)=O)C5(C)...   \n",
       "2   1624     C8H10N4O2                           Cc1noc(CCn2cc[nH]c2=O)n1   \n",
       "3   3047      C8H13NO3                            CCOC(=O)C(C(C)=O)=C(C)N   \n",
       "4   8848    C22H17N3O3    CC(=O)Nc1cccc(-c2nc(-c3ccc(Oc4ccccc4)cc3)no2)c1   \n",
       "\n",
       "                                              pred_1  \\\n",
       "0  CCc1cc(C(=O)N2CCN(C)CC2)c(NNC(=O)COC(=O)C23CCN...   \n",
       "1        CCOC(=O)C1=C(NC(=O)CSc2ncccc2)C(O)CC(C)C1OC   \n",
       "2                        CCn1cnc(=O)c(=O)n1Cc1cccnc1   \n",
       "3                                CC(=O)N1CCCC1C(=O)O   \n",
       "4    Cc1ccc2c(c1)C(=O)N(C)C3CCC(N3c2)c1ccc(=O)n(C)n1   \n",
       "\n",
       "                                              pred_2  \\\n",
       "0  CC(=O)Nc1cc(CC(=O)COC(=O)C2(C)CCCC(=O)N(CCC)n3...   \n",
       "1  COc1ccc(S(=O)(=O)N2CCN(C(=O)c3ccccc3)CCC2)c2c1...   \n",
       "2                      Nc1nc(O)ccc1NC(=O)Nc1ccc[nH]1   \n",
       "3                                CC(=O)N1CCCC1C(=O)O   \n",
       "4      O=C(Nc1cccc(Nc2ccncc2)c1)c1cc(-c2ccccn2)ccc1O   \n",
       "\n",
       "                                              pred_3  \\\n",
       "0  COc1cc(NS(=O)(=O)c2ccccc2NCCOc2cccc(Cl)c2)ccc1...   \n",
       "1    COc1ccc(C(=O)Nc2cccc(OS(=O)(=O)N3CCCC3)c2)cc1OC   \n",
       "2                             NNc1ccc(N)cc1OC(=O)C=N   \n",
       "3                             CC(=O)NCC(C)C(O)C(=O)O   \n",
       "4  O=C(Nc1cccc2c1cccc2)C1=C(C=CC=N1)c1cc(C)nc2c1c...   \n",
       "\n",
       "                                              pred_4  \\\n",
       "0  COCCN1C(=O)c2cc(Cl)c(OC)c(OCc3ccc(N4CCC=N4)cc3...   \n",
       "1  CCSCC(NC(=O)OC(=O)C1=CC=C(C)N=C1)C(=O)N1Cc2ccc...   \n",
       "2                     COc1ccnc(OC)c1N1C(=O)C(=O)N=N1   \n",
       "3                               CC1(N)CCCC(C(=O)O)C1   \n",
       "4  O=C(Nc1cccc2ccccc12)c1cccnc1-c1cnc2c(=O)[nH]cn...   \n",
       "\n",
       "                                              pred_5  \n",
       "0  COc1ccc(CN2CCN(CCNC(=O)c3cc(NCc4ccccc4Cl)nc([N...  \n",
       "1             CCOc1ccc(NC(=O)CCC(=O)NNC2CCCS2)cc1OCC  \n",
       "2                           Cc1nc(NC(=O)C=N)c(N)nc1O  \n",
       "3                              CC(C(=O)O)N1CCCCCC1=O  \n",
       "4        O=C(Nc1ccccc1)c1cc2c(c(N3CCOCC3)c1C=O)CCCC2  "
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:09:18.263291Z",
     "start_time": "2023-05-20T14:09:18.256419Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Cc1noc(CCn2cc[nH]c2=O)n1'"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[idx]['smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:09:18.442413Z",
     "start_time": "2023-05-20T14:09:18.435770Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C8H10N4O2'"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[idx]['formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T14:09:19.034282Z",
     "start_time": "2023-05-20T14:09:19.023580Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CCn1cnc(=O)c(=O)n1Cc1cccnc1'"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[idx]['pred_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rdkit",
   "language": "python",
   "name": "my-rdkit-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "170px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}