2A.[Omics_revamped]_CountSupportingGenes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils.openai_query import openai_chat\n",
    "import re\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('jsonFiles/OmicsRunLLM.json', 'r') as f:\n",
    "    config = json.load(f)\n",
    "context = config['CONTEXT']\n",
    "max_tokens = 2000\n",
    "rate_per_token = config['RATE_PER_TOKEN']\n",
    "LOG_FILE = config['LOG_NAME']\n",
    "DOLLAR_LIMIT = config['DOLLAR_LIMIT']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "135\n",
      "135\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>GeneSetID</th>\n",
       "      <th>GeneSetName</th>\n",
       "      <th>GeneList</th>\n",
       "      <th>updated GeneList</th>\n",
       "      <th>n_Genes</th>\n",
       "      <th>LLM Name</th>\n",
       "      <th>LLM Analysis</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NeST</td>\n",
       "      <td>Cluster1-10</td>\n",
       "      <td>Cluster1-10</td>\n",
       "      <td>CTRL HSD17B14 KIAA0232 PAQR8 PLA2G1B RNF145 SG...</td>\n",
       "      <td>CTRL HSD17B14 KIAA0232 PAQR8 PLA2G1B RNF145 SG...</td>\n",
       "      <td>12</td>\n",
       "      <td>Lipid Metabolism and Membrane Dynamics</td>\n",
       "      <td>1. CTRL, or carboxyl-terminal esterase/lipase,...</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NeST</td>\n",
       "      <td>Cluster1-11</td>\n",
       "      <td>Cluster1-11</td>\n",
       "      <td>LMF1 MFHAS1 MR1 PLA2G1B RASL11A RNF145 SLC2A6 ...</td>\n",
       "      <td>LMF1 MFHAS1 MR1 PLA2G1B RASL11A RNF145 SLC2A6 ...</td>\n",
       "      <td>12</td>\n",
       "      <td>Lipid Metabolism and Membrane Trafficking</td>\n",
       "      <td>1. LMF1 (Lipase Maturation Factor 1) is crucia...</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NeST</td>\n",
       "      <td>Cluster1-12</td>\n",
       "      <td>Cluster1-12</td>\n",
       "      <td>AMY2B CNPY2 EGFL7 LDLR LPL LRP8 LRPAP1 MYLIP P...</td>\n",
       "      <td>AMY2B CNPY2 EGFL7 LDLR LPL LRP8 LRPAP1 MYLIP P...</td>\n",
       "      <td>12</td>\n",
       "      <td>Lipid Metabolism and Receptor-Mediated Endocyt...</td>\n",
       "      <td>1. AMY2B (Amylase, alpha 2B) is an enzyme that...</td>\n",
       "      <td>0.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Source    GeneSetID  GeneSetName  \\\n",
       "0   NeST  Cluster1-10  Cluster1-10   \n",
       "1   NeST  Cluster1-11  Cluster1-11   \n",
       "2   NeST  Cluster1-12  Cluster1-12   \n",
       "\n",
       "                                            GeneList  \\\n",
       "0  CTRL HSD17B14 KIAA0232 PAQR8 PLA2G1B RNF145 SG...   \n",
       "1  LMF1 MFHAS1 MR1 PLA2G1B RASL11A RNF145 SLC2A6 ...   \n",
       "2  AMY2B CNPY2 EGFL7 LDLR LPL LRP8 LRPAP1 MYLIP P...   \n",
       "\n",
       "                                    updated GeneList  n_Genes  \\\n",
       "0  CTRL HSD17B14 KIAA0232 PAQR8 PLA2G1B RNF145 SG...       12   \n",
       "1  LMF1 MFHAS1 MR1 PLA2G1B RASL11A RNF145 SLC2A6 ...       12   \n",
       "2  AMY2B CNPY2 EGFL7 LDLR LPL LRP8 LRPAP1 MYLIP P...       12   \n",
       "\n",
       "                                            LLM Name  \\\n",
       "0             Lipid Metabolism and Membrane Dynamics   \n",
       "1          Lipid Metabolism and Membrane Trafficking   \n",
       "2  Lipid Metabolism and Receptor-Mediated Endocyt...   \n",
       "\n",
       "                                        LLM Analysis  Score  \n",
       "0  1. CTRL, or carboxyl-terminal esterase/lipase,...   0.85  \n",
       "1  1. LMF1 (Lipase Maturation Factor 1) is crucia...   0.85  \n",
       "2  1. AMY2B (Amylase, alpha 2B) is an enzyme that...   0.88  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the results of processing the 300 'omics datasets\n",
    "temp = pd.read_csv(\"data/omics_revamped_LLM_updated_genes_DF.tsv\", delimiter=\"\\t\")\n",
    "input_analyses = temp[temp[\"LLM Name\"] != \"System of unrelated proteins\"]\n",
    "print(len(temp[temp['Score']>0]))\n",
    "print(len(input_analyses))\n",
    "input_analyses.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_genes_prompt_template_v1 = \"\"\"\n",
    "\n",
    "The provided text is an analysis of a gene set to describe its common functions and to provide a name for the gene set reflecting the predominant function or functions.\n",
    "\n",
    "Do not critique the analysis or the name. Your job is to find the genes that support the name\n",
    "\n",
    "You are provided with this list of the gene symbols mentioned in the text. Only consider these genes.\n",
    "\n",
    "<genes mentioned in the text>{genes_in_text}</genes mentioned in the text>\n",
    " \n",
    "For each gene from that list mentioned in a sentence or paragraph, decide whether the text is making a *definite* assertion about the gene that supports the name.\n",
    "DO NOT make your own assesments about the gene's function or the validity of the analysis, just evaluate the text.\n",
    "<example of definite assertion>\n",
    "\"XRCC1 is involved in the DNA damage response\"\n",
    "</example of definite assertion>\n",
    "<example of non-definite assertion>\n",
    "\"E2F1 may be involved in homolougus recombination.\"\n",
    "\"SREBF1 could be related to RAD51's function.\"\n",
    "</example of non-definite assertion>\n",
    "\n",
    "As you work, briefly explain your reasoning for each gene symbol that supports the name.\n",
    "<example of explanation>\n",
    "Reasoning:\n",
    "\n",
    "1. CITED2: Involved in transcriptional regulation influencing gene expression, which is crucial for tissue development.\n",
    "2. TWIST1: Crucial for mesodermal tissue development and epithelial-mesenchymal transition (EMT), directly supporting tissue development.\n",
    "3. LMO2: Implicated in hematopoiesis and vascular development, supporting tissue development.\n",
    "4. COL5A1: Encodes a component of type V collagen, essential for connective tissue structure and function, supporting tissue development and extracellular matrix remodeling.\n",
    "</example of explanation>\n",
    "\n",
    "For each gene symbol that supports the name, add it to your list of supporting genes.\n",
    "\n",
    "<text>{text}</text>\n",
    "\n",
    "<name>{name}</name>\n",
    "\n",
    "Output the supporting genes and non-supporting genes in the following format: \n",
    "\n",
    "<format>\n",
    "-- Explanation --\n",
    "<explanation of the reasoning for the supporting genes>\n",
    "-- genes supporting the name: <list of gene symbols of genes supporting the name>\n",
    "</format>\n",
    "\"\"\"\n",
    "\n",
    "count_genes_prompt_template_v2 = \"\"\"\n",
    "Analyze the provided text, which describes a gene set's common functions and suggests a name reflecting its predominant function(s). Your task is to identify genes that support this name based solely on the information given in the text.\n",
    "\n",
    "Context: Gene sets are groups of genes that share common biological functions, pathways, or other characteristics. Naming these sets based on their predominant functions helps researchers quickly understand their significance.\n",
    "\n",
    "Input:\n",
    "1. A list of gene symbols in the gene set, provided in comma-separated format:\n",
    "<gene set>{genes_in_text}</gene set>\n",
    "\n",
    "2. The analysis text:\n",
    "<text>{text}</text>\n",
    "\n",
    "3. The suggested name for the gene set:\n",
    "<name>{name}</name>\n",
    "\n",
    "Instructions:\n",
    "1. Evaluate each gene from the provided list that is mentioned in the text.\n",
    "2. 2. Consider a gene as supporting the suggested name if (and only if) it makes a definite assertion about the gene in support of the supporting name.\n",
    "   - A definite assertion clearly states a gene's function or role without using speculative language.\n",
    "   - Example of a definite assertion: \"XRCC1 is involved in the DNA damage response\"\n",
    "   - Example of a non-definite assertion: \"E2F1 may be involved in homologous recombination\"\n",
    "3. If a gene is mentioned multiple times, consider the strongest assertion made about it.\n",
    "5. For each gene you determine supports the name:\n",
    "   - Briefly explain your reasoning (max 50 words per gene)\n",
    "   - Assign a confidence level (High, Medium, Low) based on the strength of the assertion\n",
    "6. Handle acronyms or alternative gene names as equivalent to official gene symbols.\n",
    "7. If no genes seem to support the name or if all genes support it, state this observation.\n",
    "\n",
    "Output your analysis in the following format:\n",
    "\n",
    "-- Summary --\n",
    "[Provide a brief summary (max 100 words) of why the selected genes support the given name]\n",
    "\n",
    "-- Explanation --\n",
    "[Gene Symbol]: [Confidence Level]\n",
    "[Explanation of reasoning (max 50 words)]\n",
    "\n",
    "[Repeat for each supporting gene]\n",
    "\n",
    "-- genes supporting the name: [List of gene symbols of genes supporting the name]\n",
    "-- genes supporting the name with confidence = High: [List of gene symbols with high confidence]\n",
    "\n",
    "Do not critique the analysis or the name. Base your evaluation solely on the information provided in the text.\n",
    "\"\"\"\n",
    "\n",
    "count_genes_prompt_template = \"\"\"\n",
    "Analyze the provided text, which describes a gene set's common functions and suggests a name reflecting its predominant function(s). Your task is to identify genes that support this name based solely on the information given in the text.\n",
    "\n",
    "Context: Gene sets are groups of genes that share common biological functions, pathways, or other characteristics. Naming these sets based on their predominant functions helps researchers quickly understand their significance.\n",
    "\n",
    "Input:\n",
    "1. A list of gene symbols in the gene set, provided in comma-separated format:\n",
    "<gene set>{genes_in_text}</gene set>\n",
    "\n",
    "2. The analysis text:\n",
    "<text>{text}</text>\n",
    "\n",
    "3. The suggested name for the gene set:\n",
    "<name>{name}</name>\n",
    "\n",
    "Instructions:\n",
    "1. Evaluate each gene from the provided list that is mentioned in the text.\n",
    "2. Determine if the text makes a definite assertion about the gene that supports the given name.\n",
    "   - A definite assertion clearly states a gene's function or role without using speculative language.\n",
    "   - Example of a definite assertion: \"XRCC1 is involved in the DNA damage response\"\n",
    "   - Example of a non-definite assertion: \"E2F1 may be involved in homologous recombination\"\n",
    "3. If a gene is mentioned multiple times, consider the strongest assertion made about it.\n",
    "4. In case of contradictory statements about a gene, favor the most recent or specific assertion.\n",
    "5. For each gene you determine supports the name:\n",
    "   - Briefly explain your reasoning (max 50 words per gene)\n",
    "   - Assign a confidence level (High, Medium, Low) based on the strength of the assertion\n",
    "6. Handle acronyms or alternative gene names as equivalent to official gene symbols.\n",
    "7. If no genes seem to support the name or if all genes support it, state this observation.\n",
    "\n",
    "Output your analysis in the following format:\n",
    "\n",
    "-- Summary --\n",
    "[Provide a brief summary (max 100 words) of why the selected genes support the given name]\n",
    "\n",
    "-- Explanation --\n",
    "[Gene Symbol]: [Confidence Level]\n",
    "[Explanation of reasoning (max 50 words)]\n",
    "\n",
    "[Repeat for each supporting gene]\n",
    "\n",
    "-- genes supporting the name: [List of gene symbols of genes supporting the name]\n",
    "-- genes supporting the name with confidence = High: [List of gene symbols with high confidence]\n",
    "\n",
    "Do not critique the analysis or the name. Base your evaluation solely on the information provided in the text.\n",
    "\"\"\"\n",
    "\n",
    "def parse_analysis(text):\n",
    "    formatted_genes = \"\"\n",
    "    supporting_count = 0\n",
    "    try:\n",
    "        #supporting_genes = re.search(r\"genes supporting the name: (.+)\", text).group(1).strip(\"[]\").split(\", \")\n",
    "        supporting_genes = re.search(r\"genes supporting the name with confidence = High: (.+)\", text).group(1).strip(\"[]\").split(\", \")\n",
    "        supporting_count = len(supporting_genes)\n",
    "        formatted_genes = \" \".join(supporting_genes)\n",
    "    except Exception as e:\n",
    "        print(text)\n",
    "\n",
    "    return formatted_genes, supporting_count\n",
    "\n",
    "def get_genecount_mentioned_in_text(genelist, text):\n",
    "    genes = set(genelist.split(\" \"))\n",
    "    number_mentioned = sum(1 for gene in genes if gene in text)\n",
    "    return number_mentioned\n",
    "\n",
    "def get_genes_supporting_name(name, text, prompt_template, genes_in_text):\n",
    "    prompt = prompt_template.format(genes_in_text=genes_in_text, text=text, name=name)\n",
    "    temperature = 0.0\n",
    "    model = \"gpt-4o\"\n",
    "    analysis, finger_print  = openai_chat(context, prompt, model, temperature, max_tokens, rate_per_token, LOG_FILE, DOLLAR_LIMIT)\n",
    "    # print(analysis)\n",
    "    supporting_genes, supporting_count = parse_analysis(analysis)\n",
    "    return supporting_genes, supporting_count, analysis\n",
    "\n",
    "def process_gene_set_analyses(input_analyses, prompt_template, number_to_process=None, progress_file=\"data/supporting_gene_counts_progress.tsv\"):\n",
    "    if number_to_process is not None:\n",
    "        analyses = input_analyses.head(number_to_process).copy()\n",
    "    else:\n",
    "        analyses = input_analyses.copy()\n",
    "    \n",
    "    print(f\"Processing {len(analyses)} analyses\")\n",
    "\n",
    "    def get_mentioned(row): \n",
    "        return get_genecount_mentioned_in_text(row[\"GeneList\"], row[\"LLM Analysis\"])\n",
    "\n",
    "    def get_supporting(row):\n",
    "        print(f'processing {row[\"GeneSetName\"]}...')\n",
    "        return get_genes_supporting_name(row[\"LLM Name\"], row[\"LLM Analysis\"], prompt_template, row[\"GeneList\"])\n",
    "    \n",
    "    #initialize columns\n",
    "    analyses[\"Supporting Genes\"] = None\n",
    "    analyses[\"Supporting Count\"] = None\n",
    "    analyses[\"LLM Support Analysis\"] = None\n",
    "    analyses[\"GenesMentionedInText\"] = None\n",
    "\n",
    "    for idx, row in analyses.iterrows():\n",
    "        supporting_genes, supporting_count, supporting_analysis = get_supporting(row)\n",
    "        mentioned_genes = get_mentioned(row)\n",
    "        analyses.loc[idx, \"Supporting Genes\"] = supporting_genes\n",
    "        analyses.loc[idx, \"Supporting Count\"] = supporting_count\n",
    "        analyses.loc[idx, \"LLM Support Analysis\"] = supporting_analysis\n",
    "        analyses.loc[idx, \"GenesMentionedInText\"] = mentioned_genes\n",
    "\n",
    "        # results = analyses[[\n",
    "        #     \"Source\", \"GeneSetID\", \"GeneSetName\", \"GeneList\", \"n_Genes\", \"Supporting Genes\", \n",
    "        #     \"Supporting Count\", \"LLM Analysis\",\n",
    "        #     \"LLM Support Analysis\", \"GenesMentionedInText\"]].copy()\n",
    "        analyses.to_csv(progress_file, sep='\\t', index=False)\n",
    "    \n",
    "    return analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = process_gene_set_analyses(input_analyses, count_genes_prompt_template, number_to_process=None)\n",
    "results_df.loc[:,'LLM_coverage'] = results_df.apply(lambda row: round(row['Supporting Count'] / row['n_Genes'], 2), axis=1)\n",
    "# results_df.to_csv(\"data/omics_revamped_LLM_Enrichr_simVals_new_genecounts_unprocessed__DF.tsv\", sep=\"\\t\", index=False)\n",
    "# results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_results_df = results_df.copy()\n",
    "\n",
    "# Replace newlines within text values with a placeholder\n",
    "processed_results_df = processed_results_df.map(lambda x: str(x).replace('\\n', '||'))\n",
    "\n",
    "processed_results_df.to_csv(\"data/omics_revamped_LLM_genecounts_DF.tsv\", sep=\"\\t\", index=False)\n",
    "processed_results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}