MergePapers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen, Request\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from html.parser import HTMLParser\n",
    "import tqdm\n",
    "import math\n",
    "\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "\n",
    "from pybtex.database.input import bibtex\n",
    "import pybtex.errors\n",
    "pybtex.errors.set_strict_mode(False)\n",
    "\n",
    "\n",
    "\n",
    "import itertools\n",
    "from itertools import permutations \n",
    "\n",
    "import os\n",
    "import time\n",
    "import config\n",
    "from enum import Enum\n",
    "import glob\n",
    "\n",
    "import bibtexparser\n",
    "import BibFilesMerge as bfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "isMerged = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFileNames(bib_folder):\n",
    "    file_names = glob.glob(bib_folder+'*.bib') \n",
    "    file_names = [item.split(\"\\\\\")[1] for item in file_names]\n",
    "    return file_names\n",
    "def mergeFiles(bib_folder, name):\n",
    "    file_names = getFileNames(bib_folder)\n",
    "    \n",
    "    bfm.run(bib_folder, file_names, name, False)\n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ieee_interact3Dreality_TitleAbstract_page0.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page1.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page10.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page11.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page12.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page13.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page14.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page15.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page16.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page17.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page18.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page19.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page2.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page20.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page21.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page22.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page23.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page24.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page25.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page26.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page27.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page28.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page29.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page3.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page30.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page31.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page32.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page33.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page34.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page35.bib: 33                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page4.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page5.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page6.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page7.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page8.bib: 50                                              \n",
      "ieee_interact3Dreality_TitleAbstract_page9.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page0.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page1.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page10.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page11.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page12.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page13.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page14.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page15.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page16.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page17.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page18.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page19.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page2.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page20.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page21.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page22.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page23.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page24.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page25.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page26.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page27.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page28.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page29.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page3.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page30.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page31.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page32.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page33.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page34.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page35.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page36.bib: 28                                              \n",
      "ieee_interactAR_TitleAbstract_page4.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page5.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page6.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page7.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page8.bib: 50                                              \n",
      "ieee_interactAR_TitleAbstract_page9.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page0.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page1.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page10.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page11.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page12.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page13.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page14.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page15.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page16.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page17.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page18.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page19.bib: 3                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page2.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page3.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page4.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page5.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page6.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page7.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page8.bib: 50                                              \n",
      "ieee_interactaugmentedenvironment_TitleAbstract_page9.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page0.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page1.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page10.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page11.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page12.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page13.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page14.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page15.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page16.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page17.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page18.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page19.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page2.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page20.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page21.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page22.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page23.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page24.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page25.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page26.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page27.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page28.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page29.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page3.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page30.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page31.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page32.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page33.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page34.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page35.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page36.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page37.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page38.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page39.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page4.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page40.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page41.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page42.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page43.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page44.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page45.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page46.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page47.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page48.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page49.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page5.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page50.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page51.bib: 50                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page52.bib: 7                                              \n",
      "ieee_interactaugmentedreality_TitleAbstract_page6.bib: 50                                              \n",
      "Key 6910435               \r"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'title'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\Jan\\Documents\\GitHub\\MA-Leusmann-VRSelectionLiteratureReview\\MergePapers.ipynb Cell 4'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/MergePapers.ipynb#ch0000003?line=0'>1</a>\u001b[0m mergeFiles(\u001b[39m\"\u001b[39;49m\u001b[39m./ieee/\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39m\"\u001b[39;49m\u001b[39m../merged/IEEE_merged.bib\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "\u001b[1;32mc:\\Users\\Jan\\Documents\\GitHub\\MA-Leusmann-VRSelectionLiteratureReview\\MergePapers.ipynb Cell 3'\u001b[0m in \u001b[0;36mmergeFiles\u001b[1;34m(bib_folder, name)\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/MergePapers.ipynb#ch0000002?line=4'>5</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmergeFiles\u001b[39m(bib_folder, name):\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/MergePapers.ipynb#ch0000002?line=5'>6</a>\u001b[0m     file_names \u001b[39m=\u001b[39m getFileNames(bib_folder)\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/MergePapers.ipynb#ch0000002?line=7'>8</a>\u001b[0m     bfm\u001b[39m.\u001b[39;49mrun(bib_folder, file_names, name, \u001b[39mFalse\u001b[39;49;00m)\n",
      "File \u001b[1;32mc:\\Users\\Jan\\Documents\\GitHub\\MA-Leusmann-VRSelectionLiteratureReview\\BibFilesMerge.py:188\u001b[0m, in \u001b[0;36mrun\u001b[1;34m(folderPath, fileList, fileNameOut, logProcess)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/BibFilesMerge.py?line=184'>185</a>\u001b[0m         oldEntry \u001b[39m=\u001b[39m entryOut\n\u001b[0;32m    <a href='file:///c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/BibFilesMerge.py?line=185'>186</a>\u001b[0m         \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/BibFilesMerge.py?line=187'>188</a>\u001b[0m cleanOutTitle \u001b[39m=\u001b[39m cleanStringToCompare(entryOut\u001b[39m.\u001b[39;49mfields[\u001b[39m'\u001b[39;49m\u001b[39mtitle\u001b[39;49m\u001b[39m'\u001b[39;49m])\n\u001b[0;32m    <a href='file:///c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/BibFilesMerge.py?line=188'>189</a>\u001b[0m \u001b[39mif\u001b[39;00m (cleanTitle \u001b[39m==\u001b[39m cleanOutTitle):\n\u001b[0;32m    <a href='file:///c%3A/Users/Jan/Documents/GitHub/MA-Leusmann-VRSelectionLiteratureReview/BibFilesMerge.py?line=189'>190</a>\u001b[0m     year \u001b[39m=\u001b[39m \u001b[39mint\u001b[39m(\u001b[39mstr\u001b[39m(entry\u001b[39m.\u001b[39mfields[\u001b[39m'\u001b[39m\u001b[39myear\u001b[39m\u001b[39m'\u001b[39m]))\n",
      "File \u001b[1;32mc:\\Users\\Jan\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\pybtex\\utils.py:163\u001b[0m, in \u001b[0;36mCaseInsensitiveDict.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Jan/AppData/Local/Programs/Python/Python310/lib/site-packages/pybtex/utils.py?line=161'>162</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__getitem__\u001b[39m(\u001b[39mself\u001b[39m, key):\n\u001b[1;32m--> <a href='file:///c%3A/Users/Jan/AppData/Local/Programs/Python/Python310/lib/site-packages/pybtex/utils.py?line=162'>163</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dict[key\u001b[39m.\u001b[39;49mlower()]\n",
      "\u001b[1;31mKeyError\u001b[0m: 'title'"
     ]
    }
   ],
   "source": [
    "mergeFiles(\"./ieee/\", \"../merged/IEEE_merged.bib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6532684ccaeb1bcbbe852b7f75c67e6f1d55df7d386020fd37670376cbe3d2c9"
  },
  "kernelspec": {
   "display_name": "Python 3.10.4 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}