Aminer_Prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import csv\n",
    "import nltk\n",
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "import re\n",
    "import warnings\n",
    "import random\n",
    "warnings.filterwarnings('ignore')\n",
    "from collections import Counter,defaultdict\n",
    "from nltk import word_tokenize\n",
    "from math import log\n",
    "import string\n",
    "import pickle\n",
    "from math import log\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "from gensim import models\n",
    "from string import digits\n",
    "from collections import defaultdict\n",
    "import ast\n",
    "cwd = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "osname = os.name\n",
    "if osname =='Windows':\n",
    "    sym = \"//\"\n",
    "else:\n",
    "    sym = \"/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating  Reviewers List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_path=cwd+sym+'nips_reviewer_data'+sym+'reviewers.txt'\n",
    "review_df=pd.read_csv(r_path,sep='\\t',header=None)\n",
    "review_df.columns=['sno','name']\n",
    "review_lt=list(review_df.name)\n",
    "review_sno={val:ind for ind,val in enumerate(review_lt)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=len(review_lt)\n",
    "coauth=[[0]*n for i in range(n)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##getting list of all files in a folder\n",
    "file_lt=[]\n",
    "fold_path=cwd+sym+'Aminer_Dataset'+sym\n",
    "file_path = [fold_path+sym+f for f in os.listdir(fold_path) if os.path.isfile(os.path.join(fold_path, f))]\n",
    "df=pd.DataFrame(columns=['id','authors','text','year','venue'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_89.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_90.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_91.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_92.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_93.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_94.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_95.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_96.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_97.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_98.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_99.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_100.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_101.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_102.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_103.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_104.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_105.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_106.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_107.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_108.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_109.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_110.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_111.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_112.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_113.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_114.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_115.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_116.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_117.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_130.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_131.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_132.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_133.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_134.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_135.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_136.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_137.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_138.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_139.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_140.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_141.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_142.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_143.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_118.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_119.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_120.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_121.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_122.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_123.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_124.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_125.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_126.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_127.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_128.txt Completed\n",
      "/home/tribikram/Paper-Reviewer/Aminer_Dataset//aminer_papers_129.txt Completed\n"
     ]
    }
   ],
   "source": [
    "'''data={}\n",
    "data['id']=[]\n",
    "data['author']=[]\n",
    "data['title']=[]\n",
    "data['abstract']=[]\n",
    "data['year']=[]\n",
    "data['venue']=[]\n",
    "paper={}\n",
    "paper['src']=[]\n",
    "paper['dest']=[]\n",
    "paper['auth']=[]'''\n",
    "for path in file_path[100:156]:\n",
    "    with open(path,'r') as f:\n",
    "        for entry in f:\n",
    "            try:\n",
    "                entry=ast.literal_eval(entry)\n",
    "                key_lt=list(entry.keys())\n",
    "                [title,abstract,venue,author]=['']*4\n",
    "                year=-1\n",
    "                row=[]\n",
    "                if 'title' in key_lt:\n",
    "                    title=entry['title']\n",
    "                if 'abstract' in key_lt:\n",
    "                    abstract=entry['abstract']\n",
    "                if 'id' in key_lt:\n",
    "                    pid=entry['id']\n",
    "                if 'abstract' in key_lt:\n",
    "                    abstract=entry['abstract']\n",
    "                if 'venue' in key_lt:\n",
    "                    venue=entry['venue']\n",
    "                if 'year' in key_lt:\n",
    "                    year=int(entry['year'])\n",
    "                if 'authors' in key_lt:\n",
    "                    val=entry['authors']\n",
    "                    aname=[]\n",
    "                    for a in val:\n",
    "                        name=a['name']\n",
    "                        if name in review_lt and (year==-1 or year<=2005):\n",
    "                            aname.append(name)\n",
    "                    n=len(aname)\n",
    "                    for i in range(n):\n",
    "                        for j in range(i+1,n):\n",
    "                            coauth[review_sno[aname[i]]][review_sno[aname[j]]]+=1\n",
    "                            coauth[review_sno[aname[j]]][review_sno[aname[i]]]+=1\n",
    "                    for name in aname:\n",
    "                        if year==-1 or (year<=2005 and year>=1995):\n",
    "                            data['id'].append(pid)\n",
    "                            data['author'].append(name)\n",
    "                            data['title'].append(title)\n",
    "                            data['year'].append(year)\n",
    "                            data['venue'].append(venue)\n",
    "                            data['abstract'].append(abstract)\n",
    "                if 'references' in key_lt:\n",
    "                    cite=entry['references']\n",
    "                    if (year==-1 or year<=2005):\n",
    "                        for ele in cite:\n",
    "                            paper['src'].append(pid)\n",
    "                            paper['dest'].append(ele)\n",
    "                            paper['auth'].append(aname)\n",
    "\n",
    "            except:\n",
    "                continue\n",
    "    print(path,'Completed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12584"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data.pkl','wb') as f:\n",
    "    pickle.dump(data,f)\n",
    "with open('citenet.pkl','wb') as f:\n",
    "    pickle.dump(paper,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data.pkl','rb') as f:\n",
    "    data=pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('citenet.pkl','rb') as f:\n",
    "    paper=pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'igraph'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-73ffca960e68>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0migraph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'igraph'"
     ]
    }
   ],
   "source": [
    "import igraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting python-igraph\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/a0/4e7134f803737aa6eebb4e5250565ace0e2599659e22be7f7eba520ff017/python-igraph-0.7.1.post6.tar.gz (377kB)\n",
      "\u001b[K     |████████████████████████████████| 378kB 402kB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: python-igraph\n",
      "  Building wheel for python-igraph (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/tribikram/.cache/pip/wheels/41/d6/02/34eebae97e25f5b87d60f4c0687e00523e3f244fa41bc3f4a7\n",
      "Successfully built python-igraph\n",
      "Installing collected packages: python-igraph\n",
      "Successfully installed python-igraph-0.7.1.post6\n"
     ]
    }
   ],
   "source": [
    "!pip install python-igraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}