hw2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from collections import defaultdict\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import DBSCAN,KMeans"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Данные предобработаны так:\n",
    "\n",
    "    удаляются все цифры и знаки препинания, так как много номеров документов и телефонов, что сниает информативность\n",
    "    текста    \n",
    "    Далее строим все 1-грамы и смотрим на частотность слов\n",
    "    Так же смотрим ни 2-грамы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "ds = pd.read_csv(\"hillary-clinton-emails/Emails.csv\")\n",
    "ds = ds.fillna('')\n",
    "persons = pd.read_csv(\"hillary-clinton-emails/Persons.csv\")\n",
    "ds.ExtractedBodyText = ds.ExtractedBodyText.apply(lambda x: re.sub(r\"\\d\",\" \", x))\n",
    "#ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(2657, u'state'),\n",
       " (2393, u'pm'),\n",
       " (1359, u'secretary'),\n",
       " (1258, u'obama'),\n",
       " (1241, u'said'),\n",
       " (1129, u'president'),\n",
       " (1125, u'office'),\n",
       " (1106, u'new'),\n",
       " (1027, u'time'),\n",
       " (972, u'house')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(ngram_range=(1,1),stop_words='english')\n",
    "X = vectorizer.fit_transform(ds.ExtractedBodyText)\n",
    "features = vectorizer.get_feature_names()\n",
    "counts = np.sum(X, axis=0)\n",
    "con = []\n",
    "for i in range(counts.shape[1]):\n",
    "    con.append(counts[0, i])\n",
    "a= zip(con, features)\n",
    "a.sort(reverse=True)\n",
    "a[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(761, u'state gov'),\n",
       " (460, u'secretary office'),\n",
       " (455, u'state department'),\n",
       " (449, u'united states'),\n",
       " (424, u'white house'),\n",
       " (315, u'en route'),\n",
       " (299, u'pm secretary'),\n",
       " (293, u'department state'),\n",
       " (284, u'office pm'),\n",
       " (248, u'secretary state')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(ngram_range=(2,2),stop_words='english', min_df=5)\n",
    "X = vectorizer.fit_transform(ds.ExtractedBodyText)\n",
    "features = vectorizer.get_feature_names()\n",
    "counts = np.sum(X, axis=0)\n",
    "con = []\n",
    "for i in range(counts.shape[1]):\n",
    "    con.append(counts[0, i])\n",
    "a= zip(con, features)\n",
    "a.sort(reverse=True)\n",
    "a[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Подготовим набор данных для обработки nltk. Нужно получить один массив, со словами-элементами:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "ds_for_bigram = ds.ExtractedBodyText.apply(lambda x: re.sub(\"[^a-zA-Z]\", \" \", x.lower()).split())\n",
    "plain_ds = []\n",
    "for item in ds_for_bigram:\n",
    "    plain_ds.extend(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('chico', 'menashe'),\n",
       " ('golestan', 'gilan'),\n",
       " ('ilam', 'bushehr'),\n",
       " ('khuzestan', 'zanjan'),\n",
       " ('nagorno', 'karabakh'),\n",
       " ('semnan', 'sistan'),\n",
       " ('zanjan', 'semnan'),\n",
       " ('buenos', 'aires'),\n",
       " ('elliott', 'trudeau'),\n",
       " ('fars', 'qazvin')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "finder = BigramCollocationFinder.from_words(plain_ds)\n",
    "finder.apply_freq_filter(5)\n",
    "finder.nbest(bigram_measures.pmi, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(ngram_range=(1,1),stop_words='english', min_df=4)\n",
    "X = vectorizer.fit_transform(ds.ExtractedBodyText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7945, 8059)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 1 3 1 1 3 2 1 2 1 2 1 1 1 1 1 1 3 3 3 5 1 1 1 1 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "model = KMeans(n_clusters=7, random_state=42)\n",
    "svd = TruncatedSVD(n_components=500, random_state=123)\n",
    "features = svd.fit_transform(X)\n",
    "preds = model.fit_predict(features)\n",
    "print(preds[:30])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "ind = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "487\n",
      "[mailto\n",
      "RELEASE IN PART\n",
      "B \n"
     ]
    }
   ],
   "source": [
    "#print preds[214]\n",
    "for i in range(7945):\n",
    "    if preds[i]==4:\n",
    "        if i > ind:\n",
    "            ind = i\n",
    "            break\n",
    "print ind\n",
    "print ds.ExtractedBodyText[ind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[38, 450, 53, 108, 175, 6503]\n"
     ]
    }
   ],
   "source": [
    "right = []\n",
    "fyi = []\n",
    "rel = []\n",
    "fifth = []\n",
    "six = []\n",
    "first = []\n",
    "for i in range(7945):\n",
    "    if preds[i]==0:\n",
    "        right.append(i)\n",
    "    if preds[i] == 1:\n",
    "        first.append(i)\n",
    "    if preds[i] == 2:\n",
    "        fyi.append(i)\n",
    "    if preds[i] == 4:\n",
    "        rel.append(i)\n",
    "    if preds[i] == 5:\n",
    "        fifth.append(i)\n",
    "    if preds[i] == 6:\n",
    "        six.append(i)\n",
    "print map(len, [right, fyi, rel, fifth, six, first])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "В результате кластеризации выделились отдельные классы коротких сообщений, вроде:\n",
    "-\"Ok\"\n",
    "-\"pls print\"\n",
    "-\"fyi\"\n",
    "-\"you right\"\n",
    "-\"release in part b (press release)\"\n",
    "etc.\n",
    "Так получается, потому что таких писем достаточно много датасете. \n",
    "На мой взгляд результат кластеризации является весьма интерпретируемым.\n",
    "Что бы точнее произвести кластеризацию, необходимо исключить из датасета короткие сообщения, и провести повторную кластеризацию.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}