data_exploration/resource_statistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Imports:\n",
    "Import packages and load models"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "import numpy as np\n",
    "import spacy\n",
    "from spacy.lang.de.stop_words import STOP_WORDS\n",
    "from matplotlib import pyplot as plt\n",
    "import re\n",
    "\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "import pyLDAvis.sklearn\n",
    "\n",
    "import string\n",
    "\n",
    "from enum import Enum\n",
    "\n",
    "nlp = spacy.load('de_core_news_md')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Party enum"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Party(Enum):\n",
    "    AFD = 0\n",
    "    CDU = 1\n",
    "    FDP = 2\n",
    "    GRUENE = 3\n",
    "    LINKE = 4\n",
    "    SPD = 5\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Preprocessing and structure:"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def all_statistics(party: Party):\n",
    "    with open('../resources/' + party.name + '.txt', encoding='utf-8', errors='ignore') as txt:\n",
    "        file = \" \".join(l.rstrip() for l in txt)\n",
    "    text = nlp(file)\n",
    "    words = [token.lemma_ for token in text\n",
    "             if not (token.is_stop\n",
    "                     or token.is_punct\n",
    "                     or token.is_space\n",
    "                     or token.like_num\n",
    "                     or token.like_url\n",
    "                     or token.like_email\n",
    "                     or token.is_currency\n",
    "                     or token.text.upper() == party.name\n",
    "                     or token.pos_ == 'VBZ'  # Verb, 3rd person singular present\n",
    "                     or token.pos_ == 'ADP'  # adposition\n",
    "                     or token.pos_ == 'PRON'  # pronoun\n",
    "                     or token.pos_ == 'AUX'  # auxiliary\n",
    "                     or token.text == '>'\n",
    "                     or (party == Party.FDP and (token.text == 'Freie' or token.text == 'Demokraten'))\n",
    "                     or not re.match('[\\x00-\\x7F]+', token.text))]  # remove non ascii\n",
    "    word_freq = Counter(words)\n",
    "    most_freq_nouns = word_freq.most_common(50)\n",
    "    words_as_text = \" \".join([word for word in words])\n",
    "    return text, words, word_freq, most_freq_nouns, words_as_text"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Gather statistics for all parties"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# statistics_tuple = (text, words, word_freq, most_freq_nouns, words_as_text)\n",
    "statistics = {}\n",
    "\n",
    "for party in Party:\n",
    "    statistics[party] = all_statistics(party)\n",
    "\n",
    "print('Loading statistics is finished!')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n",
      "/home/delta/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  from imp import reload\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "\u001B[0;32m/tmp/ipykernel_30813/2780929563.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      4\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mparty\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mParty\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 5\u001B[0;31m     \u001B[0mstatistics\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mparty\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mall_statistics\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mparty\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      6\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      7\u001B[0m \u001B[0mprint\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'Loading statistics is finished!'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/tmp/ipykernel_30813/995982044.py\u001B[0m in \u001B[0;36mall_statistics\u001B[0;34m(party)\u001B[0m\n\u001B[1;32m      2\u001B[0m     \u001B[0;32mwith\u001B[0m \u001B[0mopen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'../resources/'\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0mparty\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mname\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0;34m'.txt'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mencoding\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m'utf-8'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0merrors\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m'ignore'\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0mtxt\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m         \u001B[0mfile\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m\" \"\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mjoin\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0ml\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrstrip\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0ml\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mtxt\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 4\u001B[0;31m     \u001B[0mtext\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnlp\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfile\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      5\u001B[0m     words = [token.lemma_ for token in text\n\u001B[1;32m      6\u001B[0m              if not (token.is_stop\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/language.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, text, disable, component_cfg)\u001B[0m\n\u001B[1;32m   1012\u001B[0m                 \u001B[0merror_handler\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mproc\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget_error_handler\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1013\u001B[0m             \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1014\u001B[0;31m                 \u001B[0mdoc\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mproc\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mcomponent_cfg\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mname\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m{\u001B[0m\u001B[0;34m}\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m  \u001B[0;31m# type: ignore[call-arg]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1015\u001B[0m             \u001B[0;32mexcept\u001B[0m \u001B[0mKeyError\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1016\u001B[0m                 \u001B[0;31m# This typically happens if a component is not initialized\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/pipeline/trainable_pipe.pyx\u001B[0m in \u001B[0;36mspacy.pipeline.trainable_pipe.TrainablePipe.__call__\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/pipeline/transition_parser.pyx\u001B[0m in \u001B[0;36mspacy.pipeline.transition_parser.Parser.set_annotations\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/pipeline/_parser_internals/ner.pyx\u001B[0m in \u001B[0;36mspacy.pipeline._parser_internals.ner.BiluoPushDown.set_annotations\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/tokens/doc.pyx\u001B[0m in \u001B[0;36mspacy.tokens.doc.Doc.set_ents\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;32m~/PycharmProjects/TextAnalytics/venv/lib/python3.7/site-packages/spacy/tokens/doc.pyx\u001B[0m in \u001B[0;36mspacy.tokens.doc.SetEntsDefault.values\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;32m/usr/lib/python3.7/enum.py\u001B[0m in \u001B[0;36m__members__\u001B[0;34m(cls)\u001B[0m\n\u001B[1;32m    358\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcls\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_member_names_\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    359\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 360\u001B[0;31m     \u001B[0;34m@\u001B[0m\u001B[0mproperty\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    361\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0m__members__\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcls\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    362\u001B[0m         \"\"\"Returns a mapping of member name->value.\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Plotting:"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "source": [
    "def plotting_png(most_freq_words, name, show_plot=True, save_plot=False):\n",
    "    ranked_frequencies = []\n",
    "    x_axis = []\n",
    "    y_axis = []\n",
    "\n",
    "    for i, word in zip(range(len(most_freq_words)), most_freq_words):\n",
    "        ranked_frequencies.insert(i, word)\n",
    "\n",
    "    i = 0\n",
    "    for (x, y) in ranked_frequencies:\n",
    "        x_axis.insert(i, x)\n",
    "        y_axis.insert(i, y)\n",
    "        i += 1\n",
    "\n",
    "    y = np.arange(50)\n",
    "    plt.rcParams['figure.autolayout'] = True\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.bar(y, y_axis, .5, color='blue')\n",
    "    ax.set_xticks(y, labels=x_axis, rotation=45, ha='right')\n",
    "\n",
    "    ax.tick_params(labelcolor='white')\n",
    "\n",
    "    fig.set_figwidth(fig.get_figwidth() * 2)\n",
    "    plt.title(name, color='white')\n",
    "\n",
    "    if save_plot:\n",
    "        plt.savefig('graphs/graph_' + name + '.svg', format='svg')\n",
    "    if show_plot:\n",
    "        plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Plot the most frequent words for all parties"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for party in Party:\n",
    "    plotting_png(statistics[party][3], party.name)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Select topics with given model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def selected_topics(model, vectorizer, top_n=10):\n",
    "    for idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic %d:\" % (idx))\n",
    "        print([(vectorizer.get_feature_names()[i], topic[i])\n",
    "               for i in topic.argsort()[:-top_n - 1:-1]])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Tokenize and remove stopwords"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "punctuations = string.punctuation\n",
    "stopwords = list(STOP_WORDS)\n",
    "\n",
    "\n",
    "def spacy_tokenizer(sentence):\n",
    "    tokens = nlp(sentence)\n",
    "    tokens = [word.lemma_.lower().strip() if word.lemma_ != \"-PRON-\" else word.lower_ for word in tokens]\n",
    "\n",
    "    tokens = [word for word in tokens if word not in stopwords and word not in punctuations]\n",
    "    tokens = \" \".join([i for i in tokens])\n",
    "    return tokens"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Run model with party manifesto"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def run_models(party: Party, MODEL_LDA=False, MODEL_NMF=False, MODEL_LSI=False):\n",
    "    with open('../resources/' + party.name + '.txt', encoding='utf-8', errors='ignore') as txt:\n",
    "        file = \" \".join(l.rstrip() for l in txt)\n",
    "\n",
    "    vectorizer = CountVectorizer(min_df=.01, max_df=1, lowercase=True, token_pattern='[a-zA-Z\\-][a-zA-Z\\-]{2,}')\n",
    "\n",
    "    data_vectorizer = vectorizer.fit_transform(map(spacy_tokenizer, file.split(\"\\n\\n\")))\n",
    "\n",
    "    num_topics = 20\n",
    "    model = None\n",
    "    if MODEL_LDA:\n",
    "        model = LatentDirichletAllocation(n_components=num_topics, max_iter=20, learning_method='online', verbose=True)\n",
    "        data_lda = model.fit_transform(data_vectorizer)\n",
    "\n",
    "        print(\"LDA Model:\")\n",
    "        selected_topics(model, vectorizer)\n",
    "\n",
    "    if MODEL_NMF:\n",
    "        model = NMF(n_components=num_topics)\n",
    "        data_nmf = model.fit_transform(data_vectorizer)\n",
    "\n",
    "        print(\"NMF Model:\")\n",
    "        selected_topics(model, vectorizer)\n",
    "\n",
    "    if MODEL_LSI:\n",
    "        model = TruncatedSVD(n_components=num_topics)\n",
    "        data_lsi = model.fit_transform(data_vectorizer)\n",
    "\n",
    "        print(\"LSI Model:\")\n",
    "        selected_topics(model, vectorizer)\n",
    "\n",
    "    if model is not None:\n",
    "        pyLDAvis.enable_notebook()\n",
    "        dash = pyLDAvis.sklearn.prepare(model, data_vectorizer, vectorizer, mds='tsne')\n",
    "        dash"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "run_models(Party.FDP, MODEL_LSI=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}