corpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 5,
        "hidden": false,
        "row": 0,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "# Topic Modeling Visualisations<a id='top'></a>\n",
    "\n",
    "This page allows you to browse through topic models generated by [create_lda.ipynb](create_lda.ipynb).\n",
    "\n",
    "Scroll down to see the plots or use these links:\n",
    "\n",
    "* [Estimate number of clusters](#clusters)\n",
    "* [Corpus map](#map)\n",
    "* [Topic intensity](#topic_bubbles)\n",
    "* [Print topics](#print_topics)\n",
    "* [Development of topics over time](#stats_per_topic)\n",
    "* [Topic statistics per blog or profile](#stats_per_blog)\n",
    "* [Data table for corpus / subcorpus / blog or profile](#data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from IPython.core.pylabtools import figsize\n",
    "from sklearn.manifold import Isomap\n",
    "import seaborn\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "import qgrid\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "from IPython.display import clear_output, display\n",
    "import ipywidgets as widgets\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.spatial.distance import cdist, pdist, squareform\n",
    "import scripts.spin as sp\n",
    "from sklearn.metrics import silhouette_score\n",
    "from gensim import corpora, models\n",
    "import os\n",
    "from operator import itemgetter\n",
    "import datetime as dt\n",
    "\n",
    "figsize(15, 10)\n",
    "\n",
    "experiment = \"zeit\"\n",
    "result_path = \"projects\" + os.path.sep + experiment + os.path.sep + \"results\"\n",
    "file_name = \"topics\"\n",
    "\n",
    "number = 0\n",
    "for f in os.listdir(result_path):\n",
    "    try:\n",
    "        number = max(number, int(f.split(file_name+\"-\")[1].split(\".json\")[0]))\n",
    "    except IndexError:\n",
    "        continue\n",
    "if number > 0:\n",
    "    file_name = file_name + \"-\" + str(number)+\".json\"\n",
    "else:\n",
    "    file_name = file_name + \".json\"\n",
    "print(\"Loading\", file_name)\n",
    "    \n",
    "table = pd.read_json(os.path.join(result_path, file_name), orient='split')\n",
    "\n",
    "def percent(x, pos):\n",
    "    return str(int(x * 100)) + \"%\"\n",
    "\n",
    "formatter = FuncFormatter(percent)\n",
    "\n",
    "def corpus_stats(Subcorpus):\n",
    "    sub = table.loc[table['group'].str.contains(Subcorpus)]\n",
    "    # get top topics\n",
    "    weighted = sub['words'] / sub['words'].sum()\n",
    "    sub2 = sub.copy()\n",
    "    sub2[sub2.columns[5:]] = sub2[sub2.columns[5:]].multiply(weighted, axis='index')\n",
    "    top_topics = sub2[sub2.columns[4:]].sum()\n",
    "    new_index = ['url', 'date']\n",
    "    for idx in top_topics.index:\n",
    "        new_index.append(idx)\n",
    "    sub = sub[new_index]\n",
    "    return sub, top_topics\n",
    "    \n",
    "def topic_stats(Topics):\n",
    "    topic_stats = table.copy()\n",
    "    topic_stats['date'] = pd.to_datetime(topic_stats['date'], errors=\"coerce\", utc=True)\n",
    "    topic_stats.index = pd.DatetimeIndex(topic_stats.date)\n",
    "    weighted = topic_stats['words'] / topic_stats['words'].sum()\n",
    "    topic_stats[topic_stats.columns[5:]] = topic_stats[topic_stats.columns[5:]].multiply(weighted, axis='index')\n",
    "    \n",
    "    # reduce to topics\n",
    "    for Topic in Topics:\n",
    "        topic_stats[(topic_stats.index.year>1800)][Topic].resample('M').sum().plot(kind='line', label=Topic)\n",
    "    plt.legend(loc='best')\n",
    "    plt.show()\n",
    "    \n",
    "def plot(data, title=\"Isomap\", clrs=None, max_val=0, words=None, labels=None, huep=False):\n",
    "    if clrs is None:\n",
    "        clrs = [0]*len(data)\n",
    "    if labels is None:\n",
    "        labels = [\"\"]*len(data)\n",
    "    if words is None:\n",
    "        words = [250]*len(data)   \n",
    "    fig = plt.figure()\n",
    "    ax1 = fig.add_subplot(111)\n",
    "    matplotlib.rcParams.update({'font.size': 12})\n",
    "    plt.title(title)\n",
    "    # plt.xlim(-0.3, 0.3)\n",
    "    # plt.ylim(-0.2, 0.2)\n",
    "    ax1.tick_params(axis='both', which='both', bottom='off', top='off', left='off', right='off')\n",
    "    scatter = ax1.scatter(data.T[0], data.T[1], marker='o', s=words, c=clrs,\n",
    "                          vmin=0, vmax=max_val, cmap=matplotlib.cm.get_cmap('coolwarm'))\n",
    "    for counter in range(len(data)):\n",
    "        label_scale = words[counter]/(70*2500)\n",
    "        plt.annotate(labels[counter].split(\":\")[-1], xy=(data[counter][0]+label_scale, data[counter][1]+label_scale))\n",
    "    plt.tight_layout()\n",
    "    if huep:\n",
    "        plt.colorbar(scatter, format=formatter)\n",
    "    plt.show()\n",
    "\n",
    "# create statistics per blog\n",
    "names = table['group'].drop_duplicates()\n",
    "series = []\n",
    "for name in names:\n",
    "    sub, top_topics = corpus_stats(name)\n",
    "    series.append(top_topics)\n",
    "stats_per_blog = pd.concat(series, axis=1).transpose()\n",
    "stats_per_blog.index = names\n",
    "\n",
    "names = ['', 'everyday', 'literary', 'political']\n",
    "names.extend(pd.DataFrame({'count': table.groupby([\"group\"]).size()}).reset_index()['group'])\n",
    "\n",
    "# prepare matrix for isomap / distance matrix\n",
    "topics_per_blog = stats_per_blog.copy()\n",
    "del topics_per_blog['words']\n",
    "matrix = topics_per_blog.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimate number of clusters<a id='clusters'></a>\n",
    "\n",
    "Create an elbow plot to estimate a sane number of clusters for the corpus.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = range(1,11)\n",
    "KM = [KMeans(n_clusters=k).fit(matrix) for k in K]\n",
    "centroids = [k.cluster_centers_ for k in KM]\n",
    "\n",
    "D_k = [cdist(matrix, cent, 'euclidean') for cent in centroids]\n",
    "dist = [np.min(D,axis=1) for D in D_k]\n",
    "avgWithinSS = [sum(d)/matrix.shape[0] for d in dist]\n",
    "\n",
    "# elbow curve\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(K, avgWithinSS, 'b*-', color='r')\n",
    "plt.xlabel('Number of Clusters')\n",
    "plt.ylabel(u'Within Sum of Squares')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Corpus overview using Isomap<a id='map'></a>\n",
    "\n",
    "In this plot the 37 blogs and social media profiles are represented by their topic distribution. The Isomap algorithm is used to create a two-dimensional map of this high-dimensional topic space. Each dot represents a blog/profile; dots close to each other have a similar topic distribution. Colors indicate groups suggested by K-means clustering.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    },
    "scrolled": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "def plot_corpus(Clusters=4):\n",
    "    # cluster blogs\n",
    "    km = KMeans(n_clusters=Clusters, random_state=5)\n",
    "    clusters = km.fit_predict(matrix)\n",
    "    print(\"Silhouette score:\", silhouette_score(matrix, clusters))\n",
    "\n",
    "    # plot corpus map using Isomap\n",
    "    imap = Isomap()\n",
    "    corpus_map = imap.fit_transform(matrix)\n",
    "    figsize (13, 10)\n",
    "    plot(corpus_map, \"Corpus Map\", labels=table['group'].drop_duplicates().values, clrs=clusters, max_val=Clusters)\n",
    "    figsize (15, 10)\n",
    "\n",
    "interact(plot_corpus, Clusters=range(2,10), );"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Plot distance matrix\n",
    "\n",
    "Create a distance matrix showing distances between different blogs or social media profiles.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "distances = pdist(topics_per_blog.values, metric='euclidean')\n",
    "dist_matrix = squareform(distances)\n",
    "\n",
    "#sort blogs with SPIN\n",
    "order = np.array(range(len(dist_matrix[0])))\n",
    "new_distances, new_order = sp.spin(dist_matrix, order)\n",
    "\n",
    "# re-sort ids according to spin\n",
    "new_ids = []\n",
    "for index in new_order:\n",
    "    new_ids.append(stats_per_blog.index[index].split(\":\")[-1])\n",
    "\n",
    "dist_matrix = pd.DataFrame(new_distances)\n",
    "dist_matrix.columns = new_ids\n",
    "dist_matrix.index = new_ids\n",
    "\n",
    "mask = np.zeros_like(new_distances)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "seaborn.set_context(\"notebook\", font_scale=0.9)\n",
    "with seaborn.axes_style(\"white\"):\n",
    "    ax = seaborn.heatmap(dist_matrix, mask=mask, square=True, annot=False, fmt='.2f')\n",
    "plt.yticks(rotation=0) \n",
    "plt.xticks(rotation=90) \n",
    "plt.show()\n",
    "figsize (15, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 5,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Plot topic bubbles<a id='topic_bubbles'></a>\n",
    "\n",
    "Again, the Isomap algorithm is used to create a map of the corpus. Use the dropdown box to select a topic; colors indicate intensity of topic, bubble size is total number of words in a blog/profile.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 2,
        "hidden": false,
        "row": 5,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "words = stats_per_blog['words']\n",
    "copy = stats_per_blog.copy()\n",
    "del copy['words']\n",
    "matrix = copy.values\n",
    "\n",
    "# scale dots with blog size\n",
    "max_words = max(words)\n",
    "num = 0\n",
    "while(num < len(words)):\n",
    "    words[num] = 2500*words[num]/max_words\n",
    "    num += 1\n",
    "    \n",
    "def create_bubbles(Topic=41):\n",
    "    # Isomap\n",
    "    imap = Isomap()\n",
    "    X = imap.fit_transform(matrix)\n",
    "    plot(X, list(copy)[Topic], matrix[:, Topic], max(matrix[:, Topic]), words, huep=True)\n",
    "    \n",
    "box = {}\n",
    "counter = 0\n",
    "for label in list(table)[5:]:\n",
    "    box[label] = counter\n",
    "    counter += 1\n",
    "box = sorted(box.items(), key=lambda x: x[1])\n",
    "    \n",
    "interact(create_bubbles, Topic=box);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print topics<a id='print_topics'></a>\n",
    "\n",
    "Print the ten most typicals words for each topic.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "file_name = \"model.lda\"\n",
    "model_name = \"model\"\n",
    "\n",
    "number = 0\n",
    "for f in os.listdir(result_path):\n",
    "    try:\n",
    "        number = max(number, int(f.split(model_name+\"-\")[1].split(\".lda\")[0]))\n",
    "    except IndexError:\n",
    "        continue\n",
    "    except ValueError:\n",
    "        continue\n",
    "if number > 0:\n",
    "    file_name = model_name + \"-\" + str(number) + \".lda\"\n",
    "else: \n",
    "    file_name = model_name + \".lda\"\n",
    "\n",
    "# load LDA model and dictionary\n",
    "dictionary = corpora.Dictionary.load(os.path.join(result_path, model_name + \".dict\"))\n",
    "model = models.LdaModel.load(os.path.join(result_path, file_name))\n",
    "    \n",
    "topics = model.show_topics(num_topics=50, num_words=20, log=False, formatted=True)\n",
    "counter = 5\n",
    "for topic in sorted(topics, key=itemgetter(0)):\n",
    "    print (list(table)[counter], \":\\n\", topic[1], \"\\n\")\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": false,
        "row": 5,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "***\n",
    "## Topic development over time\n",
    "<a id='stats_per_topic'></a>\n",
    "\n",
    "Select topics to view their development over time for the whole corpus. You may select multiple topics with the CTRL key.\n",
    "\n",
    "[Back to top](#top)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 14,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "selected = [list(table)[45], list(table)[42]] \n",
    "interact(topic_stats, Topics= widgets.SelectMultiple(options=list(table)[5:], value=selected));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 5,
        "hidden": false,
        "row": 9,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Top ten topics for a blog / a profile<a id='stats_per_blog'></a>\n",
    "\n",
    "Select a blog/social media profile in the dropdown box. The naming convention is as follows: subcorpus:author:alias, e.g. everyday:goralik:zz_snorapp.\n",
    "\n",
    "[Back to top](#top)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "figsize(10,6)\n",
    "def create_bars(Subcorpus):\n",
    "    sub, top_topics = corpus_stats(Subcorpus)\n",
    "    del top_topics['words']\n",
    "    top_topics.sort_values(ascending=False, inplace=True)\n",
    "    top_topics = top_topics.head(10)\n",
    "    top_topics.plot(kind='bar')\n",
    "    plt.show()\n",
    "\n",
    "interact(create_bars, Subcorpus=names);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "## Show data table for corpus / subcorpus / blog or profile<a id='data'></a>\n",
    "Select a blog/social media profile in the dropdown box. The naming convention is as follows: subcorpus:author:alias, e.g. everyday:goralik:zz_snorapp.\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "qgrid.show_grid(table, grid_options={'fullWidthRows': True, 'explicitInitialisation':True, 'asyncEditorLoading': True, 'enableTextSelectionOnCells': True, 'syncColumnCellResize':True, 'enableColumnReorder':False,'forceFitColumns': False, 'defaultColumnWidth': 80, 'enableCellNavigation': True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "report_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}