create_lda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create LDA for Corpus<a id='top'></a>\n",
    "\n",
    "0. Download an available corpus or create a new one. For the latter, create a JSON file for each subcorpus/group of texts of your corpus; each text is then a line in a JSON file. One way to do this is to crawl websites using [scrapy](https://scrapy.org) with these flags: \"-o result.json -t json\" (see [sample crawlers](./scripts/scraper/spiders) and [example item file](./scripts/scraper/items.py)). An example JSON file is [here](./scripts/example.json).\n",
    "1. [Prepare corpus for the LDA](#prepare). This notebook demonstrates how to load a (German) TEI xml, extract metadata and texts and filter unwanted POS (only nouns are left). The result is then saved as a json which can be used in the subsequent cells. You can also prepare your corpus externally, see my [example](./scripts/text.py) which is tailored to Russian texts. It removes all non-cyrillic characters, removes all words which are not nouns and sets all nouns into first person singular using POS tagging. The result is again saved in a json file\n",
    "2. [Create LDA model for the corpus](#create)\n",
    "3. [Compute topic distribution for corpus](#compute)\n",
    "4. [Explore corpus](corpus.ipynb) (different notebook)\n",
    "\n",
    "Due to copyright reasons I cannot publish the scraped raw data. The results of the smoothing process in step 2 are [here](./projects/); they are used in the examples below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from gensim import corpora, models\n",
    "import logging\n",
    "import errno\n",
    "import pandas as pd\n",
    "from dateutil import parser\n",
    "import pytz\n",
    "import numpy as np\n",
    "import json\n",
    "import xml.etree.ElementTree as ET\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "# set global paths for corpus etc.\n",
    "experiment = \"zeit\"\n",
    "raw_path = \"projects\" + os.path.sep + experiment + os.path.sep + \"raw\"\n",
    "corpus_path = \"projects\" + os.path.sep + experiment + os.path.sep + \"corpus\"\n",
    "result_path = \"projects\" + os.path.sep + experiment + os.path.sep + \"results\"\n",
    "model_name = \"model\"\n",
    "topics_name = \"topics\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create LDA model for corpus<a id='create'></a>\n",
    "\n",
    "This cell creates the topic model for the specified corpus stored in JSON files\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_topics = 50\n",
    "\n",
    "try:\n",
    "    os.makedirs(result_path)\n",
    "except OSError as exception:\n",
    "    if exception.errno != errno.EEXIST:\n",
    "        raise\n",
    "\n",
    "# load corpus\n",
    "corpus = []   \n",
    "try:\n",
    "    # load prepared corpus\n",
    "    corpus = corpora.MmCorpus(os.path.join(result_path, model_name + \".corp\"))\n",
    "    dictionary = corpora.Dictionary.load(os.path.join(result_path, model_name + \".dict\"))\n",
    "except FileNotFoundError:\n",
    "    # convert json corpus\n",
    "    for json_file in sorted(os.listdir(corpus_path)):\n",
    "        print(\"File: \", json_file)\n",
    "        if json_file.endswith(\".json\"):\n",
    "            # get data\n",
    "            json_data = open(os.path.join(corpus_path, json_file))\n",
    "            data = json.load(json_data)\n",
    "            json_data.close()\n",
    "            for entry in data:\n",
    "                try:\n",
    "                    corpus.append(entry[\"text\"].split())\n",
    "                except AttributeError:\n",
    "                    corpus.append(entry[\"text\"])\n",
    "\n",
    "    print(\"File extraction complete.\")\n",
    "\n",
    "    dictionary = corpora.Dictionary(corpus)\n",
    "    dictionary.save(os.path.join(result_path, model_name + \".dict\"))\n",
    "\n",
    "    corpus = [dictionary.doc2bow(text) for text in corpus]\n",
    "    corpora.MmCorpus.serialize(os.path.join(result_path, model_name + \".corp\"), corpus)    \n",
    "\n",
    "lda = models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=number_of_topics, alpha='auto', eval_every=5, passes=20)\n",
    "\n",
    "start = 1\n",
    "while os.path.isfile(os.path.join(result_path, model_name + \"-\" +str(start)+ \".lda\")):\n",
    "    start += 1\n",
    "\n",
    "lda.save(os.path.join(result_path, model_name + \"-\" +str(start)+ \".lda\"))\n",
    "\n",
    "print(\"LDA saved as\", os.path.join(result_path, model_name + \"-\" +str(start)+ \".lda\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute topic distribution for corpus<a id='compute'></a>\n",
    "\n",
    "[Back to top](#top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# entries published after max_date are ignored\n",
    "utc = pytz.UTC\n",
    "max_date = parser.parse(\"3014-12-31 23:59:59\")\n",
    "max_date_utc = utc.localize(parser.parse(\"3014-12-31 23:59:59\"))\n",
    "\n",
    "number = 0\n",
    "for f in os.listdir(result_path):\n",
    "    try:\n",
    "        number = max(number, int(f.split(model_name+\"-\")[1].split(\".lda\")[0]))\n",
    "    except IndexError:\n",
    "        continue\n",
    "if number > 0:\n",
    "    file_name = model_name + \"-\" + str(number) + \".lda\"\n",
    "else: \n",
    "    file_name = model_name + \".lda\"\n",
    "\n",
    "# load LDA model and dictionary\n",
    "dictionary = corpora.Dictionary.load(os.path.join(result_path, model_name + \".dict\"))\n",
    "model = models.LdaModel.load(os.path.join(result_path, file_name))\n",
    "\n",
    "# new fields for compatibility, default values from\n",
    "# https://radimrehurek.com/gensim/models/ldamodel.html\n",
    "try:\n",
    "    x = model.minimum_probability\n",
    "except AttributeError:\n",
    "    model.minimum_probability = 0.01\n",
    "    model.minimum_phi_value = 0.01\n",
    "    model.per_word_topics = False\n",
    "    model.random_state = np.random.RandomState()\n",
    "\n",
    "columns = ['group', 'url', 'date', 'comment_count', 'words']\n",
    "columns.extend([str(topic) for topic in range(model.num_topics)])\n",
    "\n",
    "result = []\n",
    "\n",
    "# sort files\n",
    "for json_file in sorted(os.listdir(corpus_path)):\n",
    "\n",
    "    print(\"File: \", json_file)\n",
    "\n",
    "    if json_file.endswith(\".json\"):\n",
    "        # get data\n",
    "        with open(os.path.join(corpus_path, json_file)) as json_data:\n",
    "            data = json.load(json_data)\n",
    "\n",
    "        removed = 0\n",
    "        too_short = 0\n",
    "\n",
    "        for entry in data:\n",
    "            # check if entry is within data range\n",
    "            try:\n",
    "                date = parser.parse(entry[\"date\"])\n",
    "                try:\n",
    "                    if date > max_date:\n",
    "                        removed += 1\n",
    "                        continue\n",
    "                except TypeError:\n",
    "                    if date > max_date_utc:\n",
    "                        removed += 1\n",
    "                        continue\n",
    "            except ValueError:\n",
    "                print(\"Wrong format\", entry[\"date\"])\n",
    "\n",
    "            # get topic distribution for entry\n",
    "            line = {}\n",
    "            try:\n",
    "                text = entry[\"text\"].split(\" \")\n",
    "            except AttributeError:\n",
    "                text = entry[\"text\"]\n",
    "                \n",
    "            # filter too short entries\n",
    "            if len(text) < 5:\n",
    "                too_short += 1\n",
    "                continue\n",
    "\n",
    "            topics = [0] * model.num_topics\n",
    "            for (topic, prop) in model[dictionary.doc2bow(text)]:\n",
    "                topics[topic] = prop\n",
    "            line[\"group\"] = json_file.split(\".json\")[0]\n",
    "            line[\"url\"] = entry[\"url\"]\n",
    "            line[\"date\"] = entry['date']\n",
    "            line[\"words\"] = len(text)\n",
    "            line[\"comment_count\"] = entry[\"comment_count\"]\n",
    "            for counter in range(len(topics)):\n",
    "                line[str(counter)] = topics[counter]\n",
    "            result.append(line)\n",
    "\n",
    "        print(\"Total number of entries:\", len(data))\n",
    "        print(\"Removed because of date: \", removed)\n",
    "        print(\"Removed because too short: \", too_short)\n",
    "        print(\"Remaining:\", (len(data) - removed - too_short))\n",
    "            \n",
    "frame = pd.DataFrame(result)\n",
    "print(columns)\n",
    "frame = frame[columns]\n",
    "start = 1\n",
    "while os.path.isfile(os.path.join(result_path, topics_name + \"-\" +str(start)+ \".json\")):\n",
    "    start += 1\n",
    "\n",
    "frame.to_json(os.path.join(result_path, topics_name + \"-\" + str(start) + \".json\"), orient='split')\n",
    "print (\"Created\", os.path.join(result_path, topics_name + \"-\" + str(start) + \".json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}