code/sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "module functions cannot set METH_CLASS or METH_STATIC",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mValueError\u001B[0m                                Traceback (most recent call last)",
      "Input \u001B[0;32mIn [3]\u001B[0m, in \u001B[0;36m<cell line: 6>\u001B[0;34m()\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mre\u001B[39;00m\n\u001B[1;32m      5\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mpandas\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mpd\u001B[39;00m\n\u001B[0;32m----> 6\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mspacy\u001B[39;00m\n\u001B[1;32m      7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mlxml\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m etree\n\u001B[1;32m      8\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mlxml\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01metree\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m XMLSyntaxError\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/spacy/__init__.py:11\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      8\u001B[0m setup_default_warnings()  \u001B[38;5;66;03m# noqa: E402\u001B[39;00m\n\u001B[1;32m     10\u001B[0m \u001B[38;5;66;03m# These are imported as part of the API\u001B[39;00m\n\u001B[0;32m---> 11\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mthinc\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mapi\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m prefer_gpu, require_gpu, require_cpu  \u001B[38;5;66;03m# noqa: F401\u001B[39;00m\n\u001B[1;32m     12\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mthinc\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mapi\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Config\n\u001B[1;32m     14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m pipeline  \u001B[38;5;66;03m# noqa: F401\u001B[39;00m\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/thinc/api.py:2\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconfig\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Config, registry, ConfigValidationError\n\u001B[0;32m----> 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01minitializers\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m normal_init, uniform_init, glorot_uniform_init, zero_init\n\u001B[1;32m      3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01minitializers\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m configure_normal_init\n\u001B[1;32m      4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mloss\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CategoricalCrossentropy, L2Distance, CosineDistance\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/thinc/initializers.py:4\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtyping\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Callable, cast\n\u001B[1;32m      2\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mnumpy\u001B[39;00m\n\u001B[0;32m----> 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Ops\n\u001B[1;32m      5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconfig\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m registry\n\u001B[1;32m      6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtypes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m FloatsXd, Shape\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/thinc/backends/__init__.py:7\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mcontextvars\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ContextVar\n\u001B[1;32m      5\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mthreading\u001B[39;00m\n\u001B[0;32m----> 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mops\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Ops\n\u001B[1;32m      8\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcupy_ops\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CupyOps, has_cupy\n\u001B[1;32m      9\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mnumpy_ops\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m NumpyOps\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/thinc/backends/ops.py:13\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     11\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtypes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m FloatsXd, Ints1d, Ints2d, Ints3d, Ints4d, IntsXd, _Floats\n\u001B[1;32m     12\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtypes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m DeviceTypes, Generator, Padded, Batchable, SizedGenerator\n\u001B[0;32m---> 13\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutil\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m get_array_module, is_xp_array, to_numpy\n\u001B[1;32m     16\u001B[0m ArrayT \u001B[38;5;241m=\u001B[39m TypeVar(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mArrayT\u001B[39m\u001B[38;5;124m\"\u001B[39m, bound\u001B[38;5;241m=\u001B[39mArrayXd)\n\u001B[1;32m     17\u001B[0m FloatsT \u001B[38;5;241m=\u001B[39m TypeVar(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFloatsT\u001B[39m\u001B[38;5;124m\"\u001B[39m, bound\u001B[38;5;241m=\u001B[39m_Floats)\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/thinc/util.py:29\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     25\u001B[0m     has_cupy \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m     28\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:  \u001B[38;5;66;03m# pragma: no cover\u001B[39;00m\n\u001B[0;32m---> 29\u001B[0m     \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\n\u001B[1;32m     30\u001B[0m     \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m tensor\n\u001B[1;32m     31\u001B[0m     \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdlpack\u001B[39;00m\n",
      "File \u001B[0;32m/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/torch/__init__.py:202\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m    200\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m USE_GLOBAL_DEPS:\n\u001B[1;32m    201\u001B[0m         _load_global_deps()\n\u001B[0;32m--> 202\u001B[0m     \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_C\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;241m*\u001B[39m  \u001B[38;5;66;03m# noqa: F403\u001B[39;00m\n\u001B[1;32m    204\u001B[0m \u001B[38;5;66;03m# Appease the type checker; ordinarily this binding is inserted by the\u001B[39;00m\n\u001B[1;32m    205\u001B[0m \u001B[38;5;66;03m# torch._C module initialization code in C\u001B[39;00m\n\u001B[1;32m    206\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m TYPE_CHECKING:\n",
      "\u001B[0;31mValueError\u001B[0m: module functions cannot set METH_CLASS or METH_STATIC"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "import os\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import spacy\n",
    "from lxml import etree\n",
    "from lxml.etree import XMLSyntaxError\n",
    "from spacy import Language\n",
    "from spacy_langdetect import LanguageDetector\n",
    "\n",
    "\n",
    "@Language.factory('language_detector')\n",
    "def language_detector(nlp, name):\n",
    "    return LanguageDetector()\n",
    "\n",
    "\n",
    "NLP = spacy.load('de_core_news_sm')\n",
    "NLP.add_pipe('language_detector')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "NAMESPACE: dict = {'tei': 'http://www.tei-c.org/ns/1.0'}\n",
    "SAVINGS_PATH: str = '../data/plain-text/'\n",
    "\n",
    "if not os.path.exists(SAVINGS_PATH):\n",
    "    os.makedirs(SAVINGS_PATH)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def get_letter_text(path: str ='../data/xml/*.xml') -> None:\n",
    "    \"\"\"Function for extracting and cleaning text from the XML/TEI letter files.\n",
    "\n",
    "    :param path: File path given as string.\n",
    "    :return: None.\n",
    "    \"\"\"\n",
    "    for file in glob.glob(pathname=path)[:]:\n",
    "        try:\n",
    "            tree = etree.parse(file)\n",
    "\n",
    "            letter_output: str = \"\"\n",
    "\n",
    "            for text in tree.xpath('//tei:text//tei:body//text()', namespaces=NAMESPACE):\n",
    "                text = text.lower()\n",
    "                # Remove any expressions of type \"[HERE NUMBER]\" from the text.\n",
    "                if re.match(r'.*\\[.*].*', text):\n",
    "                    letter_output += re.sub(r'\\[.*]', '', text) + ' '\n",
    "                # Remove the given statements from the text as they are not needed for performing SA.\n",
    "                elif 'beyliegendes von' or 'paginierung des editors' in text:\n",
    "                    letter_output += re.sub(r'beyliegendes von|paginierung des editors', '', text) + ' '\n",
    "                # Remove any dates (except for dates in words, e.g. \"14. Februar 1833\").\n",
    "                else:\n",
    "                    letter_output += re.sub(r'^(0[1-9]|[12][0-9]|3[01])[.](0[1-9]|1[012])[.](19|20)[0-9]{2}', ' ',\n",
    "                                            text) + ' '\n",
    "\n",
    "            # Remove unnecessary whitespace and any further characters that are neither alphanumerical nor (singular) whitespace.\n",
    "            letter_output = re.sub('\\s{2,}', ' ', letter_output.lstrip())\n",
    "            letter_output = re.sub('[^\\w\\s\\d]|_', '', letter_output)\n",
    "\n",
    "            \"\"\"Using spaCy's available pipe for language detection to store only German letters. I also tried out to do this before preprocessing the texts but this lead to some major issues.\n",
    "            \"\"\"\n",
    "            doc = NLP(letter_output)\n",
    "            if doc._.language['language'] == 'de':\n",
    "                with open(SAVINGS_PATH + re.sub(r'\\.xml', '.txt', os.path.basename(file)), 'w') as f_tbw:\n",
    "                    f_tbw.write(letter_output)\n",
    "\n",
    "        except XMLSyntaxError:\n",
    "            pass\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "get_letter_text()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from textblob_de import TextBlobDE\n",
    "from textblob_de import sentiments"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "sentiment_dict: dict = {}\n",
    "\n",
    "for file in glob.glob('../data/plain-text/*.txt')[:50]:\n",
    "    with open(file, 'r') as f:\n",
    "        text = f.read()\n",
    "        letter_blob = TextBlobDE(text)\n",
    "        #letter_blob = TextBlobDE(' '.join(letter_blob.words.lemmatize()))\n",
    "        #print(letter_blob.words.lemmatize())\n",
    "        sentiment_dict[os.path.basename(file)] = letter_blob.sentiment\n",
    "\n",
    "\n",
    "s_df = pd.DataFrame.from_dict(sentiment_dict).transpose()\n",
    "s_df.index.names = ['filename']\n",
    "s_df.rename(columns={0: 'polarity', 1: 'subjectivity'}, inplace=True)\n",
    "print(s_df)\n",
    "#s_df.to_csv('../data/retrieved/sentiments.csv')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}