create_lexicon_from_ene.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create lexicon of term from ENE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from tqdm import tqdm\n",
    "from lxml import etree\n",
    "import json\n",
    "from utils_functions import get_term_occurrences_from_ene, load_json_dict, save_json\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Create the lexicon from terms embedded in ENE from Perdido XML-TEI\n",
    "\n",
    "\\<rs type=\"place\">\\<term type=\"place\">"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Corpus Traitement Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/lmoncla/Documents/Data/Corpus/Choucas/Perdido'\n",
    "output_filename = './output/Traitement_Texte_pivot_lexicon'\n",
    "\n",
    "words_TT = []\n",
    "for doc in tqdm(sorted(os.listdir(path))):\n",
    "    filename = os.path.join(path, doc, doc+'.xml') # version Traitements_Texte\n",
    "    words_TT.extend(get_term_occurrences_from_ene(filename))\n",
    "\n",
    "# list to dict with frequency\n",
    "frequency_dict_geo_TT = load_json_dict(words_TT)\n",
    "print('Size of the lexicon', len(frequency_dict_geo_TT))\n",
    "\n",
    "#save the dict in file\n",
    "save_json(output_filename + '.json', frequency_dict_geo_TT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Corpus Visorando"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/lmoncla/Documents/Data/Corpus/Visorando/Perdido'\n",
    "output_filename = './output/Visorando_pivot_lexicon'\n",
    "\n",
    "words_viso = []\n",
    "for doc in tqdm(sorted(os.listdir(path))):\n",
    "    filename = os.path.join(path, doc[:-4]+'.xml') # version visorando\n",
    "    words_viso.extend(get_term_occurrences_from_ene(filename))\n",
    "\n",
    "# list to dict with frequency\n",
    "frequency_dict_geo_viso = {value: words_viso.count(value) for value in words_viso}\n",
    "print('Size of the lexicon', len(frequency_dict_geo_viso))\n",
    "\n",
    "#save the dict in file\n",
    "save_json(output_filename + '.json', frequency_dict_geo_viso)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 Lexique TT - Viso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [item for item in frequency_dict_geo_viso.keys() if item not in frequency_dict_geo_TT.keys()]\n",
    "print(len(words), words)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 Lexicons preview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in frequency_dict_geo_viso.items():\n",
    "    print(k, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the dict from file\n",
    "# only necessery if you want to run the next section and not the previous one (adapt output_filename)\n",
    "\n",
    "frequency_dict_geo_viso = load_json_dict(output_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List of files with rs in term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/lmoncla/Documents/Data/Corpus/Visorando/Perdido'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = []\n",
    "for doc in tqdm(sorted(os.listdir(path))):\n",
    "    filename = os.path.join(path, doc+'.xml') \n",
    "    if os.path.exists(filename):\n",
    "        tree = etree.parse(filename)\n",
    "\n",
    "\n",
    "        for term in tree.xpath('.//term//rs'):\n",
    "            print(filename)\n",
    "            phrase = ''\n",
    "            for w in term.xpath('.//w'):\n",
    "                phrase += w.text.lower() + ' '\n",
    "        #print(phrase, end=' ')\n",
    "        #print()\n",
    "            \n",
    "            #print()\n",
    "\n",
    "        # list to dict with frequency\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}