02_process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load bibles and process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_path = './'\n",
    "\n",
    "struct = project_path + '00_structures/'\n",
    "lang_list_fn = struct + 'languages_versions.xml'\n",
    "book_list_fn = struct + 'book_list.xml'\n",
    "titles_list_fn = struct + 'titles.xml'\n",
    "btitles_list_fn = struct + 'book_titles.xml'\n",
    "pars_list_fn   = struct + 'paragraphs.xml'\n",
    "\n",
    "# Input path\n",
    "csv_path = project_path + '01_input/'\n",
    "# Output paths\n",
    "tex_path  = project_path + '02_outputs/latex/'\n",
    "html_path = project_path + '02_outputs/html/'\n",
    "osis_path = project_path + '02_outputs/osis/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load info\n",
    "print('Loading structural data:')\n",
    "\n",
    "print('  - List of books in the Bible with numbers of chapters')\n",
    "books = pd.read_xml(book_list_fn)\n",
    "\n",
    "print('  - Available translations')\n",
    "trans = pd.read_xml(lang_list_fn, attrs_only=True)\n",
    "\n",
    "print('  - Updated titles of books')\n",
    "btitles = pd.read_xml(btitles_list_fn)\n",
    "btitles = pd.melt(btitles, id_vars=['book'])\n",
    "btitles.rename(columns={\"variable\": \"lang\", \"value\": \"title\"}, inplace=True)\n",
    "\n",
    "print('  - Titles of sections')\n",
    "titles = pd.read_xml(titles_list_fn)\n",
    "titles = pd.melt(titles, id_vars=['book','chap','verse'])\n",
    "titles.rename(columns={\"variable\": \"lang\", \"value\": \"title\"}, inplace=True)\n",
    "titles['refs'] = titles['book'] + '.' + titles['chap'].astype(str) + '.' + titles['verse'].astype(str)\n",
    "\n",
    "print('  - Paragraphs before each verse')\n",
    "pars = pd.read_xml(pars_list_fn)\n",
    "pars['refs'] = pars['book'] + '.' + pars['chap'].astype(str) + '.' + pars['verse'].astype(str)\n",
    "\n",
    "print('Done...')\n",
    "\n",
    "# Show the data\n",
    "#display(books)\n",
    "#display(trans)\n",
    "#display(btitles)\n",
    "display(titles)\n",
    "#display(pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pd_to_latex(df, print_titles=True):\n",
    "    latex_str = ''\n",
    "    for index, row in df.iterrows():\n",
    "        try:\n",
    "            current_verse_num = str(int(row['verse_num']))\n",
    "        except:\n",
    "            pass\n",
    "        if(current_verse_num is None):\n",
    "            current_verse_num = str(row['verse_num'])\n",
    "        # Add a paragraph (if necessary)\n",
    "        current_verse_ref = row['book_abbr'] + '.' + str(row['chapter_num']) + '.' + current_verse_num\n",
    "        if(current_verse_ref in pars['refs'].values):\n",
    "            latex_str += '\\n\\n'\n",
    "            \n",
    "        # Add name of the book\n",
    "        # TODO: Don't insert chapter markers at v1, but at the first paragraph break after that\n",
    "        if((row['chapter_num'] == 1) & (row['verse_num'] == 1)):\n",
    "            book_name = row['book_name']\n",
    "            # Check if the name of the current book needs to be changed/updated\n",
    "            if((row['book_abbr'] in list(btitles['book'])) & (row['lang'] in list(btitles['lang']))):\n",
    "                book_name = btitles.loc[(btitles['book'] == row['book_abbr']) & (btitles['lang'] == row['lang']), 'title'].values[0]\n",
    "            latex_str += '\\\\nonumchapter{' + str(book_name) + '}\\n\\n'\n",
    "            \n",
    "        # Add a section title (if necessary)\n",
    "        current_title = titles.loc[(titles['refs'] == current_verse_ref) & (titles['lang'] == row['lang']),'title'].values\n",
    "        if((current_title.size > 0) & (print_titles == True)):\n",
    "            if(current_title[0] != None): # Prevents a crash if the titles haven't been translated\n",
    "                latex_str += '\\n\\n\\\\nonumsection{' + current_title[0] + '}\\n\\n'\n",
    "            \n",
    "        # Add the chapter number\n",
    "        #-----------------------\n",
    "        # Only add the chapter number as a large capital if it's at the beginning of a paragraph\n",
    "        if((row['verse_num'] == 1) & (current_verse_ref in pars['refs'].values) & (print_titles == True)):\n",
    "            latex_str += '\\\\bibchap{' + str(row['chapter_num']) + '}'\n",
    "            latex_str += '\\\\bibverse{' + current_verse_num + '}' + str(row['verse_text']) + ' '\n",
    "        # If it's not at the beginning of a paragraph, make the chapter number small\n",
    "        if((row['verse_num'] == 1) & (current_verse_ref not in pars['refs'].values) & (print_titles == True)):\n",
    "            latex_str += '\\\\bibverse{' + str(row['chapter_num']) + '.' + current_verse_num + '}' + str(row['verse_text']) + ' '\n",
    "        # If the titles aren't to be printed, then the chapter numbers will always be a large number\n",
    "        if((row['verse_num'] == 1) & (print_titles == False)):\n",
    "            latex_str += '\\n\\n\\\\bibchap{' + str(row['chapter_num']) + '}'\n",
    "            latex_str += '\\\\bibverse{' + current_verse_num + '}' + str(row['verse_text']) + ' '\n",
    "        # All other cases, just add the verse number\n",
    "        if(row['verse_num'] > 1):\n",
    "            latex_str += '\\\\bibverse{' + current_verse_num + '}' + str(row['verse_text']) + ' '\n",
    "        # Old\n",
    "        #if(row['verse_num'] == 1):\n",
    "        #    latex_str += '\\\\bibchap{' + str(row['chapter_num']) + '}'\n",
    "        #latex_str += '\\\\bibverse{' + current_verse_num + '}' + str(row['verse_text']) + ' '\n",
    "        \n",
    "    # replace double spaces\n",
    "    latex_str = ' '.join(latex_str.split(' '))\n",
    "    \n",
    "    return(latex_str)\n",
    "\n",
    "def latex_to_file(latex_str, fn):\n",
    "    with open(fn, 'w', encoding='utf-8') as f:\n",
    "        f.write(latex_str)\n",
    "    pass\n",
    "\n",
    "def process_version(version):\n",
    "    print('Converting books to latex/html:')\n",
    "    fn_list = sorted(glob.glob(csv_path + '*' + version + '/*', recursive=True))\n",
    "\n",
    "    # Iterate over the files in the list\n",
    "    for fn_i, fn in enumerate(fn_list):\n",
    "        file_arr = fn.split('\\\\')\n",
    "        print('  - ' + file_arr[1] + '/' + file_arr[2][:-4])\n",
    "        # Read file\n",
    "        df = pd.read_csv(fn)\n",
    "        # Convert to latex\n",
    "        book_tex = pd_to_latex(df, print_titles=False)\n",
    "        # Save\n",
    "        if not os.path.exists(tex_path + file_arr[1] + '/'):\n",
    "            os.makedirs(tex_path + file_arr[1] + '/')\n",
    "        latex_to_file(book_tex, tex_path + file_arr[1] + '/' + file_arr[2][:-4] + '.tex')\n",
    "    print('Done...')\n",
    "    pass\n",
    "\n",
    "def process_all():\n",
    "    fn_list = sorted(glob.glob(csv_path + '*/', recursive=False))\n",
    "    version_list = [] # Just for debug purposes\n",
    "    for version in fn_list:\n",
    "        current_version = version.split('\\\\')[1].split('_')[1]\n",
    "        process_version(current_version)\n",
    "        version_list.append(current_version) # Just for debug purposes\n",
    "    #print('Processed versions:', version_list) # Just for debug purposes\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process 1 single version:\n",
    "#process_version(version)\n",
    "process_version('ASV')\n",
    "#process_version('NASB1995')\n",
    "\n",
    "# Process all\n",
    "#process_all()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tp",
   "language": "python",
   "name": "tp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}