01_bible_download.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download bibles from Bible.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "import copy\n",
    "import json\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_path = './'\n",
    "\n",
    "struct = project_path + '00_structures/'\n",
    "lang_list_fn = struct + 'languages_versions.xml'\n",
    "book_list_fn = struct + 'book_list.xml'\n",
    "\n",
    "# Output paths\n",
    "csv_path = project_path + '01_input/'\n",
    "tex_path = project_path + '02_outputs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load info\n",
    "print('Loading structural data:')\n",
    "\n",
    "print('  - List of books in the Bible with numbers of chapters')\n",
    "books = pd.read_xml(book_list_fn)\n",
    "\n",
    "print('  - Available translations')\n",
    "trans = pd.read_xml(lang_list_fn, attrs_only=True)\n",
    "\n",
    "print('Done...')\n",
    "\n",
    "# Show the data\n",
    "#display(books)\n",
    "#display(trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Downloads a bible chapter, using the following inputs:\n",
    "- version: Bible version as named at Bible.com (e.g., ASV)\n",
    "- vnum: Version number, e.g., for ASV it's 12\n",
    "- book: Book acronym, e.g. GEN\n",
    "- chapter: chapter number\n",
    "- underscore (True/False): German bibles sometimes use an _ separator between book and chapter\n",
    "'''\n",
    "def download_chapter(version, vnum, book, chapter, underscore=False):\n",
    "    # Create download link\n",
    "    download_link = 'https://www.bible.com/bible/' + str(vnum)+ '/' + book + '.' + str(chapter) + '.' + version\n",
    "    if(underscore):\n",
    "        download_link = 'https://www.bible.com/bible/' + str(vnum)+ '/' + book + '.' + str(chapter) + '_1.' + version\n",
    "    # Download the text\n",
    "    html_doc = requests.get(download_link)\n",
    "    # Extract into Pandas df\n",
    "    df = extract_verses(html_doc, version, underscore=underscore)\n",
    "    return(df)\n",
    "\n",
    "def extract_verses(html_doc, version, underscore=False, silent=False):\n",
    "    soup = BeautifulSoup(html_doc.text, 'html.parser')\n",
    "    # Initialize empty list to store verse data\n",
    "    verse_data = []\n",
    "    # Find the div element with class 'chapter'\n",
    "    chapter = copy.copy(soup).find(\"div\", {\"class\": \"chapter\"})\n",
    "    book_abbr = chapter['data-usfm'].split(\".\")[0]\n",
    "    try:\n",
    "        chapter_num = int(chapter.find('div', {'class': 'label'}).text) # Most bibles use numbers for chapters\n",
    "    except:\n",
    "        chapter_num = chapter.find('div', {'class': 'label'}).text # Except Hebrew\n",
    "    if(underscore & (~isinstance(chapter_num, int))): # Remove 'Kapitel'\n",
    "        chapter_num = int(chapter_num.split(' ')[1])\n",
    "    # Find the book name\n",
    "    info = copy.copy(soup).find(\"script\", {\"type\": \"application/ld+json\"}).string\n",
    "    code = json.loads(info)\n",
    "    book_name = code['itemListElement'][2]['item']['name'].rsplit(' ', 1)[0]\n",
    "    if(underscore): # Remove 'Kapitel'\n",
    "        book_name = ' '.join(book_name.split(' ')[:-1])\n",
    "    # Language\n",
    "    version_div = soup.find(\"div\", {\"class\": \"version\"})\n",
    "    if version_div:\n",
    "        language = version_div.get(\"data-iso6393\")\n",
    "    # Status\n",
    "    if(not silent):\n",
    "        print('      -', book_abbr, chapter_num, '('+ book_name +')')\n",
    "    # Find all verse elements\n",
    "    verses = copy.copy(chapter).find_all('span', {'class': 'verse'})\n",
    "    # Iterate over each verse\n",
    "    prev_verse = 1\n",
    "    for verse in verses:\n",
    "        # In some cases, multiple verses are referenced as text\n",
    "        verse_num = ''\n",
    "        try:\n",
    "            verse_num = int(copy.copy(verse).find('span', {'class': 'label'}).text)\n",
    "        except:\n",
    "            pass\n",
    "        if(verse_num is None):\n",
    "            verse_num = copy.copy(verse).find('span', {'class': 'label'}).text\n",
    "        if(verse_num == ''):\n",
    "            verse_num = prev_verse\n",
    "        prev_verse = verse_num\n",
    "        # Replace the divine name with capitals\n",
    "        try:\n",
    "            # English uses class nd\n",
    "            divineName = verse.find('span', {'class':'nd'}).string\n",
    "            verse.find('span', {'class':'nd'}).string.replace_with('\\\\textsc{' + divineName + '}')\n",
    "        except:\n",
    "            pass\n",
    "        try:\n",
    "            # underscore uses class sc\n",
    "            divineName = verse.find('span', {'class':'sc'}).string\n",
    "            verse.find('span', {'class':'sc'}).string.replace_with('\\\\textsc{' + divineName + '}')\n",
    "        except:\n",
    "            pass\n",
    "        # Keep only relevant text data\n",
    "        verse_text = ''\n",
    "        for match in verse.find_all('span', {'class':'content'}):\n",
    "            verse_text += match.get_text()\n",
    "        verse_text = verse_text.strip()\n",
    "        # Append verse data to list\n",
    "        verse_data.append({'lang': language,\n",
    "                           'version': version,\n",
    "                           'book_name': book_name,\n",
    "                           'book_abbr': book_abbr,\n",
    "                           'chapter_num': chapter_num,\n",
    "                           'verse_num': verse_num,\n",
    "                           'verse_text': verse_text})\n",
    "    # Create a pandas dataframe from the list of verse data\n",
    "    df = pd.DataFrame(verse_data)\n",
    "    # Remove extra line breaks etc\n",
    "    df = df.groupby(['lang','version','book_name','book_abbr','chapter_num','verse_num'])['verse_text'].apply(lambda x: ' '.join(x)).reset_index()\n",
    "    return(df)\n",
    "\n",
    "def download_book(current_version, current_book):\n",
    "    current_lang = trans.loc[trans['version'] == current_version, 'lang'].values[0]\n",
    "    current_vnum = trans.loc[trans['version'] == current_version, 'vnum'].values[0]\n",
    "    current_lcode = trans.loc[trans['version'] == current_version, 'lcode'].values[0]\n",
    "    current_order = books.loc[books['book'] == current_book, 'order'].values[0]\n",
    "    current_nb_chaps = books.loc[books['book'] == current_book, 'chapters'].values[0]\n",
    "    print('  - ' + current_lang + ' - ' + current_version)\n",
    "    print('    - ' + current_book)\n",
    "    # Some (German) bibles use a different format for the download link\n",
    "    if(current_version in ['SCHL2000', 'NGU2011']):\n",
    "        underscore = True\n",
    "    else:\n",
    "        underscore = False\n",
    "    chapter_df = []\n",
    "    for chap in range(1,current_nb_chaps+1):\n",
    "        temp = download_chapter(current_version, current_vnum, current_book, chap, underscore=underscore)\n",
    "        chapter_df.append(temp)\n",
    "        time.sleep(5) # This website will not let us download quickly (protection mechanism). Pause between downloads...\n",
    "        pass\n",
    "    print('      - Concatenate book df...')\n",
    "    book_df = pd.concat(chapter_df)\n",
    "    print('      - Save to csv file: ', end = '')\n",
    "    book_csv_fn = '{:02d}'.format(current_order) + '_' + current_book + '.csv'\n",
    "    folder_name = current_lcode + '_' + current_version\n",
    "    print(csv_path + folder_name + '/' + book_csv_fn)\n",
    "    if not os.path.exists(csv_path + '/' + folder_name):\n",
    "        os.makedirs(csv_path + '/' + folder_name)\n",
    "    book_df.to_csv(csv_path + '/' + folder_name + '/' + book_csv_fn, index=False)\n",
    "    print('Done...')\n",
    "    pass\n",
    "\n",
    "def download_version(current_version):\n",
    "    print('Downloading:')\n",
    "    # Cycle through and download\n",
    "    for index, book in books.iterrows():\n",
    "        download_book(current_version, book['book'])\n",
    "    pass\n",
    "\n",
    "def download_all():\n",
    "    # Cycle through and download\n",
    "    for index, tr in trans.iterrows():\n",
    "        download_version(tr['version'])\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download everything\n",
    "#download_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download a specific version\n",
    "download_version('ASV')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download a specific book\n",
    "#download_book('ASV', 'PSA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tp",
   "language": "python",
   "name": "tp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}