create_class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "from typing import NamedTuple, Iterable\n",
    "from dataclasses import dataclass\n",
    "\n",
    "URN_PREFIX = 'URN:NBN:'\n",
    "    \n",
    "@dataclass\n",
    "class AltoString:\n",
    "    id: str\n",
    "    token: str    \n",
    "        \n",
    "@dataclass\n",
    "class TextLine:\n",
    "    id: str\n",
    "    strings: Iterable[AltoString]\n",
    "    \n",
    "@dataclass\n",
    "class TextBlock:\n",
    "    id: str\n",
    "    text_lines: str\n",
    "    \n",
    "@dataclass\n",
    "class ComposedBlock:\n",
    "    id: str\n",
    "    text_blocks: Iterable[TextBlock]\n",
    "    \n",
    "@dataclass\n",
    "class PageData:\n",
    "    composed_blocks: Iterable[ComposedBlock]\n",
    "    \n",
    "\n",
    "class Book:\n",
    "    def __init__(self, path: str):\n",
    "        self.path = path\n",
    "        self.pages = []\n",
    "        self.parse()\n",
    "        \n",
    "    @property  \n",
    "    def urn_suffix(self):\n",
    "        return os.path.basename(self.path).split('.')[0]  \n",
    "    \n",
    "    @property\n",
    "    def urn(self):\n",
    "        return URN_PREFIX + self.urn_suffix  \n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'<Book {self.urn}>'\n",
    "        \n",
    "    def parse(self):\n",
    "        for path, _, files in os.walk(self.path):\n",
    "            for file in tqdm(files):\n",
    "                if file.endswith('.xml'):\n",
    "                    self.pages.append(Page(os.path.join(path, file)))\n",
    "                    \n",
    "    def print_pages(self):\n",
    "        for page in self.pages:\n",
    "            print(page.path)\n",
    "            \n",
    "    def __getitem__(self, slice):\n",
    "        return self.pages[slice]\n",
    "    \n",
    "    def export_to_txt(self, path: str):\n",
    "        os.makedirs(path, exist_ok=True)        \n",
    "        for page in self.pages:\n",
    "            with open(os.path.join(path, page.urn_suffix + '.txt'), 'w') as f:\n",
    "                for line in page.text:\n",
    "                    f.write(line)\n",
    "        \n",
    "        \n",
    "        \n",
    "class Page:\n",
    "    def __init__(self, path: str):\n",
    "        self.path = path\n",
    "        self.components = []\n",
    "        self.parse()\n",
    "    \n",
    "    @property  \n",
    "    def urn_suffix(self):\n",
    "        return os.path.basename(self.path).split('.')[0]  \n",
    "    \n",
    "    @property\n",
    "    def urn(self):\n",
    "        return URN_PREFIX + self.urn_suffix      \n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'<Page {self.urn}>'\n",
    "\n",
    "    def parse(self):\n",
    "         # Parse the ALTO XML file\n",
    "        tree = etree.parse(self.path)\n",
    "        namespaces = {'alto': 'http://www.loc.gov/standards/alto/ns-v3#'}\n",
    "        \n",
    "        data = PageData([])\n",
    "        self.components.append(data)\n",
    "        \n",
    "        \n",
    "        composed_blocks = tree.xpath('.//alto:ComposedBlock', namespaces=namespaces)\n",
    "        \n",
    "        for composed_block in composed_blocks:\n",
    "            id = composed_block.get('ID')\n",
    "            target_composed_block = ComposedBlock(id, [])\n",
    "            data.composed_blocks.append(target_composed_block)\n",
    "                \n",
    "            blocks = composed_block.xpath('./alto:TextBlock', namespaces=namespaces)\n",
    "            for block in blocks:\n",
    "                target_block = TextBlock(block.get('ID'), [])\n",
    "                target_composed_block.text_blocks.append(target_block)\n",
    "                \n",
    "                lines = block.xpath('./alto:TextLine', namespaces=namespaces)\n",
    "                \n",
    "                for line in lines:\n",
    "                    target_line = TextLine(line.get('ID'), [])\n",
    "                    target_block.text_lines.append(target_line)\n",
    "                    \n",
    "                    strings = line.xpath('./alto:String', namespaces=namespaces)\n",
    "                    strings = [AltoString(s.get('ID'), s.get('CONTENT')) for s in strings]\n",
    "                    target_line.strings = strings\n",
    "                    \n",
    "                    \n",
    "    def print_text(self):\n",
    "        for data in self.components:\n",
    "            for composed_block in data.composed_blocks:\n",
    "                for block in composed_block.text_blocks:\n",
    "                    for line in block.text_lines:\n",
    "                        print(\" \".join([t.token for t in line.strings]))\n",
    "                print()\n",
    "            print()\n",
    "    \n",
    "    @property    \n",
    "    def text(self):\n",
    "        for data in self.components:\n",
    "            for composed_block in data.composed_blocks:\n",
    "                for block in composed_block.text_blocks:\n",
    "                    for line in block.text_lines:\n",
    "                        yield \" \".join([t.token for t in line.strings])\n",
    "                    yield \"\\n\"\n",
    "                yield \"\\n\"\n",
    "            yield \"\\n\"\n",
    "                    \n",
    "                    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0265aee1bb314df9bdc376ebf16d74ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a4745bef9684e76ad51dc6d6d75faf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/203 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_path = \"data/test/no-nb_digibok_2006080900007\"\n",
    "\n",
    "test = Book(test_path)\n",
    "# test.print_pages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Book URN:NBN:no-nb_digibok_2006080900007>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.export_to_txt('data/test_txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Book' object has no attribute 'urn'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[87], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murn\u001b[49m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Book' object has no attribute 'urn'"
     ]
    }
   ],
   "source": [
    "test.urn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'print_text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[118], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpages\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprint_text\u001b[49m()\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'print_text'"
     ]
    }
   ],
   "source": [
    "test.pages[2:3].print_text()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "re-ocr-resource-RqHojbl_-py3.10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}