brat2corpus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\"\"\"\n",
    "return annotationDict{\n",
    "    <start_index>: {\n",
    "        string: <named entity string>,\n",
    "        end_index: <end_index>,\n",
    "        tag: <named entity tag>\n",
    "    },\n",
    "    ...\n",
    "}\n",
    "\"\"\"\n",
    "def get_annotations(annotationFileName):\n",
    "    annDict = {}\n",
    "    with open(annotationFileName) as f:\n",
    "        for line in f:\n",
    "            parts = re.split(\" |\\t\", line.strip())\n",
    "            annDict[int(parts[2])] = {\n",
    "                'string': \" \".join(parts[4:]),\n",
    "                'end_index': int(parts[3]),\n",
    "                'tag': parts[1]\n",
    "            }\n",
    "            \n",
    "    return annDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_line(token, tag):\n",
    "    return token + \"\\t\" + tag + \"\\n\"\n",
    "\n",
    "def write_empty_line(file):\n",
    "    with open(file, \"a+\") as f:\n",
    "        f.write(\"\\n\")\n",
    "        \n",
    "def write_tokens(expression, tag, file):\n",
    "    with open(file, \"a+\") as f:\n",
    "        tokens = expression.strip().split(\" \")\n",
    "        if len(tokens) == 1 and len(tokens[0]) > 0:\n",
    "            f.write(create_line(tokens[0], tag))\n",
    "            return\n",
    "        for i, j in enumerate(tokens):\n",
    "            if len(j) == 0:\n",
    "                continue\n",
    "            if i == 0:\n",
    "                f.write(create_line(j, \"B-\"+tag if tag != \"O\" else tag))\n",
    "            else:\n",
    "                f.write(create_line(j, \"I-\"+tag if tag != \"O\" else tag))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def handle_line(line, start_index, output_file, annotations):\n",
    "    delimiters = set([\".\", \"?\", \"!\", \";\", \":\", \"\\\"\", \"\\n\", \",\"])\n",
    "    in_chunk = False\n",
    "    endi = None\n",
    "    starti = 0\n",
    "    tag = \"O\"\n",
    "    for i, c in enumerate(line):\n",
    "        if not in_chunk and annotations.__contains__(start_index+i):\n",
    "            if i - starti > 0:\n",
    "                write_tokens(line[starti:i], tag, output_file)\n",
    "            in_chunk = True\n",
    "            starti = i\n",
    "            endi = annotations.get(start_index+i).get('end_index')\n",
    "            tag = annotations.get(start_index+i).get('tag')\n",
    "        if in_chunk and (start_index+i == endi):\n",
    "            write_tokens(line[starti:i], tag, output_file)\n",
    "            in_chunk = False\n",
    "            endi = None\n",
    "            starti = i+1\n",
    "            tag = \"O\"\n",
    "        if in_chunk and c in delimiters:\n",
    "            write_tokens(line[starti:i], tag, output_file)\n",
    "            in_chunk = False\n",
    "            endi = None\n",
    "            starti = i+1\n",
    "            tag = \"O\"\n",
    "        if not in_chunk and c in delimiters:\n",
    "            write_tokens(line[starti:i], tag, output_file)\n",
    "            endi = None\n",
    "            starti = i+1\n",
    "            tag = \"O\"\n",
    "        if c in delimiters and c != \"\\n\":\n",
    "            write_tokens(c, \"O\", output_file)\n",
    "            \n",
    "def create_corpus(input_file, annotations, en_output_file, ewo_output_file):\n",
    "    current_index = 0\n",
    "    with open(input_file) as in_file:\n",
    "        line = in_file.readline()\n",
    "        current_index += len(line)\n",
    "        line = in_file.readline()\n",
    "        current_index += len(line)\n",
    "        line = in_file.readline()\n",
    "        while line:\n",
    "            verset = line.strip()\n",
    "            write_empty_line(en_output_file)\n",
    "            write_empty_line(ewo_output_file)\n",
    "            current_index += len(line)\n",
    "            line = in_file.readline()\n",
    "            while line != \"\\n\":\n",
    "                handle_line(line, current_index, en_output_file, annotations)\n",
    "                current_index += len(line)\n",
    "                line = in_file.readline()\n",
    "            current_index += len(line)\n",
    "            line = in_file.readline()\n",
    "            while line and line != \"\\n\":\n",
    "                handle_line(line, current_index, ewo_output_file, annotations)\n",
    "                current_index += len(line)\n",
    "                line = in_file.readline()\n",
    "            current_index += len(line)\n",
    "            line = in_file.readline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acts1-en-ewo.ann  Acts3-en-ewo.txt\t brat2corpus.ipynb  Luke1-en-ewo.txt\r\n",
      "Acts1-en-ewo.txt  Acts4-en-ewo.ann\t corpus-en.txt\t    Matthew1-en-ewo.ann\r\n",
      "Acts2-en-ewo.ann  Acts4-en-ewo.txt\t corpus-ewo.txt     Matthew1-en-ewo.txt\r\n",
      "Acts2-en-ewo.txt  best-model-conll.hdfs  desktop.ini\r\n",
      "Acts3-en-ewo.ann  bible_rnn_ner.ipynb\t Luke1-en-ewo.ann\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30, 102, 127, 145, 214]\n",
      "file>  Acts1-en-ewo\n",
      "file>  Acts3-en-ewo\n",
      "file>  Acts4-en-ewo\n",
      "file>  Luke1-en-ewo\n",
      "file>  Matthew1-en-ewo\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "input_file_names = [\"Acts1-en-ewo\", \"Acts3-en-ewo\", \"Acts4-en-ewo\", \"Luke1-en-ewo\", \"Matthew1-en-ewo\"]\n",
    "# input_file_names = [\"Acts2-en-ewo\"]\n",
    "en_side = \"corpus-en.txt\"\n",
    "ewo_side = \"corpus-ewo.txt\"\n",
    "print(list(annotations.keys())[:5])\n",
    "try:\n",
    "    os.remove(en_side)\n",
    "    os.remove(ewo_side)\n",
    "except:\n",
    "    pass\n",
    "for file_name in input_file_names:\n",
    "    print(\"file> \", file_name)\n",
    "    annotations = get_annotations(file_name+\".ann\")\n",
    "    create_corpus(file_name+\".txt\", annotations, en_side, ewo_side)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}