CRF_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0dd49e29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting dodfminer\n",
      "  Downloading dodfminer-1.4.8-py3-none-any.whl (40.8 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 MB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0mm\n",
      "\u001b[?25hCollecting beautifulsoup4==4.12.0\n",
      "  Downloading beautifulsoup4-4.12.0-py3-none-any.whl (132 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.2/132.2 kB\u001b[0m \u001b[31m593.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting tqdm==4.65.0\n",
      "  Downloading tqdm-4.65.0-py3-none-any.whl (77 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.1/77.1 kB\u001b[0m \u001b[31m315.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mMB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting chardet==5.1.0\n",
      "  Downloading chardet-5.1.0-py3-none-any.whl (199 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.1/199.1 kB\u001b[0m \u001b[31m514.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting numpy==1.24.2\n",
      "  Downloading numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting certifi==2022.12.7\n",
      "  Downloading certifi-2022.12.7-py3-none-any.whl (155 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m155.3/155.3 kB\u001b[0m \u001b[31m495.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting pytz==2023.3\n",
      "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m502.3/502.3 kB\u001b[0m \u001b[31m703.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting PyMuPDF==1.21.1\n",
      "  Downloading PyMuPDF-1.21.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (14.0 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.0/14.0 MB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting coveralls==3.3.1\n",
      "  Downloading coveralls-3.3.1-py2.py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: pandas==1.5.3 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from dodfminer) (1.5.3)\n",
      "Collecting lxml==4.9.2\n",
      "  Downloading lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (7.1 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting pytest-cov==4.0.0\n",
      "  Downloading pytest_cov-4.0.0-py3-none-any.whl (21 kB)\n",
      "Collecting pylint==2.17.1\n",
      "  Downloading pylint-2.17.1-py3-none-any.whl (535 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m535.8/535.8 kB\u001b[0m \u001b[31m875.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting idna==3.4\n",
      "  Downloading idna-3.4-py3-none-any.whl (61 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.5/61.5 kB\u001b[0m \u001b[31m187.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting requests==2.28.2\n",
      "  Downloading requests-2.28.2-py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m162.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting pytest-mock==3.10.0\n",
      "  Downloading pytest_mock-3.10.0-py3-none-any.whl (9.3 kB)\n",
      "Collecting requests-mock==1.10.0\n",
      "  Downloading requests_mock-1.10.0-py2.py3-none-any.whl (28 kB)\n",
      "Collecting scikit-learn==1.2.2\n",
      "  Downloading scikit_learn-1.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.6 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: sklearn-crfsuite==0.3.6 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from dodfminer) (0.3.6)\n",
      "Collecting nltk==3.8.1\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting python-dateutil==2.8.1\n",
      "  Downloading python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.2/227.2 kB\u001b[0m \u001b[31m769.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting pytest==7.2.2\n",
      "  Downloading pytest-7.2.2-py3-none-any.whl (317 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.2/317.2 kB\u001b[0m \u001b[31m875.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: joblib==1.2.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from dodfminer) (1.2.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from beautifulsoup4==4.12.0->dodfminer) (2.3.1)\n",
      "Collecting coverage!=6.0.*,!=6.1,!=6.1.1,<7.0,>=4.1\n",
      "  Downloading coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.8/212.8 kB\u001b[0m \u001b[31m583.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting docopt>=0.6.1\n",
      "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: click in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from nltk==3.8.1->dodfminer) (8.0.4)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from nltk==3.8.1->dodfminer) (2022.7.9)\n",
      "Requirement already satisfied: dill>=0.2 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (0.3.4)\n",
      "Requirement already satisfied: mccabe<0.8,>=0.6 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (0.7.0)\n",
      "Requirement already satisfied: typing-extensions>=3.10.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (4.3.0)\n",
      "Requirement already satisfied: isort<6,>=4.2.5 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (5.9.3)\n",
      "Requirement already satisfied: platformdirs>=2.2.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (2.5.2)\n",
      "Requirement already satisfied: tomlkit>=0.10.1 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (0.11.1)\n",
      "Collecting astroid<=2.17.0-dev0,>=2.15.0\n",
      "  Downloading astroid-2.15.2-py3-none-any.whl (277 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m277.9/277.9 kB\u001b[0m \u001b[31m876.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tomli>=1.1.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pylint==2.17.1->dodfminer) (2.0.1)\n",
      "Requirement already satisfied: iniconfig in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pytest==7.2.2->dodfminer) (1.1.1)\n",
      "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pytest==7.2.2->dodfminer) (1.0.0)\n",
      "Requirement already satisfied: packaging in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pytest==7.2.2->dodfminer) (21.3)\n",
      "Collecting exceptiongroup>=1.0.0rc8\n",
      "  Downloading exceptiongroup-1.1.1-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from pytest==7.2.2->dodfminer) (21.4.0)\n",
      "Collecting coverage[toml]>=5.2.1\n",
      "  Downloading coverage-7.2.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (227 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.5/227.5 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: six>=1.5 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from python-dateutil==2.8.1->dodfminer) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from requests==2.28.2->dodfminer) (2.0.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from requests==2.28.2->dodfminer) (1.26.11)\n",
      "Requirement already satisfied: scipy>=1.3.2 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from scikit-learn==1.2.2->dodfminer) (1.9.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from scikit-learn==1.2.2->dodfminer) (2.2.0)\n",
      "Requirement already satisfied: python-crfsuite>=0.8.3 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from sklearn-crfsuite==0.3.6->dodfminer) (0.9.9)\n",
      "Requirement already satisfied: tabulate in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from sklearn-crfsuite==0.3.6->dodfminer) (0.8.10)\n",
      "Requirement already satisfied: wrapt<2,>=1.11 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from astroid<=2.17.0-dev0,>=2.15.0->pylint==2.17.1->dodfminer) (1.14.1)\n",
      "Requirement already satisfied: lazy-object-proxy>=1.4.0 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from astroid<=2.17.0-dev0,>=2.15.0->pylint==2.17.1->dodfminer) (1.6.0)\n",
      "  Downloading coverage-7.2.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (227 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.0/227.0 kB\u001b[0m \u001b[31m652.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (226 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.9/226.9 kB\u001b[0m \u001b[31m756.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (226 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.9/226.9 kB\u001b[0m \u001b[31m863.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (226 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.2/226.2 kB\u001b[0m \u001b[31m830.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (225 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m225.4/225.4 kB\u001b[0m \u001b[31m627.8 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (224 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.9/224.9 kB\u001b[0m \u001b[31m688.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (224 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.3/224.3 kB\u001b[0m \u001b[31m432.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (223 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m223.9/223.9 kB\u001b[0m \u001b[31m544.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (215 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m215.6/215.6 kB\u001b[0m \u001b[31m741.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25h  Downloading coverage-7.0.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (215 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m215.5/215.5 kB\u001b[0m \u001b[31m983.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/thiagodepaulo/anaconda3/lib/python3.9/site-packages (from packaging->pytest==7.2.2->dodfminer) (3.0.9)\n",
      "Building wheels for collected packages: docopt\n",
      "  Building wheel for docopt (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13706 sha256=3fdc239bb8e551967a50c9addb8bc1a514e105f8217cac2e7e1d5de8926893a4\n",
      "  Stored in directory: /home/thiagodepaulo/.cache/pip/wheels/70/4a/46/1309fc853b8d395e60bafaf1b6df7845bdd82c95fd59dd8d2b\n",
      "Successfully built docopt\n",
      "Installing collected packages: pytz, docopt, tqdm, python-dateutil, PyMuPDF, numpy, lxml, idna, exceptiongroup, coverage, chardet, certifi, beautifulsoup4, astroid, requests, pytest, pylint, nltk, scikit-learn, requests-mock, pytest-mock, pytest-cov, coveralls, dodfminer\n",
      "  Attempting uninstall: pytz\n",
      "    Found existing installation: pytz 2022.1\n",
      "    Uninstalling pytz-2022.1:\n",
      "      Successfully uninstalled pytz-2022.1\n",
      "  Attempting uninstall: tqdm\n",
      "    Found existing installation: tqdm 4.64.1\n",
      "    Uninstalling tqdm-4.64.1:\n",
      "      Successfully uninstalled tqdm-4.64.1\n",
      "  Attempting uninstall: python-dateutil\n",
      "    Found existing installation: python-dateutil 2.8.2\n",
      "    Uninstalling python-dateutil-2.8.2:\n",
      "      Successfully uninstalled python-dateutil-2.8.2\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.21.5\n",
      "    Uninstalling numpy-1.21.5:\n",
      "      Successfully uninstalled numpy-1.21.5\n",
      "  Attempting uninstall: lxml\n",
      "    Found existing installation: lxml 4.9.1\n",
      "    Uninstalling lxml-4.9.1:\n",
      "      Successfully uninstalled lxml-4.9.1\n",
      "  Attempting uninstall: idna\n",
      "    Found existing installation: idna 3.3\n",
      "    Uninstalling idna-3.3:\n",
      "      Successfully uninstalled idna-3.3\n",
      "  Attempting uninstall: chardet\n",
      "    Found existing installation: chardet 4.0.0\n",
      "    Uninstalling chardet-4.0.0:\n",
      "      Successfully uninstalled chardet-4.0.0\n",
      "  Attempting uninstall: certifi\n",
      "    Found existing installation: certifi 2022.9.14\n",
      "    Uninstalling certifi-2022.9.14:\n",
      "      Successfully uninstalled certifi-2022.9.14\n",
      "  Attempting uninstall: beautifulsoup4\n",
      "    Found existing installation: beautifulsoup4 4.11.1\n",
      "    Uninstalling beautifulsoup4-4.11.1:\n",
      "      Successfully uninstalled beautifulsoup4-4.11.1\n",
      "  Attempting uninstall: astroid\n",
      "    Found existing installation: astroid 2.11.7\n",
      "    Uninstalling astroid-2.11.7:\n",
      "      Successfully uninstalled astroid-2.11.7\n",
      "  Attempting uninstall: requests\n",
      "    Found existing installation: requests 2.28.1\n",
      "    Uninstalling requests-2.28.1:\n",
      "      Successfully uninstalled requests-2.28.1\n",
      "  Attempting uninstall: pytest\n",
      "    Found existing installation: pytest 7.1.2\n",
      "    Uninstalling pytest-7.1.2:\n",
      "      Successfully uninstalled pytest-7.1.2\n",
      "  Attempting uninstall: pylint\n",
      "    Found existing installation: pylint 2.14.5\n",
      "    Uninstalling pylint-2.14.5:\n",
      "      Successfully uninstalled pylint-2.14.5\n",
      "  Attempting uninstall: nltk\n",
      "    Found existing installation: nltk 3.7\n",
      "    Uninstalling nltk-3.7:\n",
      "      Successfully uninstalled nltk-3.7\n",
      "  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 1.0.2\n",
      "    Uninstalling scikit-learn-1.0.2:\n",
      "      Successfully uninstalled scikit-learn-1.0.2\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "spyder 5.3.3 requires pyqt5<5.16, which is not installed.\n",
      "spyder 5.3.3 requires pyqtwebengine<5.16, which is not installed.\n",
      "daal4py 2021.6.0 requires daal==2021.4.0, which is not installed.\n",
      "anaconda-project 0.11.1 requires ruamel-yaml, which is not installed.\n",
      "numba 0.55.1 requires numpy<1.22,>=1.18, but you have numpy 1.24.2 which is incompatible.\n",
      "jupyter-client 7.3.4 requires python-dateutil>=2.8.2, but you have python-dateutil 2.8.1 which is incompatible.\n",
      "conda-repo-cli 1.0.20 requires clyent==1.2.1, but you have clyent 1.2.2 which is incompatible.\n",
      "conda-repo-cli 1.0.20 requires nbformat==5.4.0, but you have nbformat 5.5.0 which is incompatible.\n",
      "conda-repo-cli 1.0.20 requires python-dateutil==2.8.2, but you have python-dateutil 2.8.1 which is incompatible.\n",
      "conda-repo-cli 1.0.20 requires requests==2.28.1, but you have requests 2.28.2 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed PyMuPDF-1.21.1 astroid-2.15.2 beautifulsoup4-4.12.0 certifi-2022.12.7 chardet-5.1.0 coverage-6.5.0 coveralls-3.3.1 docopt-0.6.2 dodfminer-1.4.8 exceptiongroup-1.1.1 idna-3.4 lxml-4.9.2 nltk-3.8.1 numpy-1.24.2 pylint-2.17.1 pytest-7.2.2 pytest-cov-4.0.0 pytest-mock-3.10.0 python-dateutil-2.8.1 pytz-2023.3 requests-2.28.2 requests-mock-1.10.0 scikit-learn-1.2.2 tqdm-4.65.0\n"
     ]
    }
   ],
   "source": [
    "!pip install dodfminer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8fa3bab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pipeline imports\n",
    "import pandas as pd\n",
    "import nltk\n",
    "\n",
    "#Elementos do pipeline precisam herdar de TransformerMixin, sendo o ultimo (mais da direita) o Modelo que herda de BaseEstimator\n",
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "\n",
    "#Pipeline precisa ser do sklearn.pipeline\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "#Modelo usado no DODFMiner\n",
    "from sklearn_crfsuite import CRF\n",
    "\n",
    "99999999999999999999999999999999999999999999999999\n",
    "from tqdm import tqdm\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c0297cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('DODFCorpus_contratos_licitacoes_v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "995c19fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>id_dodf</th>\n",
       "      <th>tipo_rel</th>\n",
       "      <th>id_rel</th>\n",
       "      <th>anotador_rel</th>\n",
       "      <th>tipo_ent</th>\n",
       "      <th>id_ent</th>\n",
       "      <th>anotador_ent</th>\n",
       "      <th>offset</th>\n",
       "      <th>length</th>\n",
       "      <th>texto</th>\n",
       "      <th>id_ato</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>modalidade_licitacao</td>\n",
       "      <td>1</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>432851</td>\n",
       "      <td>17</td>\n",
       "      <td>PREGAO ELETRONICO</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>numero_licitacao</td>\n",
       "      <td>2</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>432872</td>\n",
       "      <td>8</td>\n",
       "      <td>116/2018</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>valor_estimado_contratacao</td>\n",
       "      <td>6</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>433432</td>\n",
       "      <td>10</td>\n",
       "      <td>767.907,37</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>data_abertura_licitacao</td>\n",
       "      <td>7</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>433599</td>\n",
       "      <td>10</td>\n",
       "      <td>20/11/2018</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>processo_gdf</td>\n",
       "      <td>490</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>433640</td>\n",
       "      <td>22</td>\n",
       "      <td>00410-00024534/2017-65</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>sistema_compras</td>\n",
       "      <td>8</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>433716</td>\n",
       "      <td>21</td>\n",
       "      <td>www.compras.df.gov.br</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>nome_responsavel</td>\n",
       "      <td>3</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>433818</td>\n",
       "      <td>25</td>\n",
       "      <td>GERARDA DA SILVA CARVALHO</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>tipo_objeto</td>\n",
       "      <td>540</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>432927</td>\n",
       "      <td>9</td>\n",
       "      <td>aquisicao</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>objeto_licitacao</td>\n",
       "      <td>542</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>432889</td>\n",
       "      <td>516</td>\n",
       "      <td>Registro de Precos visando a eventual aquisica...</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>12_22.11.2018</td>\n",
       "      <td>REL_AVISO_LICITACAO</td>\n",
       "      <td>R53</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>AVISO_LICITACAO</td>\n",
       "      <td>545</td>\n",
       "      <td>anotador_6043174</td>\n",
       "      <td>432830</td>\n",
       "      <td>1023</td>\n",
       "      <td>AVISOS DE LICITACOES\\nPREGAO ELETRONICO No 116...</td>\n",
       "      <td>12_22.11.2018-R53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1        id_dodf             tipo_rel id_rel  \\\n",
       "0           0             0  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "1           1             1  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "2           2             2  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "3           3             3  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "4           4             4  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "5           5             5  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "6           6             6  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "7           7             7  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "8           8             8  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "9           9             9  12_22.11.2018  REL_AVISO_LICITACAO    R53   \n",
       "\n",
       "       anotador_rel                    tipo_ent  id_ent      anotador_ent  \\\n",
       "0  anotador_6043174        modalidade_licitacao       1  anotador_6043174   \n",
       "1  anotador_6043174            numero_licitacao       2  anotador_6043174   \n",
       "2  anotador_6043174  valor_estimado_contratacao       6  anotador_6043174   \n",
       "3  anotador_6043174     data_abertura_licitacao       7  anotador_6043174   \n",
       "4  anotador_6043174                processo_gdf     490  anotador_6043174   \n",
       "5  anotador_6043174             sistema_compras       8  anotador_6043174   \n",
       "6  anotador_6043174            nome_responsavel       3  anotador_6043174   \n",
       "7  anotador_6043174                 tipo_objeto     540  anotador_6043174   \n",
       "8  anotador_6043174            objeto_licitacao     542  anotador_6043174   \n",
       "9  anotador_6043174             AVISO_LICITACAO     545  anotador_6043174   \n",
       "\n",
       "   offset  length                                              texto  \\\n",
       "0  432851      17                                  PREGAO ELETRONICO   \n",
       "1  432872       8                                           116/2018   \n",
       "2  433432      10                                         767.907,37   \n",
       "3  433599      10                                         20/11/2018   \n",
       "4  433640      22                             00410-00024534/2017-65   \n",
       "5  433716      21                              www.compras.df.gov.br   \n",
       "6  433818      25                          GERARDA DA SILVA CARVALHO   \n",
       "7  432927       9                                          aquisicao   \n",
       "8  432889     516  Registro de Precos visando a eventual aquisica...   \n",
       "9  432830    1023  AVISOS DE LICITACOES\\nPREGAO ELETRONICO No 116...   \n",
       "\n",
       "              id_ato  \n",
       "0  12_22.11.2018-R53  \n",
       "1  12_22.11.2018-R53  \n",
       "2  12_22.11.2018-R53  \n",
       "3  12_22.11.2018-R53  \n",
       "4  12_22.11.2018-R53  \n",
       "5  12_22.11.2018-R53  \n",
       "6  12_22.11.2018-R53  \n",
       "7  12_22.11.2018-R53  \n",
       "8  12_22.11.2018-R53  \n",
       "9  12_22.11.2018-R53  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6598380b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Unnamed: 0.1', 'id_dodf', 'tipo_rel', 'id_rel',\n",
       "       'anotador_rel', 'tipo_ent', 'id_ent', 'anotador_ent', 'offset',\n",
       "       'length', 'texto', 'id_ato'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "453f8434",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_atos = ['AVISO_ANUL_REV_LICITACAO', 'AVISO_LICITACAO', 'AVISO_SUSPENSAO_LICITACAO', \n",
    "             'EXTRATO_ADITAMENTO_CONTRATUAL', 'EXTRATO_CONTRATO', 'EXTRATO_CONVENIO']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8982e0f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sep = '-||-'\n",
    "df_contrato = df.pivot_table(index=['id_ato'], columns=['tipo_ent'],values='texto', aggfunc=lambda x: sep.join(x))\n",
    "df_contrato.rename({'EXTRATO_CONTRATO':'texto'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7cc29b44",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS No 03/2019\\nNOS TERMOS DO PADRAO CEB\\nPROCESSO: 00145-00000647/2019-23; DAS PARTES: ADMINISTRACAO REGIONAL DO\\nRECANTO DAS EMAS - RA-XV x CEB DISTRIBUICAO S.A - CEB; DO FUNDAMENTO\\nLEGAL: Inexigibilidade de Licitacao com base no caput do artigo 25 da Lei 8.666/1993; DO\\nOBJETO: Prestacao e utilizacao de servico publico de energia eletrica entre a distribuidora e o\\nconsumidor, de acordo com o Projeto Basico que integra o presente termo. DO VALOR: O valor\\nanual estimado do contrato e de R$ 133.242,81 (cento e trinta e tres mil, duzentos e quarenta e dois\\nreais e oitenta e um centavos); DA DOTACAO ORCAMENTARIA: Unidade Orcamentaria: 09117;\\nPrograma de Trabalho: 04.128.6001.8517.0055; Natureza da Despesa: 3.3.90.39; Fonte de Recursos:\\n100 e 120; Nota de Empenho no 2019NE00204, no valor inicial de R$ 9.993,20 (nove mil, novecentos\\ne noventa e tres reais e vinte centavos); DA VIGENCIA: O Contrato tera vigencia de 12 (doze) meses\\na contar da data da assinatura, permitida a prorrogacao na forma da Lei vigente. DATA DE\\nASSINATURA: 08 de outubro de 2019. SIGNATARIOS: Pela Administracao Regional do Recanto das\\nEmas - RA-XV, CARLOS DALVAN SOARES DE OLIVEIRA, na qualidade de Administrador\\nRegional, e pela contratada, SELMA BATISTA DO REGO LEAL, na qualidade de Gerente de\\nGrandes Clientes.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_contrato['texto'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3798bcea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_contrato = df_contrato[~df_contrato['texto'].isnull() ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ead7dc98",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>tipo_ent</th>\n",
       "      <th>AVISO_ANUL_REV_LICITACAO</th>\n",
       "      <th>AVISO_LICITACAO</th>\n",
       "      <th>AVISO_SUSPENSAO_LICITACAO</th>\n",
       "      <th>EXTRATO_ADITAMENTO_CONTRATUAL</th>\n",
       "      <th>texto</th>\n",
       "      <th>EXTRATO_CONVENIO</th>\n",
       "      <th>cnpj_entidade_contratada</th>\n",
       "      <th>cnpj_entidade_convenente</th>\n",
       "      <th>cnpj_orgao_concedente</th>\n",
       "      <th>cnpj_orgao_contratante</th>\n",
       "      <th>...</th>\n",
       "      <th>processo_gdf</th>\n",
       "      <th>programa_trabalho</th>\n",
       "      <th>sistema_compras</th>\n",
       "      <th>tipo_objeto</th>\n",
       "      <th>unidade_orcamentaria</th>\n",
       "      <th>valor_contrato</th>\n",
       "      <th>valor_convenio</th>\n",
       "      <th>valor_estimado_contratacao</th>\n",
       "      <th>vigencia_contrato</th>\n",
       "      <th>vigencia_convenio</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_ato</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10_11.10.2019-R1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00145-00000647/2019-23</td>\n",
       "      <td>04.128.6001.8517.0055</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>09117</td>\n",
       "      <td>133.242,81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O Contrato tera vigencia de 12 (doze) meses\\na...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_11.10.2019-R10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO No 17/2019\\nExtrato do Con...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02.633.335/0001-72</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00121.0000.0393/2019-21</td>\n",
       "      <td>04.122.6003.2396.0033-||-04122600385170104</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.000,00-||-10.000,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_11.10.2019-R11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DE CONTRATO\\nContrato de prestacao de ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>05.411.801/0001-63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00071-00001158/2018-55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.072,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12 (doze) meses,\\ncontados a partir da assinatura</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_11.10.2019-R14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO N 57/2019\\nProcesso: 00054...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00054-00018200/2019-48</td>\n",
       "      <td>28845090300NR0053</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>170393/00001</td>\n",
       "      <td>200.500,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12 (doze) meses a contar da\\ndata de sua assin...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_11.10.2019-R15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00400-00020718/2019-55</td>\n",
       "      <td>14.122.6002.8517.7250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>44101</td>\n",
       "      <td>494.959,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O contrato tera vigencia de 12 (doze) meses</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_10.9.2018-R1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DE CONTRATO\\nPROCESSO: 00390-00001500/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00390-00001500/2018-31</td>\n",
       "      <td>15.122.6001.8517.0131</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28101</td>\n",
       "      <td>9.220,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O contrato tera vigencia de 12 (doze) meses, a...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_10.9.2018-R2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO PARA PRESTACAO DE SERVICOS...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00.546.952/0001-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00.394.718/0001-00</td>\n",
       "      <td>...</td>\n",
       "      <td>00050-00025147/2018-63</td>\n",
       "      <td>14.421.62 11 . 2 7 2 7 . 0 0 0 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24101</td>\n",
       "      <td>12.600,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A vigencia contratual sera de 24 (vinte e quat...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_10.9.2018-R3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO PARA AQUISICAO DE BENS PEL...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00.716.050/0001-70</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00.394.718/0001-00</td>\n",
       "      <td>...</td>\n",
       "      <td>00050-00032679/2018-57</td>\n",
       "      <td>14.421.6211.2727.0001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24101</td>\n",
       "      <td>27.292,00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A vigencia contratual sera de ate 31/12/2018,\\...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_10.9.2018-R4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO PARA AQUISICAO DE BENS PEL...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01.429.437/0001-08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00.394.718/0001-00</td>\n",
       "      <td>...</td>\n",
       "      <td>00050-00001515/2018-88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_10.9.2018-R5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>00050.000.10359/2018-46</td>\n",
       "      <td>06.122.6002.8517.0006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24101</td>\n",
       "      <td>5.832.343,30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>o prazo de vigencia do presente contrato\\nsera...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1734 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "tipo_ent          AVISO_ANUL_REV_LICITACAO AVISO_LICITACAO  \\\n",
       "id_ato                                                       \n",
       "10_11.10.2019-R1                       NaN             NaN   \n",
       "10_11.10.2019-R10                      NaN             NaN   \n",
       "10_11.10.2019-R11                      NaN             NaN   \n",
       "10_11.10.2019-R14                      NaN             NaN   \n",
       "10_11.10.2019-R15                      NaN             NaN   \n",
       "...                                    ...             ...   \n",
       "9_10.9.2018-R1                         NaN             NaN   \n",
       "9_10.9.2018-R2                         NaN             NaN   \n",
       "9_10.9.2018-R3                         NaN             NaN   \n",
       "9_10.9.2018-R4                         NaN             NaN   \n",
       "9_10.9.2018-R5                         NaN             NaN   \n",
       "\n",
       "tipo_ent          AVISO_SUSPENSAO_LICITACAO EXTRATO_ADITAMENTO_CONTRATUAL  \\\n",
       "id_ato                                                                      \n",
       "10_11.10.2019-R1                        NaN                           NaN   \n",
       "10_11.10.2019-R10                       NaN                           NaN   \n",
       "10_11.10.2019-R11                       NaN                           NaN   \n",
       "10_11.10.2019-R14                       NaN                           NaN   \n",
       "10_11.10.2019-R15                       NaN                           NaN   \n",
       "...                                     ...                           ...   \n",
       "9_10.9.2018-R1                          NaN                           NaN   \n",
       "9_10.9.2018-R2                          NaN                           NaN   \n",
       "9_10.9.2018-R3                          NaN                           NaN   \n",
       "9_10.9.2018-R4                          NaN                           NaN   \n",
       "9_10.9.2018-R5                          NaN                           NaN   \n",
       "\n",
       "tipo_ent                                                       texto  \\\n",
       "id_ato                                                                 \n",
       "10_11.10.2019-R1   EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...   \n",
       "10_11.10.2019-R10  EXTRATO DO CONTRATO No 17/2019\\nExtrato do Con...   \n",
       "10_11.10.2019-R11  EXTRATO DE CONTRATO\\nContrato de prestacao de ...   \n",
       "10_11.10.2019-R14  EXTRATO DO CONTRATO N 57/2019\\nProcesso: 00054...   \n",
       "10_11.10.2019-R15  EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...   \n",
       "...                                                              ...   \n",
       "9_10.9.2018-R1     EXTRATO DE CONTRATO\\nPROCESSO: 00390-00001500/...   \n",
       "9_10.9.2018-R2     EXTRATO DO CONTRATO PARA PRESTACAO DE SERVICOS...   \n",
       "9_10.9.2018-R3     EXTRATO DO CONTRATO PARA AQUISICAO DE BENS PEL...   \n",
       "9_10.9.2018-R4     EXTRATO DO CONTRATO PARA AQUISICAO DE BENS PEL...   \n",
       "9_10.9.2018-R5     EXTRATO DO CONTRATO DE PRESTACAO DE SERVICOS N...   \n",
       "\n",
       "tipo_ent          EXTRATO_CONVENIO cnpj_entidade_contratada  \\\n",
       "id_ato                                                        \n",
       "10_11.10.2019-R1               NaN                      NaN   \n",
       "10_11.10.2019-R10              NaN       02.633.335/0001-72   \n",
       "10_11.10.2019-R11              NaN       05.411.801/0001-63   \n",
       "10_11.10.2019-R14              NaN                      NaN   \n",
       "10_11.10.2019-R15              NaN                      NaN   \n",
       "...                            ...                      ...   \n",
       "9_10.9.2018-R1                 NaN                      NaN   \n",
       "9_10.9.2018-R2                 NaN       00.546.952/0001-05   \n",
       "9_10.9.2018-R3                 NaN       00.716.050/0001-70   \n",
       "9_10.9.2018-R4                 NaN       01.429.437/0001-08   \n",
       "9_10.9.2018-R5                 NaN                      NaN   \n",
       "\n",
       "tipo_ent          cnpj_entidade_convenente cnpj_orgao_concedente  \\\n",
       "id_ato                                                             \n",
       "10_11.10.2019-R1                       NaN                   NaN   \n",
       "10_11.10.2019-R10                      NaN                   NaN   \n",
       "10_11.10.2019-R11                      NaN                   NaN   \n",
       "10_11.10.2019-R14                      NaN                   NaN   \n",
       "10_11.10.2019-R15                      NaN                   NaN   \n",
       "...                                    ...                   ...   \n",
       "9_10.9.2018-R1                         NaN                   NaN   \n",
       "9_10.9.2018-R2                         NaN                   NaN   \n",
       "9_10.9.2018-R3                         NaN                   NaN   \n",
       "9_10.9.2018-R4                         NaN                   NaN   \n",
       "9_10.9.2018-R5                         NaN                   NaN   \n",
       "\n",
       "tipo_ent          cnpj_orgao_contratante  ...             processo_gdf  \\\n",
       "id_ato                                    ...                            \n",
       "10_11.10.2019-R1                     NaN  ...   00145-00000647/2019-23   \n",
       "10_11.10.2019-R10                    NaN  ...  00121.0000.0393/2019-21   \n",
       "10_11.10.2019-R11                    NaN  ...   00071-00001158/2018-55   \n",
       "10_11.10.2019-R14                    NaN  ...   00054-00018200/2019-48   \n",
       "10_11.10.2019-R15                    NaN  ...   00400-00020718/2019-55   \n",
       "...                                  ...  ...                      ...   \n",
       "9_10.9.2018-R1                       NaN  ...   00390-00001500/2018-31   \n",
       "9_10.9.2018-R2        00.394.718/0001-00  ...   00050-00025147/2018-63   \n",
       "9_10.9.2018-R3        00.394.718/0001-00  ...   00050-00032679/2018-57   \n",
       "9_10.9.2018-R4        00.394.718/0001-00  ...   00050-00001515/2018-88   \n",
       "9_10.9.2018-R5                       NaN  ...  00050.000.10359/2018-46   \n",
       "\n",
       "tipo_ent                                    programa_trabalho sistema_compras  \\\n",
       "id_ato                                                                          \n",
       "10_11.10.2019-R1                        04.128.6001.8517.0055             NaN   \n",
       "10_11.10.2019-R10  04.122.6003.2396.0033-||-04122600385170104             NaN   \n",
       "10_11.10.2019-R11                                         NaN             NaN   \n",
       "10_11.10.2019-R14                           28845090300NR0053             NaN   \n",
       "10_11.10.2019-R15                       14.122.6002.8517.7250             NaN   \n",
       "...                                                       ...             ...   \n",
       "9_10.9.2018-R1                          15.122.6001.8517.0131             NaN   \n",
       "9_10.9.2018-R2               14.421.62 11 . 2 7 2 7 . 0 0 0 1             NaN   \n",
       "9_10.9.2018-R3                          14.421.6211.2727.0001             NaN   \n",
       "9_10.9.2018-R4                                            NaN             NaN   \n",
       "9_10.9.2018-R5                          06.122.6002.8517.0006             NaN   \n",
       "\n",
       "tipo_ent          tipo_objeto unidade_orcamentaria          valor_contrato  \\\n",
       "id_ato                                                                       \n",
       "10_11.10.2019-R1          NaN                09117              133.242,81   \n",
       "10_11.10.2019-R10         NaN                  NaN  10.000,00-||-10.000,00   \n",
       "10_11.10.2019-R11         NaN                  NaN                9.072,00   \n",
       "10_11.10.2019-R14         NaN         170393/00001              200.500,00   \n",
       "10_11.10.2019-R15         NaN                44101              494.959,00   \n",
       "...                       ...                  ...                     ...   \n",
       "9_10.9.2018-R1            NaN                28101                9.220,00   \n",
       "9_10.9.2018-R2            NaN                24101               12.600,00   \n",
       "9_10.9.2018-R3            NaN                24101               27.292,00   \n",
       "9_10.9.2018-R4            NaN                  NaN                     NaN   \n",
       "9_10.9.2018-R5            NaN                24101            5.832.343,30   \n",
       "\n",
       "tipo_ent          valor_convenio valor_estimado_contratacao  \\\n",
       "id_ato                                                        \n",
       "10_11.10.2019-R1             NaN                        NaN   \n",
       "10_11.10.2019-R10            NaN                        NaN   \n",
       "10_11.10.2019-R11            NaN                        NaN   \n",
       "10_11.10.2019-R14            NaN                        NaN   \n",
       "10_11.10.2019-R15            NaN                        NaN   \n",
       "...                          ...                        ...   \n",
       "9_10.9.2018-R1               NaN                        NaN   \n",
       "9_10.9.2018-R2               NaN                        NaN   \n",
       "9_10.9.2018-R3               NaN                        NaN   \n",
       "9_10.9.2018-R4               NaN                        NaN   \n",
       "9_10.9.2018-R5               NaN                        NaN   \n",
       "\n",
       "tipo_ent                                           vigencia_contrato  \\\n",
       "id_ato                                                                 \n",
       "10_11.10.2019-R1   O Contrato tera vigencia de 12 (doze) meses\\na...   \n",
       "10_11.10.2019-R10                                                NaN   \n",
       "10_11.10.2019-R11  12 (doze) meses,\\ncontados a partir da assinatura   \n",
       "10_11.10.2019-R14  12 (doze) meses a contar da\\ndata de sua assin...   \n",
       "10_11.10.2019-R15        O contrato tera vigencia de 12 (doze) meses   \n",
       "...                                                              ...   \n",
       "9_10.9.2018-R1     O contrato tera vigencia de 12 (doze) meses, a...   \n",
       "9_10.9.2018-R2     A vigencia contratual sera de 24 (vinte e quat...   \n",
       "9_10.9.2018-R3     A vigencia contratual sera de ate 31/12/2018,\\...   \n",
       "9_10.9.2018-R4                                                   NaN   \n",
       "9_10.9.2018-R5     o prazo de vigencia do presente contrato\\nsera...   \n",
       "\n",
       "tipo_ent          vigencia_convenio  \n",
       "id_ato                               \n",
       "10_11.10.2019-R1                NaN  \n",
       "10_11.10.2019-R10               NaN  \n",
       "10_11.10.2019-R11               NaN  \n",
       "10_11.10.2019-R14               NaN  \n",
       "10_11.10.2019-R15               NaN  \n",
       "...                             ...  \n",
       "9_10.9.2018-R1                  NaN  \n",
       "9_10.9.2018-R2                  NaN  \n",
       "9_10.9.2018-R3                  NaN  \n",
       "9_10.9.2018-R4                  NaN  \n",
       "9_10.9.2018-R5                  NaN  \n",
       "\n",
       "[1734 rows x 46 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_contrato"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "77ccffab",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Processing(TransformerMixin):\n",
    "  def __init__(self):\n",
    "    pass\n",
    "\n",
    "  def tokenize(self, sentence):\n",
    "    text = nltk.word_tokenize(sentence)\n",
    "    return text\n",
    "\n",
    "  def fit(self, X, y = None):\n",
    "    return self\n",
    "\n",
    "  def transform(self, X, y = None):\n",
    "    transformed = []\n",
    "    for x in X:\n",
    "      tokens = self.tokenize(x)\n",
    "      transformed.append(tokens)\n",
    "    return transformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2615c779",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeatureExtractor(TransformerMixin):\n",
    "  def __init__(self):\n",
    "    pass\n",
    "\n",
    "  def get_features(self, sentence):\n",
    "    sent_features = []\n",
    "    for i in range(len(sentence)):\n",
    "      word_feat = {\n",
    "        # Palavra atual\n",
    "        'word': sentence[i].lower(),\n",
    "        'capital_letter': sentence[i][0].isupper(),\n",
    "        'all_capital': sentence[i].isupper(),\n",
    "        'isdigit': sentence[i].isdigit(),\n",
    "        # Uma palavra antes\n",
    "        'word_before': '' if i == 0 else sentence[i-1].lower(),\n",
    "        'word_before_isdigit': '' if i == 0 else sentence[i-1].isdigit(),\n",
    "        'word_before_isupper': '' if i == 0 else sentence[i-1].isupper(),\n",
    "        'word_before_istitle': '' if i == 0 else sentence[i-1].istitle(),\n",
    "        # Uma palavra depois\n",
    "        'word_after': '' if i+1 >= len(sentence) else sentence[i+1].lower(),\n",
    "        'word_after_isdigit': '' if i+1 >= len(sentence) else sentence[i+1].isdigit(),\n",
    "        'word_after_isupper': '' if i+1 >= len(sentence) else sentence[i+1].isupper(),\n",
    "        'word_after_istitle': '' if i+1 >= len(sentence) else sentence[i+1].istitle(),\n",
    "\n",
    "        'BOS': i == 0,\n",
    "        'EOS': i == len(sentence)-1\n",
    "      }\n",
    "      sent_features.append(word_feat)\n",
    "    return sent_features\n",
    "\n",
    "  def fit(self, X, y = None):\n",
    "    return self\n",
    "\n",
    "  def transform(self, X, y = None):\n",
    "    transformed = []\n",
    "    for x in X:\n",
    "      features = self.get_features(x)\n",
    "      transformed.append(features)\n",
    "    return transformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "89c347cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(BaseEstimator):\n",
    "\n",
    "    def __init__(self):\n",
    "        self.crf = CRF(\n",
    "          algorithm = 'lbfgs',\n",
    "          c1=0.35,\n",
    "          c2=0.2,\n",
    "          max_iterations=60,\n",
    "          all_possible_transitions=True\n",
    "        )\n",
    "\n",
    "    def transform(self, X, y = None):\n",
    "      return X\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.crf.fit(X, y)\n",
    "        return self\n",
    "\n",
    "    def predict(self, x):\n",
    "        pred = self.crf.predict(x)\n",
    "        return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "79280100",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_custom = Pipeline([('pre-processing', Processing()), \n",
    "                            ('feature-extraction', FeatureExtractor()), ('model', Model())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d63f4b41",
   "metadata": {},
   "outputs": [],
   "source": [
    "DEFAULT_TOKENIZER = nltk.RegexpTokenizer(r\"\\w+\").tokenize\n",
    "class Tokenizer(TransformerMixin, BaseEstimator):\n",
    "    \"\"\" Class to apply tokenizer to pandas DataFrame.\n",
    "    \"\"\"\n",
    "    def __init__(self, tokenizer=DEFAULT_TOKENIZER):\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "\n",
    "    def __call__(self, X, **kw_params):\n",
    "        return self.tokenizer(X, **kw_params)\n",
    "\n",
    "\n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        return self\n",
    "\n",
    "\n",
    "    def transform(self, X, **kw_params):\n",
    "        if not isinstance(X, pd.Series):\n",
    "            print(\"[preprocess.Tokenizer.transform] TYPE:\", type(X))\n",
    "            print('X:::: ', X)\n",
    "            X = pd.Series(X)\n",
    "        return X.map(self)\n",
    "\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class IOBifyer(TransformerMixin, BaseEstimator):\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def find_entity(row, token, ignore_idx=0,\n",
    "        tokenizer=DEFAULT_TOKENIZER):\n",
    "        # TODO: aceitar opção de offset, para não ter tennhum tipo de problema\n",
    "        \"\"\"Searches for named entities on columns, except by ignore_idx-columns.\n",
    "\n",
    "        ignore_idx: int indicating which column has\n",
    "                    the TEXT where the named entity were extracted from\n",
    "        \"\"\"\n",
    "        for idx, column in enumerate(row.keys()):\n",
    "            if idx == ignore_idx:\n",
    "                continue                    \n",
    "                \n",
    "            \n",
    "            if isinstance(row[column], str):\n",
    "                if token == word_tokenize(row[column])[0]:\n",
    "                    return column\n",
    "                spt = row[column].split('-||-')\n",
    "                if len(spt) > 0:\n",
    "                    for spt_token in spt:\n",
    "                        if token == word_tokenize(spt_token)[0]:\n",
    "                            print(token, spt)\n",
    "                            return column\n",
    "                       \n",
    "\n",
    "        return None\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def generate_IOB_labels(row, idx, tokenizer, dbg={}):\n",
    "        \"\"\"[summary]\n",
    "\n",
    "        Args:\n",
    "            row ([pd.Series]): [pandas series having act text and entities text]\n",
    "            idx ([int]): [index such that `row[idx]` has the whole act]\n",
    "            tokenizer ([Callable]): [function to use to tokenize `row[idx]`]\n",
    "            dbg (dict, optional): [dictionay for debug purposes]. Defaults to {}.\n",
    "\n",
    "        Returns:\n",
    "            [Iter[Iter[str]]]: [matrix of IOB labels]\n",
    "        \"\"\"\n",
    "        labels = []\n",
    "        entity_started = False\n",
    "        text = row.iloc[idx]\n",
    "        for token in word_tokenize(text):                         # Itera sobre cada token da anotação do ato.\n",
    "            if not entity_started:                               # Caso uma entidade ainda n tenha sido identificada nos tokens.\n",
    "                entity = IOBifyer.find_entity(row, token, idx)                 # Busca o token atual no primeiro token de todos os campos do df.\n",
    "                if entity is not None:                           # Se foi encontrado o token no inicio de alguma entidade ele inicia a comparação token a token com a entidade.\n",
    "                    entity_started = True\n",
    "                    token_index = 1\n",
    "                    labels.append('B-' + entity)\n",
    "                else:\n",
    "                    labels.append('O')\n",
    "            else:     # Caso uma entidade já tenha sido identificada\n",
    "                if token_index < len(word_tokenize(row[entity])) and \\\n",
    "                    token == word_tokenize(row[entity])[token_index]:\n",
    "                    # Checa se o próximo token pertence à entidade\n",
    "                    # e se o tamanho da entidade chegou ao fim.\n",
    "                    labels.append('I-' + entity)\n",
    "                    # Se a entidade ainda possui tokens e a comparação foi bem\n",
    "                    # sucedida adicione o label I.\n",
    "                    token_index += 1\n",
    "                    if token_index >= len(word_tokenize(row[entity])):\n",
    "                        entity_started = False\n",
    "                else:\n",
    "                    # Se o token n for igual ou a entidade chegou ao fim.\n",
    "                    entity_started = False\n",
    "                    labels.append('O')\n",
    "        if labels[0] != 'O':\n",
    "            dbg['l'] = dbg.get('l', []) + [(row, idx)]\n",
    "        \n",
    "        return labels\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def dump_iob(tokens_mat, labels_mat, path='dump.txt',\n",
    "                            sep=' X X ', sent_sep='\\n',):\n",
    "        \"\"\"This method dumps the token matrix according its IOB labels.\n",
    "\n",
    "        For debug purposes, a list of list of pairs (token, label) is returned.\n",
    "        Args:\n",
    "            tokens_mat ([Iter[Iter[str]]]): [matrix of strings corresponding to tokens]\n",
    "            labels_mat ([Iter[Iter[str]]]): [matrix of strings corresponding to IOB labels]\n",
    "            path (str, optional): [Path to dump text file]. Defaults to 'dump.txt'.\n",
    "            sep (str, optional): [description]. Defaults to ' X X '.\n",
    "            sent_sep (str, optional): [description]. Defaults to '\\n'.\n",
    "\n",
    "        Returns:\n",
    "            [List[LIst[Tuple(str, str)]]]: [list of list of pairs (token, label), as dumped. For debug purposes.]\n",
    "        \"\"\"\n",
    "        dbg_mat = []\n",
    "        if isinstance(path, Path):\n",
    "            path = path.as_posix()\n",
    "        if '/' in path:\n",
    "            os.makedirs('/'.join(path.split('/')[:-1]), exist_ok=True)\n",
    "\n",
    "        with open(path, 'w') as fp:\n",
    "            for tokens_lis, labels_lis in zip(tokens_mat, labels_mat):\n",
    "                dbg_mat.append([])\n",
    "                for token, label in zip(tokens_lis, labels_lis):\n",
    "                    dbg_mat[-1].append((token, label))\n",
    "                    fp.write(f\"{token}{sep}{label}\\n\")\n",
    "                fp.write(sent_sep)\n",
    "        return dbg_mat\n",
    "\n",
    "\n",
    "    def __init__(self, column='act_column',\n",
    "        tokenizer=DEFAULT_TOKENIZER):\n",
    "        self.column = column\n",
    "        self.tokenizer = tokenizer\n",
    "        self.dbg = {}\n",
    "\n",
    "\n",
    "    def fit(self, X=None, y=None, **fit_params):\n",
    "        return self\n",
    "\n",
    "\n",
    "    def transform(self, df):\n",
    "        if not isinstance(df, pd.DataFrame):\n",
    "            raise TypeError(f\"`df` expected to be a pd.DataFrame. Got {type(df)}\")\n",
    "        if df.empty:\n",
    "            print(\"[core.preprocess]Warning: empty DataFrame. There won't be ioblabels.\")\n",
    "            return pd.Series()\n",
    "\n",
    "        idx = self.column if isinstance(self.column, int) else  \\\n",
    "                df.columns.get_loc(self.column)\n",
    "        labels_row = []\n",
    "        \n",
    "        for index, row in tqdm(df.iterrows()):\n",
    "            try:\n",
    "                labels_row.append(\n",
    "                    IOBifyer.generate_IOB_labels(\n",
    "                        row, idx, self.tokenizer, self.dbg\n",
    "                    )\n",
    "                )\n",
    "            except Exception as e:\n",
    "                print(\"problem iobifyin row:\", row)\n",
    "                raise e\n",
    "        return pd.Series(labels_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "51574f22",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 ['100', '120']\n",
      "120 ['100', '120']\n",
      "17/2019 ['17/2019', '17/2019']\n",
      "17/2019 ['17/2019', '17/2019']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "2it [00:01,  2.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "04.122.6003.2396.0033 ['04.122.6003.2396.0033', '04122600385170104']\n",
      "33.9039 ['33.9039', '339039']\n",
      "100 ['100', '100']\n",
      "2019NE00469 ['2019NE00469', '2019NE00470']\n",
      "10.000,00 ['10.000,00', '10.000,00']\n",
      "04122600385170104 ['04.122.6003.2396.0033', '04122600385170104']\n",
      "339039 ['33.9039', '339039']\n",
      "100 ['100', '100']\n",
      "2019NE00470 ['2019NE00469', '2019NE00470']\n",
      "10.000,00 ['10.000,00', '10.000,00']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25it [00:08,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16//2019 ['16/2019', '16//2019']\n",
      "16/2019 ['16/2019', '16//2019']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [00:10,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "279.300,00 ['279.300,00', '46.550,00']\n",
      "2018NE03864 ['2018NE03864', '2018NE03865']\n",
      "2018NE03865 ['2018NE03864', '2018NE03865']\n",
      "46.550,00 ['279.300,00', '46.550,00']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [00:15,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018NE00463 ['2018NE00463', '2018NE00464']\n",
      "2018NE00464 ['2018NE00463', '2018NE00464']\n",
      "732014482 ['732014482', '100000000']\n",
      "100000000 ['732014482', '100000000']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51it [00:17,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018NE00514 ['2018NE00514', '2018NE00515']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2679857/4137321741.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0miob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mIOBifyer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'texto'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_contrato\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/sklearn/utils/_set_output.py\u001b[0m in \u001b[0;36mwrapped\u001b[0;34m(self, X, *args, **kwargs)\u001b[0m\n\u001b[1;32m    138\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m         \u001b[0mdata_to_wrap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_to_wrap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m             \u001b[0;31m# only wrap the first output for cross decomposition\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_2679857/2465733059.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, df)\u001b[0m\n\u001b[1;32m    159\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m                 labels_row.append(\n\u001b[0;32m--> 161\u001b[0;31m                     IOBifyer.generate_IOB_labels(\n\u001b[0m\u001b[1;32m    162\u001b[0m                         \u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdbg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m                     )\n",
      "\u001b[0;32m/tmp/ipykernel_2679857/2465733059.py\u001b[0m in \u001b[0;36mgenerate_IOB_labels\u001b[0;34m(row, idx, tokenizer, dbg)\u001b[0m\n\u001b[1;32m     74\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m                         \u001b[0;31m# Itera sobre cada token da anotação do ato.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mentity_started\u001b[0m\u001b[0;34m:\u001b[0m                               \u001b[0;31m# Caso uma entidade ainda n tenha sido identificada nos tokens.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m                 \u001b[0mentity\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mIOBifyer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_entity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m                 \u001b[0;31m# Busca o token atual no primeiro token de todos os campos do df.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mentity\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m                           \u001b[0;31m# Se foi encontrado o token no inicio de alguma entidade ele inicia a comparação token a token com a entidade.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m                     \u001b[0mentity_started\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_2679857/2465733059.py\u001b[0m in \u001b[0;36mfind_entity\u001b[0;34m(row, token, ignore_idx, tokenizer)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m                 \u001b[0;32mif\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m                 \u001b[0mspt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'-||-'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36mword_tokenize\u001b[0;34m(text, language, preserve_line)\u001b[0m\n\u001b[1;32m    128\u001b[0m     \"\"\"\n\u001b[1;32m    129\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m     return [\n\u001b[0m\u001b[1;32m    131\u001b[0m         \u001b[0mtoken\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msentences\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_treebank_word_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m     ]\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    129\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m     return [\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0mtoken\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msentences\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_treebank_word_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m     ]\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/nltk/tokenize/destructive.py\u001b[0m in \u001b[0;36mtokenize\u001b[0;34m(self, text, convert_parentheses, return_str)\u001b[0m\n\u001b[1;32m    179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mregexp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCONTRACTIONS2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 181\u001b[0;31m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\" \\1 \\2 \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    182\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mregexp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCONTRACTIONS3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\" \\1 \\2 \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "iob = IOBifyer(column='texto')\n",
    "r = iob.transform(df_contrato)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "a7f63bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oi-||-como', 'vai', '?']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize('oi-||-como vai?')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}