fnn-ne-projection-side-based.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 88,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2799,
     "status": "ok",
     "timestamp": 1516045322673,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "NamUCuj1bBjd",
    "outputId": "7874258f-7c3e-4646-f238-36437759767b"
   },
   "outputs": [],
   "source": [
    "# import\n",
    "import keras\n",
    "import sys\n",
    "import numpy as np\n",
    "import string\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Sequential\n",
    "from keras.layers import SimpleRNN, Dense\n",
    "from keras.utils import np_utils, plot_model\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, precision_recall_fscore_support\n",
    "import h5py as h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "t9h7fyDCbkMG"
   },
   "outputs": [],
   "source": [
    "# if we are doeing binary classification. That means say if a token is a named entity or not\n",
    "BINARY = False\n",
    "\n",
    "# number of epochs for training\n",
    "epochs = 10 \n",
    "\n",
    "# the english side of the corpus\n",
    "en_corpus_file = \"corpus-en.txt\"\n",
    "\n",
    "# the ewondo side of the corpus\n",
    "ewo_corpus_file = \"corpus-ewo.txt\"\n",
    "\n",
    "# name of the file to same the model \n",
    "best_model_file = \"best-model-conll.hdfs\"\n",
    "\n",
    "# The maximal number of phrases to use\n",
    "max_nb_of_phrases =  -1\n",
    "\n",
    "# the maximal number of duplicates for each word in the corpus\n",
    "duplication = 1\n",
    "\n",
    "# wether we are using only the vocabulary, ro redundancy\n",
    "is_only_vocab = True\n",
    "\n",
    "# if word should be shuffle or not\n",
    "shuffle = is_only_vocab\n",
    "\n",
    "# normalization strategy\n",
    "# log, max, mean_log, \n",
    "# None: for no normalization\n",
    "normalization_strategy = \"mean_log\"\n",
    "\n",
    "# if we are using the Zennaki et al. signature\n",
    "is_zennaki = False\n",
    "\n",
    "# the number of neurons in the first layer\n",
    "h1_size = 640\n",
    "\n",
    "# number of neurons in the second layer\n",
    "h2_size = 160  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "def getTag(aString):\n",
    "    \"\"\"\n",
    "        convert a string to a tag\n",
    "    \"\"\"\n",
    "    tag = \"O\"\n",
    "    if BINARY:\n",
    "        if aString != \"O\":\n",
    "            return \"NE\"\n",
    "    else:\n",
    "        tag = aString\n",
    "    return tag\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "def load_corpus(file, max_nb_of_phrases):\n",
    "    \"\"\"\n",
    "    Load a corpus stored in a file\n",
    "    Input:\n",
    "        - file: the name of the file of the corpus\n",
    "        - max_nb_of_phases: maximal number of phrases to load\n",
    "    \n",
    "    Return:\n",
    "        - a DataFrame representing the corpus\n",
    "        - the number of phrases in the corpus\n",
    "    \"\"\"\n",
    "    nb_of_phrases = 0\n",
    "    dataset = {\"word\": [], \"ne-tag\": []}\n",
    "    with open(file) as f:\n",
    "        prev_line = None\n",
    "        for cpt, line in enumerate(f):\n",
    "            if cpt == 0:\n",
    "                continue\n",
    "            if nb_of_phrases == max_nb_of_phrases:\n",
    "                break;\n",
    "\n",
    "            l = line.strip()\n",
    "            if len(l) == 0 and len(prev_line) != 0:\n",
    "                nb_of_phrases += 1\n",
    "                dataset[\"word\"].append(line)\n",
    "                dataset[\"ne-tag\"].append(None)\n",
    "            else:\n",
    "                l = l.split(\"\\t\")\n",
    "                if l[0] not in string.punctuation:\n",
    "                    dataset[\"word\"].append(l[0])\n",
    "                    dataset[\"ne-tag\"].append(ne_type(l[1]))\n",
    "            prev_line = line.strip()\n",
    "        \n",
    "    return pd.DataFrame(dataset), nb_of_phrases+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_normalization(fingerprints):\n",
    "    fingerprints[fingerprints > 0] = 1 + np.log(fingerprints[fingerprints > 0])\n",
    "    return fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_normalization(fingerprints):\n",
    "    maxis = fingerprints.max(axis = 1)\n",
    "    fingerprints[fingerprints > 0] = fingerprints[fingerprints > 0].apply(lambda row: 0.5 + 0.5 * row / maxis[row.index])\n",
    "    return fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_log_normalization(fingerprints):\n",
    "    means = fingerprints.mean(axis=1)\n",
    "    fingerprints[fingerprints > 0] = fingerprints[fingerprints > 0].apply(lambda row: (1 + np.log(row)) / 1 + np.log(means[row.index]))\n",
    "    return fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(fingerprints):\n",
    "    if normalization_strategy == \"log\":\n",
    "        return log_normalization(fingerprints)\n",
    "    elif normalization_strategy == \"max\":\n",
    "        return max_normalization(fingerprints)\n",
    "    elif normalization_strategy == \"mean_log\":\n",
    "        return mean_log_normalization(fingerprints)\n",
    "    elif normalization_strategy == \"log_inv\":\n",
    "        fp = log_normalization(fingerprints)\n",
    "        fp = 1 / (1 + fp)\n",
    "        return fp\n",
    "    elif normalization_strategy == \"max_inv\":\n",
    "        fp = max_normalization(fingerprints)\n",
    "        fp = 1 / (1 + fp)\n",
    "        return fp\n",
    "    elif normalization_strategy == \"mean_log_inv\":\n",
    "        fp = mean_log_normalization(fingerprints)\n",
    "        fp = 1 / (1 + fp)\n",
    "        return fp\n",
    "    elif normalization_strategy == \"tf\":\n",
    "        fp = fingerprints\n",
    "        fp[fp > 0] = 1 / fp[fp > 0]\n",
    "        return fp\n",
    "    else:\n",
    "        return fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "def corpus_fingerprint(aDataframe, nb_of_biphrases):\n",
    "    \"\"\"\n",
    "    Create the distributionnal signature of each word in the corpus\n",
    "    Input:\n",
    "        -aDataFrame: the corpus DataFrame\n",
    "        -nb_of_biphrases: number of phrases in the corpus\n",
    "    Return:\n",
    "        a DataFrame: corpus fingerprint, the columns are the words in the corpus\n",
    "    \"\"\"\n",
    "    print(\"Normalization strategy:\", normalization_strategy)\n",
    "    tf = {}\n",
    "    fingerprints = {}\n",
    "    current_bi_phrase_index = 0\n",
    "    nb_word_in_corpus = aDataframe[aDataframe.word != \"\\n\"].word.size\n",
    "    words_in_current_phrase = []\n",
    "    for index, row in aDataframe.iterrows():\n",
    "        if current_bi_phrase_index > nb_of_biphrases:\n",
    "            break\n",
    "            \n",
    "        word = row['word']\n",
    "        \n",
    "        if word != \"\\n\":\n",
    "            words_in_current_phrase.append(word)\n",
    "            if word not in fingerprints:\n",
    "                fingerprints[word] = np.zeros(nb_of_biphrases, dtype=np.float32)\n",
    "                tf[word] = 0\n",
    "            tf[word] += 1\n",
    "            fingerprints[word][current_bi_phrase_index] = 1\n",
    "        else:\n",
    "            nb_word_in_current_phrase = len(words_in_current_phrase)\n",
    "            current_bi_phrase_index += 1\n",
    "            words_in_current_phrase = []\n",
    "        \n",
    "    if not is_zennaki:\n",
    "        for word in fingerprints:\n",
    "            for i in range(nb_of_biphrases):\n",
    "                if fingerprints[word][i] != 0:\n",
    "                    fingerprints[word][i] = nb_word_in_corpus / tf[word]\n",
    "    ret = pd.DataFrame(fingerprints)\n",
    "        \n",
    "    return normalize(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "def corpus2trainingdata(aDataframe, fingerprintsDataFrame):\n",
    "    \"\"\"\n",
    "    Convert corpus to training data => numpy array\n",
    "    \n",
    "    Input:\n",
    "        -aDataFrame: Corpus dataframe\n",
    "        -fingerprintsDataFrame: distributionnal signature of words in the corpus\n",
    "    Return:\n",
    "        (X, y): X is the array of words (signature) in the corpus and y is the corresponding labels (NE tags)\n",
    "    \"\"\"\n",
    "    X = np.zeros((aDataframe.shape[0], fingerprintsDataFrame.shape[0]), dtype=np.int8)\n",
    "    y = np.zeros(aDataframe.shape[0], dtype=np.int8)\n",
    "    i = 0\n",
    "    for row in aDataframe.iterrows():\n",
    "        X[i] = fingerprintsDataFrame[row[1]['word']].values\n",
    "        y[i] = tag2int[getTag(row[1]['ne-tag'])]\n",
    "        i += 1\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "# A utility function to convert NE tags\n",
    "def ne_type(aType):\n",
    "    aType = aType.lower()\n",
    "    if 'per' in aType:\n",
    "        t =  'NE' if BINARY else 'PER' \n",
    "    elif 'loc' in aType:\n",
    "        t =  'NE' if BINARY else 'LOC'\n",
    "    elif 'org' in aType:\n",
    "        t =  'NE' if BINARY else 'ORG'\n",
    "    elif 'hour' in aType:\n",
    "        t =  'NE' if BINARY else 'MISC'\n",
    "    elif aType != 'o' and len(aType) > 0 :\n",
    "        t =  'NE' if BINARY else 'MISC'\n",
    "    else:\n",
    "        t = 'O'\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "q1OlcYPo1Att"
   },
   "outputs": [],
   "source": [
    "def P_R_F1(y_pred, y_true, neg_class):\n",
    "    same = y_pred[y_true==y_pred]\n",
    "    tp = same[same != neg_class].size\n",
    "    nb_of_pos_exple = y_true[y_true != neg_class].size\n",
    "    nb_of_pos_pred = y_pred[y_pred != neg_class].size\n",
    "    p = r = f1 = 0\n",
    "    try:\n",
    "        p = np.round(tp*100/nb_of_pos_pred, 2)\n",
    "    except ZeroDivisionError:\n",
    "        print(\"number of correct positive predictions is 0\")\n",
    "        \n",
    "    try:\n",
    "        r = np.round(tp*100/nb_of_pos_exple, 2)\n",
    "    except ZeroDivisionError:\n",
    "        print(\"number of position exple is 0\")\n",
    "        \n",
    "    try:\n",
    "        f1 = np.round(2*r*p/(r+p), 2)\n",
    "    except ZeroDivisionError:\n",
    "        print(\"Recall and precision are 0\")\n",
    "\n",
    "    return p, r, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_data(X, y):\n",
    "    indices = [i for i in  range(X.shape[0])]\n",
    "    np.random.shuffle(indices)\n",
    "    return X[indices], y[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 238,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1145,
     "status": "ok",
     "timestamp": 1515671862146,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "cvpl6zMzxm6X",
    "outputId": "ef520c78-0fd3-494a-defd-fb3e2f01d371"
   },
   "outputs": [],
   "source": [
    "def create_model(input_dim, output_dim):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(h1_size, input_dim=input_dim, activation='sigmoid', name=\"hidden1\"))\n",
    "    model.add(Dense(h2_size, activation='sigmoid', name=\"hidden2\"))\n",
    "    if BINARY:\n",
    "        model.add(Dense(1, activation='sigmoid', name=\"outputlayer\"))\n",
    "        model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['binary_accuracy'])\n",
    "    else:\n",
    "        model.add(Dense(output_dim, activation='softmax', name=\"outputlayer\"))\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=\"rmsprop\", metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 343,
     "output_extras": [
      {
       "item_id": 99
      },
      {
       "item_id": 223
      },
      {
       "item_id": 284
      },
      {
       "item_id": 347
      },
      {
       "item_id": 394
      },
      {
       "item_id": 445
      },
      {
       "item_id": 490
      },
      {
       "item_id": 542
      },
      {
       "item_id": 590
      },
      {
       "item_id": 639
      },
      {
       "item_id": 684
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 676115,
     "status": "ok",
     "timestamp": 1515672664054,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "uY2-NcCZxpXe",
    "outputId": "adcd5c23-d0c6-463a-bc11-db5d415eba1d"
   },
   "outputs": [],
   "source": [
    "def train_model(model, X_train, y_train, X_val, y_val, epochs=epochs):\n",
    "    # stop learning if the error is the same between two consecutive epochs\n",
    "    early_stop = EarlyStopping(patience=20, verbose=2)\n",
    "    \n",
    "    # saving best model\n",
    "    best_model_cp = ModelCheckpoint(best_model_file, save_best_only=True, verbose=1)\n",
    "    \n",
    "    model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=epochs, verbose=0, shuffle=shuffle, callbacks=[best_model_cp, early_stop])\n",
    "    \n",
    "    #loading and returning the best model\n",
    "    return keras.models.load_model(best_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, X, y, binary=BINARY):\n",
    "    if BINARY:\n",
    "        y_pred = np.round(model.predict(X))\n",
    "        y_true = y\n",
    "    else:\n",
    "        predictions = model.predict(X)\n",
    "        y_pred = np.array([np.argmax(p) for p in predictions])\n",
    "        y_true = np.array([np.argmax(t) for t in y ])\n",
    "    return y_true, y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 23179,
     "status": "ok",
     "timestamp": 1515672689915,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "N33cmgKEyPOb",
    "outputId": "917cdaec-b68c-47d5-a8ea-bd6ef7387a67"
   },
   "outputs": [],
   "source": [
    "def model_performance(y_true, y_pred):\n",
    "    return P_R_F1(y_pred, y_true, tag2int['O']) #precision, recall, f1-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_performace_by_tag(y_true, y_pred, tag):\n",
    "    p, r, f1 = 0, 0, 0\n",
    "    \n",
    "    eq = y_pred[y_pred==y_true]\n",
    "    correctly_pred = eq[eq==tag].size\n",
    "    try:\n",
    "        p = np.round(100 * correctly_pred / y_pred[y_pred==tag].size, 2)\n",
    "    except ZeroDivisionError:\n",
    "        pass\n",
    "    \n",
    "    try:\n",
    "        r = np.round(100 * correctly_pred / y_true[y_true==tag].size, 2)\n",
    "    except ZeroDivisionError:\n",
    "        pass\n",
    "    \n",
    "    try:\n",
    "        f1 = np.round(2 * r * p / (r + p), 2)\n",
    "    except ZeroDivisionError:\n",
    "        pass\n",
    "    \n",
    "    return p, r, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "def algoEval(X_train, y_train, X_val, y_val, X_ewo, y_ewo, epochs=epochs, model=None):\n",
    "    \"\"\"\n",
    "    Train a model on (X, y) and validate on (X_val, y_val) then project on (X_ewo)\n",
    "    \"\"\"\n",
    "    test_precision, train_precision, ewo_precision = [], [], []\n",
    "    test_recall, train_recall, ewo_recall = [], [], []\n",
    "    test_fscore, train_fscore, ewo_fscore = [], [], []\n",
    "    \n",
    "    test_result_by_tag = {}\n",
    "    train_result_by_tag = {}\n",
    "    ewo_result_by_tag = {}\n",
    "    for t in tagSet:\n",
    "        f1_key = \"F1-\"+t\n",
    "        p_key = \"P-\"+t\n",
    "        r_key = \"R-\"+t\n",
    "        train_result_by_tag[f1_key], train_result_by_tag[p_key], train_result_by_tag[r_key] = [], [], []\n",
    "        test_result_by_tag[f1_key], test_result_by_tag[p_key], test_result_by_tag[r_key] = [], [], []\n",
    "        ewo_result_by_tag[f1_key], ewo_result_by_tag[p_key], ewo_result_by_tag[r_key] = [], [], []\n",
    "\n",
    "    m = train_model(model, X_train, y_train, X_val, y_val, epochs=epochs)\n",
    "        \n",
    "    y_true, y_pred = predict(m, X_train, y_train)\n",
    "    p_train, r_train, f1_train = model_performance(y_true, y_pred)\n",
    "        \n",
    "    y_true_val, y_pred_val = predict(m, X_val, y_val)\n",
    "    p_val, r_val, f1_val = model_performance(y_true_val, y_pred_val)\n",
    "        \n",
    "    y_true_ewo, y_pred_ewo = predict(m, X_ewo, y_ewo) \n",
    "    p_ewo, r_ewo, f1_ewo = model_performance(y_true_ewo, y_pred_ewo)\n",
    "        \n",
    "    for t in range(len(int2tag)):\n",
    "        f1_key = \"F1-\" + int2tag[t]\n",
    "        p_key = \"P-\" + int2tag[t]\n",
    "        r_key = \"R-\" + int2tag[t]\n",
    "            \n",
    "        p, r, f1 = model_performace_by_tag(y_true, y_pred, t)\n",
    "        train_result_by_tag[p_key].append(p)\n",
    "        train_result_by_tag[r_key].append(r)\n",
    "        train_result_by_tag[f1_key].append(f1)\n",
    "            \n",
    "        p, r, f1 = model_performace_by_tag(y_true_val, y_pred_val, t)\n",
    "        test_result_by_tag[p_key].append(p)\n",
    "        test_result_by_tag[r_key].append(r)\n",
    "        test_result_by_tag[f1_key].append(f1)\n",
    "            \n",
    "        p, r, f1 = model_performace_by_tag(y_true_ewo, y_pred_ewo, t)\n",
    "        ewo_result_by_tag[p_key].append(p)\n",
    "        ewo_result_by_tag[r_key].append(r)\n",
    "        ewo_result_by_tag[f1_key].append(f1)\n",
    "                \n",
    "    test_precision.append(p_val)\n",
    "    train_precision.append(p_train)\n",
    "    ewo_precision.append(p_ewo)\n",
    "        \n",
    "    test_recall.append(r_val)\n",
    "    train_recall.append(r_train)\n",
    "    ewo_recall.append(r_ewo)\n",
    "        \n",
    "    test_fscore.append(f1_val)\n",
    "    train_fscore.append(f1_train)\n",
    "    ewo_fscore.append(f1_ewo)\n",
    "    return pd.DataFrame({\n",
    "        'P_val': test_precision, \n",
    "        'P_train': train_precision, \n",
    "        'P_ewo': ewo_precision, 'R_val': test_recall, 'R_train': train_recall, \n",
    "        'R_ewo': ewo_recall, 'F1-val': test_fscore, 'F1-train': train_fscore, 'F1-ewo': ewo_fscore}), pd.DataFrame(train_result_by_tag), pd.DataFrame(test_result_by_tag), pd.DataFrame(ewo_result_by_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "def algoCrossVal(X, y, X_ewo, y_ewo, k = 10, repeat=1): \n",
    "    \"\"\"\n",
    "    Traing a model with k-fold cross validation\n",
    "    We train the model `repeat` times to check it's stability\n",
    "    \"\"\"\n",
    "    block_size = int(X.shape[0] / k)   \n",
    "    output = None\n",
    "    model = None\n",
    "    train_by_tags, test_by_tags, ewo_by_tags = None, None, None\n",
    "    for it in range(repeat):\n",
    "        print(\"AlgoCrossValIter -\", it+1)\n",
    "        model = create_model(X.shape[1], len(tagSet))\n",
    "        results = None\n",
    "        train_by_tagsTmp, test_by_tagsTmp, ewo_by_tagsTmp = None, None, None\n",
    "        for i in range(k):\n",
    "            X_val, y_val = X[i*block_size:i*block_size+block_size], y[i*block_size:i*block_size+block_size]\n",
    "            X_train = np.concatenate((X[0:i*block_size], X[i*block_size+block_size:]))\n",
    "            y_train = np.concatenate((y[0:i*block_size], y[i*block_size+block_size:]))\n",
    "\n",
    "            X_train = X_train.reshape(X_train.shape[0], X_train.shape[1])\n",
    "            X_val = X_val.reshape(X_val.shape[0], X_val.shape[1])\n",
    "\n",
    "            result, train_by_tag, test_by_tag, ewo_by_tag = algoEval(X_train, y_train, X_val, y_val, X_ewo, y_ewo, model=model)\n",
    "            if results is None:\n",
    "                results = result.copy()\n",
    "                train_by_tagsTmp, test_by_tagsTmp, ewo_by_tagsTmp = train_by_tag.copy(), test_by_tag.copy(), ewo_by_tag.copy()\n",
    "            else:\n",
    "                results = pd.concat([results, result], ignore_index=True)\n",
    "                train_by_tagsTmp = pd.concat([train_by_tagsTmp, train_by_tag], ignore_index=True)\n",
    "                test_by_tagsTmp = pd.concat([test_by_tagsTmp, test_by_tag], ignore_index=True)\n",
    "                ewo_by_tagsTmp = pd.concat([ewo_by_tagsTmp, ewo_by_tag], ignore_index=True)\n",
    "        \n",
    "        if output is None:\n",
    "            output = results.mean(axis=0).to_frame()\n",
    "            train_by_tags = train_by_tagsTmp.mean(axis=0).to_frame()\n",
    "            test_by_tags = test_by_tagsTmp.mean(axis=0).to_frame()\n",
    "            ewo_by_tags = ewo_by_tagsTmp.mean(axis=0).to_frame()\n",
    "        else:\n",
    "            output = pd.concat([output, results.mean(axis=0).to_frame()], axis=1)\n",
    "            train_by_tags = pd.concat([train_by_tags, train_by_tagsTmp.mean(axis=0).to_frame()], axis=1)\n",
    "            test_by_tags = pd.concat([test_by_tags, test_by_tagsTmp.mean(axis=0).to_frame()], axis=1)\n",
    "            ewo_by_tags = pd.concat([ewo_by_tags, ewo_by_tagsTmp.mean(axis=0).to_frame()], axis=1)\n",
    "\n",
    "    return output, train_by_tags, test_by_tags, ewo_by_tags, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "TNN2TBckE8m_"
   },
   "outputs": [],
   "source": [
    "en_corpus, en_nb_of_phrases = load_corpus(en_corpus_file, max_nb_of_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nb word in corpus 4170\n"
     ]
    }
   ],
   "source": [
    "nb_word_in_corpus = en_corpus[en_corpus.word != \"\\n\"].word.size\n",
    "print(\"Nb word in corpus\", nb_word_in_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1335</th>\n",
       "      <td>Sadducees</td>\n",
       "      <td>ORG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           word ne-tag\n",
       "1335  Sadducees    ORG"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus.head()\n",
    "en_corpus.loc[en_corpus['ne-tag'] == 'ORG']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'O': 0, 'MISC': 1, 'PER': 2, 'LOC': 3, 'ORG': 4}\n"
     ]
    }
   ],
   "source": [
    "tagSet = en_corpus[\"ne-tag\"].dropna().unique()\n",
    "if BINARY:\n",
    "    tagSet = ['NE', 'O']\n",
    "tag2int = {j: i for i, j in enumerate(tagSet)}\n",
    "int2tag = {i: j for i, j in enumerate(tagSet)}\n",
    "print(tag2int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 794,
     "status": "ok",
     "timestamp": 1515664141558,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "m85WcghdzCph",
    "outputId": "9fa6817e-15c4-4205-f8f4-82c30c2cb610"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "210"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_nb_of_phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 173,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 992,
     "status": "ok",
     "timestamp": 1515664144298,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "Gw9r-Q9jlmvg",
    "outputId": "8ee33794-5639-4c97-ea43-06a66d89e207"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4379</td>\n",
       "      <td>4170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>904</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>the</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>313</td>\n",
       "      <td>3779</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word ne-tag\n",
       "count   4379   4170\n",
       "unique   904      5\n",
       "top      the      O\n",
       "freq     313   3779"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 204,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1056,
     "status": "ok",
     "timestamp": 1515664147270,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "EZ_5FqH3yxhU",
    "outputId": "a129592b-9fa2-4937-a35a-a73245aef4a8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Promise</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Holy</td>\n",
       "      <td>MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Spirit</td>\n",
       "      <td>MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\\n</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>In</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>the</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>first</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      word ne-tag\n",
       "0      The      O\n",
       "1  Promise      O\n",
       "2       of      O\n",
       "3      the      O\n",
       "4     Holy   MISC\n",
       "5   Spirit   MISC\n",
       "6       \\n   None\n",
       "7       In      O\n",
       "8      the      O\n",
       "9    first      O"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O % = 86.3 %\n",
      "MISC % = 2.4 %\n",
      "PER % = 5.59 %\n",
      "LOC % = 0.91 %\n",
      "ORG % = 0.02 %\n"
     ]
    }
   ],
   "source": [
    "print(\"---------Whole corpus stats-----------\")\n",
    "corpus = en_corpus[en_corpus['word']!='\\n']\n",
    "total=0\n",
    "for tag in tagSet:\n",
    "    percent = corpus[corpus['ne-tag']==tag].shape[0] *100 / corpus.shape[0]\n",
    "    total += percent\n",
    "    print(\"{0} % = {1} %\".format(tag, percent))\n",
    "print(\"Total:\", total, \"%\\n\")\n",
    "\n",
    "print(\"---------Vocabulary stats-------------\")\n",
    "total = 0\n",
    "percents = {}\n",
    "for tag in tagSet:\n",
    "    percent = corpus[corpus['ne-tag']==tag].word.unique().size # * 100 / corpus.word.unique().size\n",
    "    total += percent\n",
    "    percents[tag] = percent\n",
    "for t in percents:\n",
    "    print(\"{0} % = {1} %\".format(t, percents[t] * 100 /total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(209, 2)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus[en_corpus.word == \"\\n\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 190,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40378,
     "status": "ok",
     "timestamp": 1515671247748,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "Mx1Yl-Npvh12",
    "outputId": "29d26922-46c0-47b6-9d1b-26ac26b118d4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nb of bi-phrases 210\n"
     ]
    }
   ],
   "source": [
    "print(\"Nb of bi-phrases\", en_nb_of_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 190,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40378,
     "status": "ok",
     "timestamp": 1515671247748,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "Mx1Yl-Npvh12",
    "outputId": "29d26922-46c0-47b6-9d1b-26ac26b118d4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalization strategy: mean_log\n"
     ]
    }
   ],
   "source": [
    "en_fingerprints = corpus_fingerprint(en_corpus, en_nb_of_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>The</th>\n",
       "      <th>Promise</th>\n",
       "      <th>of</th>\n",
       "      <th>the</th>\n",
       "      <th>Holy</th>\n",
       "      <th>Spirit</th>\n",
       "      <th>In</th>\n",
       "      <th>first</th>\n",
       "      <th>book</th>\n",
       "      <th>O</th>\n",
       "      <th>...</th>\n",
       "      <th>considered</th>\n",
       "      <th>dream</th>\n",
       "      <th>She</th>\n",
       "      <th>save</th>\n",
       "      <th>fulfill</th>\n",
       "      <th>Immanuel</th>\n",
       "      <th>us)</th>\n",
       "      <th>woke</th>\n",
       "      <th>sleep</th>\n",
       "      <th>knew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.680833</td>\n",
       "      <td>11.078728</td>\n",
       "      <td>5.842286</td>\n",
       "      <td>5.332525</td>\n",
       "      <td>8.370678</td>\n",
       "      <td>8.439671</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.599773</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.554217</td>\n",
       "      <td>11.65283</td>\n",
       "      <td>11.65283</td>\n",
       "      <td>11.65283</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.292995</td>\n",
       "      <td>9.331148</td>\n",
       "      <td>9.400141</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.800138</td>\n",
       "      <td>7.290377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.449357</td>\n",
       "      <td>6.939595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.071361</td>\n",
       "      <td>9.109513</td>\n",
       "      <td>9.178506</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8.554736</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.109940</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.847793</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.408932</td>\n",
       "      <td>6.899171</td>\n",
       "      <td>9.937324</td>\n",
       "      <td>10.006316</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 903 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        The    Promise        of       the      Holy     Spirit         In  \\\n",
       "0  8.680833  11.078728  5.842286  5.332525  8.370678   8.439671   0.000000   \n",
       "1  0.000000   0.000000  0.000000  6.599773  0.000000   0.000000  10.554217   \n",
       "2  0.000000   0.000000  0.000000  6.292995  9.331148   9.400141   0.000000   \n",
       "3  0.000000   0.000000  7.800138  7.290377  0.000000   0.000000   0.000000   \n",
       "4  0.000000   0.000000  7.449357  6.939595  0.000000   0.000000   0.000000   \n",
       "5  0.000000   0.000000  0.000000  6.071361  9.109513   9.178506   0.000000   \n",
       "6  8.554736   0.000000  0.000000  0.000000  0.000000   0.000000   0.000000   \n",
       "7  0.000000   0.000000  0.000000  6.109940  0.000000   0.000000   0.000000   \n",
       "8  0.000000   0.000000  0.000000  6.847793  0.000000   0.000000   0.000000   \n",
       "9  0.000000   0.000000  7.408932  6.899171  9.937324  10.006316   0.000000   \n",
       "\n",
       "      first      book         O  ...  considered  dream  She  save  fulfill  \\\n",
       "0   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "1  11.65283  11.65283  11.65283  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "2   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "3   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "4   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "5   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "6   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "7   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "8   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "9   0.00000   0.00000   0.00000  ...         0.0    0.0  0.0   0.0      0.0   \n",
       "\n",
       "   Immanuel  us)  woke  sleep  knew  \n",
       "0       0.0  0.0   0.0    0.0   0.0  \n",
       "1       0.0  0.0   0.0    0.0   0.0  \n",
       "2       0.0  0.0   0.0    0.0   0.0  \n",
       "3       0.0  0.0   0.0    0.0   0.0  \n",
       "4       0.0  0.0   0.0    0.0   0.0  \n",
       "5       0.0  0.0   0.0    0.0   0.0  \n",
       "6       0.0  0.0   0.0    0.0   0.0  \n",
       "7       0.0  0.0   0.0    0.0   0.0  \n",
       "8       0.0  0.0   0.0    0.0   0.0  \n",
       "9       0.0  0.0   0.0    0.0   0.0  \n",
       "\n",
       "[10 rows x 903 columns]"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_fingerprints.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(4170 / nb_word_in_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4379, 2)"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210,)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_fingerprints['you'].values.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4170, 2)"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_corpus[en_corpus.word != \"\\n\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>estate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          text\n",
       "count      903\n",
       "unique     903\n",
       "top     estate\n",
       "freq         1"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if is_only_vocab:\n",
    "    text = list(en_corpus[en_corpus.word != \"\\n\"].word.unique())\n",
    "else:\n",
    "    text = list(en_corpus[en_corpus.word != \"\\n\"].word)\n",
    "en_vocab = pd.DataFrame({'text': text})\n",
    "en_vocab.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 190,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40378,
     "status": "ok",
     "timestamp": 1515671247748,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "Mx1Yl-Npvh12",
    "outputId": "29d26922-46c0-47b6-9d1b-26ac26b118d4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(903, 210) (210, 903) (903,)\n"
     ]
    }
   ],
   "source": [
    "if is_only_vocab:\n",
    "    X = np.zeros((en_vocab.shape[0] * duplication, en_nb_of_phrases))\n",
    "    target = np.zeros((en_vocab.shape[0] * duplication))\n",
    "    p=0\n",
    "    for i, row in en_vocab.iterrows():\n",
    "        c = row.text\n",
    "        for j in range(duplication):\n",
    "            X[p] = en_fingerprints[c.split(\" \")[0]]\n",
    "            target[p] = tag2int[getTag(en_corpus[en_corpus.word == c.split(\" \")[-1:][0]]['ne-tag'].iloc[0])]\n",
    "            p+=1\n",
    "    X, target = shuffle_data(X, target)\n",
    "    print(X.shape, en_fingerprints.shape, target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>Eliud</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>884</th>\n",
       "      <td>Eleazar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>Matthan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>husband</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>fourteen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>888</th>\n",
       "      <td>unwilling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>shame</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>resolved</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>divorce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>quietly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893</th>\n",
       "      <td>considered</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>dream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>She</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>save</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>fulfill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>Immanuel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>us)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>900</th>\n",
       "      <td>woke</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>901</th>\n",
       "      <td>sleep</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>902</th>\n",
       "      <td>knew</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           text\n",
       "883       Eliud\n",
       "884     Eleazar\n",
       "885     Matthan\n",
       "886     husband\n",
       "887    fourteen\n",
       "888   unwilling\n",
       "889       shame\n",
       "890    resolved\n",
       "891     divorce\n",
       "892     quietly\n",
       "893  considered\n",
       "894       dream\n",
       "895         She\n",
       "896        save\n",
       "897     fulfill\n",
       "898    Immanuel\n",
       "899         us)\n",
       "900        woke\n",
       "901       sleep\n",
       "902        knew"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_vocab[-20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not is_only_vocab:\n",
    "    X, target = corpus2trainingdata(en_corpus[en_corpus.word != \"\\n\"], en_fingerprints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "if shuffle:\n",
    "    X, target = shuffle_data(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 102,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1032,
     "status": "ok",
     "timestamp": 1515671250872,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "HLn_an5ExZSC",
    "outputId": "382c3159-7917-40e3-e469-27c244d86663"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(903, 5)"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = target.copy()\n",
    "y[0:100]\n",
    "if not BINARY:\n",
    "    y = np_utils.to_categorical(y, len(tagSet))\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "def visualize(X, y):\n",
    "    pca = PCA(n_components=2)\n",
    "    X_embeded = pca.fit_transform(X)\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    plt.scatter(X_embeded[:, 0], X_embeded[:, 1], c=y)\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize(X, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 899,
     "status": "ok",
     "timestamp": 1515671252070,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "sF3M9I53xlY1",
    "outputId": "8d011439-2df0-4440-b334-7e21e44c2208"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape = (605, 210)\n",
      "y_train.shape = (605, 5)\n",
      "X_val.shape = (298, 210)\n",
      "y_val.shape = (298, 5)\n",
      "O % in training data = 89.42 %\n",
      "O % in validation data = 85.91 %\n",
      "MISC % in training data = 0.83 %\n",
      "MISC % in validation data = 2.01 %\n",
      "PER % in training data = 7.77 %\n",
      "PER % in validation data = 10.07 %\n",
      "LOC % in training data = 1.98 %\n",
      "LOC % in validation data = 1.68 %\n",
      "ORG % in training data = 0.0 %\n",
      "ORG % in validation data = 0.34 %\n"
     ]
    }
   ],
   "source": [
    "X_train, X_val, y_train, y_val = model_selection.train_test_split(X, y, test_size=0.33)\n",
    "X_train = X_train.reshape(X_train.shape[0], X_train.shape[1])\n",
    "X_val = X_val.reshape(X_val.shape[0], X_val.shape[1])\n",
    "print(\"X_train.shape =\", X_train.shape)\n",
    "print(\"y_train.shape =\", y_train.shape)\n",
    "print(\"X_val.shape =\", X_val.shape)\n",
    "print(\"y_val.shape =\", y_val.shape)\n",
    "\n",
    "tTarget = np.array([np.argmax(yy) for yy in y_train])\n",
    "vTarget = np.array([np.argmax(yy) for yy in y_val])\n",
    "\n",
    "for tag in tagSet:\n",
    "    print(\"{0} % in training data = {1} %\".format(tag, np.round(tTarget[tTarget==tag2int[tag]].size * 100 / tTarget.shape[0], 2)))\n",
    "    print(\"{0} % in validation data = {1} %\".format(tag, np.round(vTarget[vTarget==tag2int[tag]].size * 100 / vTarget.shape[0], 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 173,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 758,
     "status": "ok",
     "timestamp": 1515664313240,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "OObRLXTr268N",
    "outputId": "6361a898-fd04-4e15-8aa7-936a12b3221a"
   },
   "outputs": [],
   "source": [
    "ewo_corpus, ewo_nb_of_phrases = load_corpus(ewo_corpus_file, max_nb_of_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Teofil</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Yesus</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Yohannes</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Yesus</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>Yesus</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3676</th>\n",
       "      <td>Maria</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3697</th>\n",
       "      <td>Yesus</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3740</th>\n",
       "      <td>Emmanuel</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3750</th>\n",
       "      <td>Yosef</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778</th>\n",
       "      <td>Yesus</td>\n",
       "      <td>PER</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>253 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          word ne-tag\n",
       "6       Teofil    PER\n",
       "15       Yesus    PER\n",
       "86    Yohannes    PER\n",
       "104      Yesus    PER\n",
       "230      Yesus    PER\n",
       "...        ...    ...\n",
       "3676     Maria    PER\n",
       "3697     Yesus    PER\n",
       "3740  Emmanuel    PER\n",
       "3750     Yosef    PER\n",
       "3778     Yesus    PER\n",
       "\n",
       "[253 rows x 2 columns]"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ewo_corpus.loc[ewo_corpus['ne-tag'] == 'PER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 173,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 758,
     "status": "ok",
     "timestamp": 1515664313240,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "OObRLXTr268N",
    "outputId": "6361a898-fd04-4e15-8aa7-936a12b3221a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "210"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ewo_nb_of_phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O % = 89.94 %\n",
      "MISC % = 1.17 %\n",
      "PER % = 8.3 %\n",
      "LOC % = 1.86 %\n",
      "ORG % = 0.2 %\n"
     ]
    }
   ],
   "source": [
    "print(\"---------Whole corpus stats-----------\")\n",
    "corpus = ewo_corpus[ewo_corpus['word']!='\\n']\n",
    "total=0\n",
    "for tag in tagSet:\n",
    "    percent = corpus[corpus['ne-tag']==tag].shape[0] *100 / corpus.shape[0]\n",
    "    total += percent\n",
    "    print(\"{0} % = {1} %\".format(tag, percent))\n",
    "print(\"Total:\", total, \"%\\n\")\n",
    "\n",
    "print(\"---------Vocabulary stats-------------\")\n",
    "total = 0\n",
    "percents = {}\n",
    "for tag in tagSet:\n",
    "    percent = corpus[corpus['ne-tag']==tag].word.unique().size # * 100 / corpus.word.unique().size\n",
    "    total += percent\n",
    "    percents[tag] = percent\n",
    "for t in percents:\n",
    "    print(\"{0} % = {1} %\".format(t, percents[t] * 100 /total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 173,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 758,
     "status": "ok",
     "timestamp": 1515664313240,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "OObRLXTr268N",
    "outputId": "6361a898-fd04-4e15-8aa7-936a12b3221a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3779</td>\n",
       "      <td>3570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>1024</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>\\n</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>209</td>\n",
       "      <td>3180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word ne-tag\n",
       "count   3779   3570\n",
       "unique  1024      5\n",
       "top       \\n      O\n",
       "freq     209   3180"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ewo_corpus.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 173,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 758,
     "status": "ok",
     "timestamp": 1515664313240,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "OObRLXTr268N",
    "outputId": "6361a898-fd04-4e15-8aa7-936a12b3221a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>ne-tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mfufub</td>\n",
       "      <td>MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Nsisim</td>\n",
       "      <td>MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ayi</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sò</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\n</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     word ne-tag\n",
       "0  Mfufub   MISC\n",
       "1  Nsisim   MISC\n",
       "2     ayi      O\n",
       "3      sò      O\n",
       "4      \\n   None"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ewo_corpus.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 214,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1473,
     "status": "error",
     "timestamp": 1515688720429,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "aRDNM0C8yX5N",
    "outputId": "b5a14626-4a9b-484c-c597-f1b1f37673c9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalization strategy: mean_log\n"
     ]
    }
   ],
   "source": [
    "ewo_fingerprints = corpus_fingerprint(ewo_corpus, en_nb_of_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 214,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1473,
     "status": "error",
     "timestamp": 1515688720429,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "aRDNM0C8yX5N",
    "outputId": "b5a14626-4a9b-484c-c597-f1b1f37673c9"
   },
   "outputs": [],
   "source": [
    "if is_only_vocab:\n",
    "    text = list(ewo_corpus[ewo_corpus.word != \"\\n\"].word.unique())\n",
    "else:\n",
    "    text = list(ewo_corpus[ewo_corpus.word != \"\\n\"].word)\n",
    "ewo_vocab = pd.DataFrame({\"text\":text})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 214,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1473,
     "status": "error",
     "timestamp": 1515688720429,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "aRDNM0C8yX5N",
    "outputId": "b5a14626-4a9b-484c-c597-f1b1f37673c9"
   },
   "outputs": [],
   "source": [
    "if is_only_vocab:\n",
    "    X_ewo = np.zeros((ewo_vocab.shape[0] * duplication, en_nb_of_phrases))\n",
    "    ewo_target = np.zeros((ewo_vocab.shape[0] * duplication))\n",
    "    p=0\n",
    "    for i, row in ewo_vocab.iterrows():\n",
    "        c = row.text\n",
    "        for j in range(duplication):\n",
    "            X_ewo[p] = ewo_fingerprints[c.split(\" \")[0]]\n",
    "            ewo_target[p] = tag2int[getTag(ewo_corpus[ewo_corpus.word == c.split(\" \")[-1:][0]]['ne-tag'].iloc[0])]\n",
    "            p+=1\n",
    "    X_ewo, ewo_target = shuffle_data(X_ewo, ewo_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1013</th>\n",
       "      <td>sik</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1014</th>\n",
       "      <td>Ntud</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1015</th>\n",
       "      <td>bëyole</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016</th>\n",
       "      <td>Emmanuel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1017</th>\n",
       "      <td>Avëbë</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1018</th>\n",
       "      <td>angavëbë</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>oyò</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>angabende</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1021</th>\n",
       "      <td>anganòṅ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>angayole</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           text\n",
       "1013        sik\n",
       "1014       Ntud\n",
       "1015     bëyole\n",
       "1016   Emmanuel\n",
       "1017      Avëbë\n",
       "1018   angavëbë\n",
       "1019        oyò\n",
       "1020  angabende\n",
       "1021    anganòṅ\n",
       "1022   angayole"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ewo_vocab[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not is_only_vocab:\n",
    "    X_ewo, ewo_target = corpus2trainingdata(ewo_corpus[ewo_corpus.word != \"\\n\"], ewo_fingerprints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "if shuffle:\n",
    "    X_ewo, ewo_target = shuffle_data(X_ewo, ewo_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1023,) 1023\n"
     ]
    }
   ],
   "source": [
    "y_ewo = ewo_target.copy()\n",
    "print(y_ewo.shape, len(ewo_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1023, 210)"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_ewo.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 768,
     "status": "ok",
     "timestamp": 1514134592547,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "fuSYSwUPDiQY",
    "outputId": "658c1a63-1a76-4487-e151-47364c6dbc85"
   },
   "outputs": [],
   "source": [
    "y_ewo = ewo_target.copy()\n",
    "y_ewo[:20]\n",
    "if not BINARY:\n",
    "    y_ewo = np_utils.to_categorical(y_ewo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6835,
     "status": "ok",
     "timestamp": 1514134601113,
     "user": {
      "displayName": "Michael Franklin Mbouopda",
      "photoUrl": "//lh3.googleusercontent.com/-2_Vuj9ESsJ0/AAAAAAAAAAI/AAAAAAAABSc/r-SRcN0jWPw/s50-c-k-no/photo.jpg",
      "userId": "117384351130599115261"
     },
     "user_tz": -60
    },
    "id": "Cev5j8YFzPYl",
    "outputId": "87ab5b4b-a75d-43b0-d346-5ba18a4522c0"
   },
   "outputs": [],
   "source": [
    "X_ewo = X_ewo.reshape((X_ewo.shape[0], en_nb_of_phrases))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = create_model(X.shape[1], len(tagSet))\n",
    "# resultEval, train_by_tag, test_by_tag, ewo_by_tag = algoEval(X_train, y_train, X_val, y_val, X_ewo, y_ewo, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "# resultEval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_by_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_by_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ewo_by_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "# resultEval.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "# resultEval.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 1\n",
      "Model: \"sequential_21\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25042, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25042 to 0.23072, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.23072\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.23072\n",
      "number of correct positive predictions is 0\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.17474, saving model to best-model-conll.hdfs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:18: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.17474\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.17474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.15540, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.15540 to 0.13522, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.13522\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.13522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09829, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09829\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09952, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09952\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09671, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09671\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11358, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11358\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11358\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04422, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04422\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.07577, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.07577\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.07577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04071, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04071\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04071\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 2\n",
      "Model: \"sequential_22\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.23907, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.23907\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.23907 to 0.20286, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.20286\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.20286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:18: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24000, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24000 to 0.23393, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.23393\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.23393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.13167, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.13167\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.13167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.18803, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.18803\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.18803 to 0.16696, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.16696\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.16696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11817, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11817\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10813, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10813\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10466, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10466\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05553, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.05553 to 0.05345, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05345\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06420, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06420\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06420\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05336, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05336\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 3\n",
      "Model: \"sequential_23\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.30190, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.30190\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.30190 to 0.24213, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24213\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24213\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.24213\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.24213\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.24213 to 0.19050, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.19050\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.19050\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.17897, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.17897\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.17897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12827, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12827\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10730, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10730\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10559, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.10559 to 0.10059, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10059\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11469, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11469\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10460, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10460\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10460\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.07346, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.07346 to 0.06770, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06770\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.06770 to 0.06237, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06237\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.07368, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.07368\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.07368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04539, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04539\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 4\n",
      "Model: \"sequential_24\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.35599, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.35599 to 0.22507, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.22507 to 0.20881, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.20881 to 0.20590, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.20590\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.20590\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.20590\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.20590\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.20590\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.20590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.18318, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.18318\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.18318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10872, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10872\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12137, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12137\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12137\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11322, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11322\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11322\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10644, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10644\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10549, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10549\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04865, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.04865 to 0.04859, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04859\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06518, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06518\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04832, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04832\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 5\n",
      "Model: \"sequential_25\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25224, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25224 to 0.22601, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.22601\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.22601\n",
      "number of correct positive predictions is 0\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.18073, saving model to best-model-conll.hdfs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:18: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.18073\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.18073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11092, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11092\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.08711, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.08711\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.08711\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10984, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10984\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11318, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11318\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11424, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11424\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11424\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05356, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05356\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06664, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06664\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04204, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04204\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 6\n",
      "Model: \"sequential_26\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.72878, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.72878 to 0.32068, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.32068 to 0.20240, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.20240\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.20240\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.20240 to 0.17458, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.17458\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.17458\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.17458\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.17458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.20865, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.20865 to 0.20350, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.20350\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.20350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12301, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12301\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10322, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10322\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10322\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10376, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10376\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10376\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10147, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10147\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10661, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10661\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05380, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05380\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06367, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06367\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06367\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06249, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06249\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06249\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 7\n",
      "Model: \"sequential_27\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.28338, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.28338 to 0.21029, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.21029\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.21029\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.21029\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.21029 to 0.19009, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.19009\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.19009\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.19009\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.19009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.17984, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.17984\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.17984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11549, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11549\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09467, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09467\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09568, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09568\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09026, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09026\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09026\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12259, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12259\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12259\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04323, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04323\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06691, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06691\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05002, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.05002 to 0.04883, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04883\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 8\n",
      "Model: \"sequential_28\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.56571, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.56571 to 0.22250, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.22250 to 0.22007, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.22007 to 0.21000, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.21000 to 0.20580, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.20580\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.20580\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.20580\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.20580\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.20580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.19235, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.19235\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.19235\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11330, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11330\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11330\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12812, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.12812 to 0.12101, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12101\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09325, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.09325\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.09325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.14544, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.14544 to 0.10911, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10911\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10768, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10768\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05018, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05018\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.08091, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.08091\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.08091\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06783, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.06783 to 0.06045, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06045\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 9\n",
      "Model: \"sequential_29\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24366, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24366 to 0.24244, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24244\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24244 to 0.21771, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.21771\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.21771\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.21771\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.21771\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.21771\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.21771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.16627, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.16627\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.16627\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10422, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10422\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10422\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11350, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11350\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11455, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11455\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10047, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10047\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10173, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10173\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04088, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04088\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06786, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06786\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.04098, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.04098\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.04098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlgoCrossValIter - 10\n",
      "Model: \"sequential_30\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hidden1 (Dense)              (None, 640)               135040    \n",
      "_________________________________________________________________\n",
      "hidden2 (Dense)              (None, 160)               102560    \n",
      "_________________________________________________________________\n",
      "outputlayer (Dense)          (None, 5)                 805       \n",
      "=================================================================\n",
      "Total params: 238,405\n",
      "Trainable params: 238,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.27203, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.27203\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.27203 to 0.23500, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.23500 to 0.21253, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.21253 to 0.19524, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.19524\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.19524 to 0.17628, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.17628\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.17628\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.17628\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.18409, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.18409\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.18409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10303, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10303\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10303\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.12327, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.12327\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.12327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.14613, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.14613 to 0.14395, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.14395\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.14395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.10607, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.10607\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.10607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.11538, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.11538\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.11538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05689, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05689\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06783, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06783\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.06358, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.06358 to 0.05871, saving model to best-model-conll.hdfs\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.05871\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.05871\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/share/miniconda3/envs/ner-projection/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "resultCrossVal, trainByTagResult, testByTagResult, ewoByTagResult, model = algoCrossVal(X, y, X_ewo, y_ewo, repeat=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mean_log'"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalization_strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>P_val</th>\n",
       "      <td>78.282000</td>\n",
       "      <td>75.075000</td>\n",
       "      <td>79.710</td>\n",
       "      <td>79.154</td>\n",
       "      <td>76.961000</td>\n",
       "      <td>80.975</td>\n",
       "      <td>83.369</td>\n",
       "      <td>79.989</td>\n",
       "      <td>82.291</td>\n",
       "      <td>81.156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_train</th>\n",
       "      <td>77.042000</td>\n",
       "      <td>84.914000</td>\n",
       "      <td>84.127</td>\n",
       "      <td>86.251</td>\n",
       "      <td>76.950000</td>\n",
       "      <td>86.880</td>\n",
       "      <td>85.426</td>\n",
       "      <td>88.491</td>\n",
       "      <td>86.356</td>\n",
       "      <td>86.243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_ewo</th>\n",
       "      <td>66.003000</td>\n",
       "      <td>70.388000</td>\n",
       "      <td>70.997</td>\n",
       "      <td>74.374</td>\n",
       "      <td>66.537000</td>\n",
       "      <td>73.928</td>\n",
       "      <td>71.995</td>\n",
       "      <td>76.143</td>\n",
       "      <td>74.662</td>\n",
       "      <td>73.364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_val</th>\n",
       "      <td>69.843000</td>\n",
       "      <td>70.026000</td>\n",
       "      <td>82.712</td>\n",
       "      <td>70.804</td>\n",
       "      <td>72.027000</td>\n",
       "      <td>70.249</td>\n",
       "      <td>77.247</td>\n",
       "      <td>63.888</td>\n",
       "      <td>70.638</td>\n",
       "      <td>72.954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_train</th>\n",
       "      <td>73.344000</td>\n",
       "      <td>72.893000</td>\n",
       "      <td>82.292</td>\n",
       "      <td>76.200</td>\n",
       "      <td>73.376000</td>\n",
       "      <td>75.981</td>\n",
       "      <td>76.772</td>\n",
       "      <td>72.639</td>\n",
       "      <td>74.374</td>\n",
       "      <td>78.362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_ewo</th>\n",
       "      <td>61.759000</td>\n",
       "      <td>61.574000</td>\n",
       "      <td>69.445</td>\n",
       "      <td>63.797</td>\n",
       "      <td>61.018000</td>\n",
       "      <td>63.425</td>\n",
       "      <td>64.075</td>\n",
       "      <td>59.353</td>\n",
       "      <td>62.036</td>\n",
       "      <td>65.186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-val</th>\n",
       "      <td>80.521111</td>\n",
       "      <td>79.398889</td>\n",
       "      <td>80.621</td>\n",
       "      <td>73.670</td>\n",
       "      <td>82.453333</td>\n",
       "      <td>73.768</td>\n",
       "      <td>79.883</td>\n",
       "      <td>69.878</td>\n",
       "      <td>75.264</td>\n",
       "      <td>75.510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-train</th>\n",
       "      <td>82.860000</td>\n",
       "      <td>75.831000</td>\n",
       "      <td>82.792</td>\n",
       "      <td>79.549</td>\n",
       "      <td>83.132222</td>\n",
       "      <td>80.140</td>\n",
       "      <td>80.283</td>\n",
       "      <td>77.336</td>\n",
       "      <td>78.578</td>\n",
       "      <td>80.879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-ewo</th>\n",
       "      <td>70.371111</td>\n",
       "      <td>63.555000</td>\n",
       "      <td>69.735</td>\n",
       "      <td>67.471</td>\n",
       "      <td>70.072222</td>\n",
       "      <td>67.342</td>\n",
       "      <td>67.178</td>\n",
       "      <td>64.467</td>\n",
       "      <td>65.752</td>\n",
       "      <td>67.812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0          0       0       0          0       0       0  \\\n",
       "P_val     78.282000  75.075000  79.710  79.154  76.961000  80.975  83.369   \n",
       "P_train   77.042000  84.914000  84.127  86.251  76.950000  86.880  85.426   \n",
       "P_ewo     66.003000  70.388000  70.997  74.374  66.537000  73.928  71.995   \n",
       "R_val     69.843000  70.026000  82.712  70.804  72.027000  70.249  77.247   \n",
       "R_train   73.344000  72.893000  82.292  76.200  73.376000  75.981  76.772   \n",
       "R_ewo     61.759000  61.574000  69.445  63.797  61.018000  63.425  64.075   \n",
       "F1-val    80.521111  79.398889  80.621  73.670  82.453333  73.768  79.883   \n",
       "F1-train  82.860000  75.831000  82.792  79.549  83.132222  80.140  80.283   \n",
       "F1-ewo    70.371111  63.555000  69.735  67.471  70.072222  67.342  67.178   \n",
       "\n",
       "               0       0       0  \n",
       "P_val     79.989  82.291  81.156  \n",
       "P_train   88.491  86.356  86.243  \n",
       "P_ewo     76.143  74.662  73.364  \n",
       "R_val     63.888  70.638  72.954  \n",
       "R_train   72.639  74.374  78.362  \n",
       "R_ewo     59.353  62.036  65.186  \n",
       "F1-val    69.878  75.264  75.510  \n",
       "F1-train  77.336  78.578  80.879  \n",
       "F1-ewo    64.467  65.752  67.812  "
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultCrossVal.to_csv(\"results.csv\")\n",
    "resultCrossVal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>P_val</th>\n",
       "      <td>79.696200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_train</th>\n",
       "      <td>84.268000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_ewo</th>\n",
       "      <td>71.839100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_val</th>\n",
       "      <td>72.038800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_train</th>\n",
       "      <td>75.623300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_ewo</th>\n",
       "      <td>63.166800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-val</th>\n",
       "      <td>77.096733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-train</th>\n",
       "      <td>80.138022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-ewo</th>\n",
       "      <td>67.375533</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0\n",
       "P_val     79.696200\n",
       "P_train   84.268000\n",
       "P_ewo     71.839100\n",
       "R_val     72.038800\n",
       "R_train   75.623300\n",
       "R_ewo     63.166800\n",
       "F1-val    77.096733\n",
       "F1-train  80.138022\n",
       "F1-ewo    67.375533"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultCrossVal.mean(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>P_val</th>\n",
       "      <td>2.476679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_train</th>\n",
       "      <td>4.005486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P_ewo</th>\n",
       "      <td>3.409437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_val</th>\n",
       "      <td>4.987530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_train</th>\n",
       "      <td>3.016970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R_ewo</th>\n",
       "      <td>2.783715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-val</th>\n",
       "      <td>4.037503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-train</th>\n",
       "      <td>2.430602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-ewo</th>\n",
       "      <td>2.303243</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "P_val     2.476679\n",
       "P_train   4.005486\n",
       "P_ewo     3.409437\n",
       "R_val     4.987530\n",
       "R_train   3.016970\n",
       "R_ewo     2.783715\n",
       "F1-val    4.037503\n",
       "F1-train  2.430602\n",
       "F1-ewo    2.303243"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultCrossVal.std(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainByTagResult.to_csv(\"results/train-by-tag.csv\")\n",
    "# trainByTagResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# trainByTagResult.mean(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainByTagResult.std(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testByTagResult.to_csv(\"results/test-by-tag.csv\")\n",
    "# testByTagResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testByTagResult.mean(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testByTagResult.std(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ewoByTagResult.to_csv(\"results/ewo-by-tag.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ewoByTagResult = pd.read_csv(\"results/ewo-by-tag.csv\", index_col=0)\n",
    "# ewoByTagResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ewoByTagResult.mean(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ewoByTagResult.std(axis=1).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns = en_fingerprints.columns\n",
    "\n",
    "# print(\"Pred\", \"Real\", \"Freq\", \"Word\", sep=\"\\t\")\n",
    "# for c in columns:\n",
    "#     prediction = model.predict(en_fingerprints[c].values.reshape((1, 210)))\n",
    "#     pred_tag = int2tag[np.argmax(prediction)]\n",
    "#     real_tag = en_corpus[en_corpus.word == c].iloc[0]['ne-tag']\n",
    "    \n",
    "#     if pred_tag != real_tag:\n",
    "#         print(pred_tag, real_tag, en_fingerprints[c].max(), c, sep=\"\\t\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "rnn-ner.ipynb",
   "provenance": [
    {
     "file_id": "1bSiRRO29rixupIV6ume9T9B4KUKtYVKI",
     "timestamp": 1513688449690
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}