change_seq_len.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/vp.shivasan/word_segmentation/working_dir/data/10K_test-v1/overall_test_chardata.txt','r') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def return_max_len(lines):\n",
    "    lengths = []\n",
    "    curr_len = 0\n",
    "    for i in range(len(lines)):\n",
    "        curr_len+=1\n",
    "        if(lines[i] == '\\n'):\n",
    "            lengths.append(curr_len)\n",
    "            curr_len = 0\n",
    "    return max(lengths)            \n",
    "            \n",
    "            \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4228"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "return_max_len(lines=lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove overflow\n",
    "curr_len = 0\n",
    "new_lines = []\n",
    "noverflow = 0\n",
    "do_not_add = False\n",
    "curr_lines = []\n",
    "skip = False\n",
    "for i in range(len(lines)):\n",
    "    if(skip):\n",
    "        if(lines[i] == \"\\n\"):\n",
    "            skip = False\n",
    "            curr_lines = []\n",
    "            curr_len = 0\n",
    "            \n",
    "        continue\n",
    "\n",
    "    curr_len+=1\n",
    "    curr_lines.append(lines[i])\n",
    "\n",
    "    if(lines[i] == '\\n'):\n",
    "        new_lines.extend(curr_lines)\n",
    "        curr_lines = []\n",
    "        curr_len = 0\n",
    "        continue\n",
    "        \n",
    "    if(curr_len>150):\n",
    "        skip = True\n",
    "        noverflow+=1\n",
    "        continue\n",
    "\n",
    "              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2297"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noverflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "549177"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1187470"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "return_max_len(new_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/vp.shivasan/word_segmentation/working_dir/data/new_training_data/character_data/overall_test_cut_150.txt','w') as f:\n",
    "    f.writelines(new_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('/home/vp.shivasan/word_segmentation/working_dir/data/10K_test-v1/overall_test_sents.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df[new_df[\"input\"].str.len() <= 150].to_csv(\"/home/vp.shivasan/word_segmentation/working_dir/data/10K_test-v1/overall_test_sents_cut_150.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('/home/vp.shivasan/word_segmentation/working_dir/data/new_training_data/sentence_data/collection_train_1_2_cut_150.csv')\n",
    "df2 = pd.read_csv('/home/vp.shivasan/word_segmentation/working_dir/data/new_training_data/sentence_data/collection_test_1_2_cut_150.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>306690</td>\n",
       "      <td>medasvino_viSoDyA_ye_ca_narAH_kuzWamehArtAH</td>\n",
       "      <td>medasvinaH_viSoDyAH_ye_ca_narAH_kuzWa_meha_ArtAH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>135745</td>\n",
       "      <td>smarantI_tasya_mandasya_kaM_vA_sAdyopatizWati</td>\n",
       "      <td>smarantI_tasya_mandasya_kam_vA_sA_adya_upatizWati</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>19559</td>\n",
       "      <td>te_vA_ete_yajQakramAH</td>\n",
       "      <td>te_vA_ete_yajQa_kramAH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>593287</td>\n",
       "      <td>imaM_navajvare_dadyAnmAzamAtraM_rasasya_tu</td>\n",
       "      <td>imam_navajvare_dadyAt_mAza_mAtram_rasasya_tu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>100331</td>\n",
       "      <td>lokayAtrA_hi_kArtsnyena_trizvetezu_pratizWitA</td>\n",
       "      <td>lokayAtrA_hi_kArtsnyena_trizu_etezu_pratizWitA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2122</th>\n",
       "      <td>2241</td>\n",
       "      <td>233756</td>\n",
       "      <td>ganDatAlakasinDUtTaM_cUlIwaFkaNaBUKagam</td>\n",
       "      <td>ganDa_tAlaka_sinDUtTam_cUlI_waFkaNa_BUKagam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2123</th>\n",
       "      <td>2242</td>\n",
       "      <td>15920</td>\n",
       "      <td>PANwAhRtimimatAByAM_NaPiQO</td>\n",
       "      <td>PANwAhRti_mimatAByAm_Na_PiQO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124</th>\n",
       "      <td>2243</td>\n",
       "      <td>93103</td>\n",
       "      <td>ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...</td>\n",
       "      <td>ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2125</th>\n",
       "      <td>2244</td>\n",
       "      <td>503422</td>\n",
       "      <td>pUrvavaddakziNe_BAge_viprAnzoxaSa_saMsTitAn</td>\n",
       "      <td>pUrvavat_dakziNe_BAge_viprAn_zoxaSa_saMsTitAn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>2245</td>\n",
       "      <td>18673</td>\n",
       "      <td>aDiSIFsTAsAM_karma</td>\n",
       "      <td>aDi_SIF_sTA_AsAm_karma</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>505255 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0     UID                                              input  \\\n",
       "0              0  306690        medasvino_viSoDyA_ye_ca_narAH_kuzWamehArtAH   \n",
       "1              1  135745      smarantI_tasya_mandasya_kaM_vA_sAdyopatizWati   \n",
       "2              2   19559                              te_vA_ete_yajQakramAH   \n",
       "3              3  593287         imaM_navajvare_dadyAnmAzamAtraM_rasasya_tu   \n",
       "4              4  100331      lokayAtrA_hi_kArtsnyena_trizvetezu_pratizWitA   \n",
       "...          ...     ...                                                ...   \n",
       "2122        2241  233756            ganDatAlakasinDUtTaM_cUlIwaFkaNaBUKagam   \n",
       "2123        2242   15920                         PANwAhRtimimatAByAM_NaPiQO   \n",
       "2124        2243   93103  ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...   \n",
       "2125        2244  503422        pUrvavaddakziNe_BAge_viprAnzoxaSa_saMsTitAn   \n",
       "2126        2245   18673                                 aDiSIFsTAsAM_karma   \n",
       "\n",
       "                                                 output  \n",
       "0      medasvinaH_viSoDyAH_ye_ca_narAH_kuzWa_meha_ArtAH  \n",
       "1     smarantI_tasya_mandasya_kam_vA_sA_adya_upatizWati  \n",
       "2                                te_vA_ete_yajQa_kramAH  \n",
       "3          imam_navajvare_dadyAt_mAza_mAtram_rasasya_tu  \n",
       "4        lokayAtrA_hi_kArtsnyena_trizu_etezu_pratizWitA  \n",
       "...                                                 ...  \n",
       "2122        ganDa_tAlaka_sinDUtTam_cUlI_waFkaNa_BUKagam  \n",
       "2123                       PANwAhRti_mimatAByAm_Na_PiQO  \n",
       "2124  ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...  \n",
       "2125      pUrvavat_dakziNe_BAge_viprAn_zoxaSa_saMsTitAn  \n",
       "2126                             aDi_SIF_sTA_AsAm_karma  \n",
       "\n",
       "[505255 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_df = pd.concat([df1,df2])\n",
    "concat_df.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>306690</td>\n",
       "      <td>medasvino_viSoDyA_ye_ca_narAH_kuzWamehArtAH</td>\n",
       "      <td>medasvinaH_viSoDyAH_ye_ca_narAH_kuzWa_meha_ArtAH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>135745</td>\n",
       "      <td>smarantI_tasya_mandasya_kaM_vA_sAdyopatizWati</td>\n",
       "      <td>smarantI_tasya_mandasya_kam_vA_sA_adya_upatizWati</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19559</td>\n",
       "      <td>te_vA_ete_yajQakramAH</td>\n",
       "      <td>te_vA_ete_yajQa_kramAH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>593287</td>\n",
       "      <td>imaM_navajvare_dadyAnmAzamAtraM_rasasya_tu</td>\n",
       "      <td>imam_navajvare_dadyAt_mAza_mAtram_rasasya_tu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100331</td>\n",
       "      <td>lokayAtrA_hi_kArtsnyena_trizvetezu_pratizWitA</td>\n",
       "      <td>lokayAtrA_hi_kArtsnyena_trizu_etezu_pratizWitA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2122</th>\n",
       "      <td>233756</td>\n",
       "      <td>ganDatAlakasinDUtTaM_cUlIwaFkaNaBUKagam</td>\n",
       "      <td>ganDa_tAlaka_sinDUtTam_cUlI_waFkaNa_BUKagam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2123</th>\n",
       "      <td>15920</td>\n",
       "      <td>PANwAhRtimimatAByAM_NaPiQO</td>\n",
       "      <td>PANwAhRti_mimatAByAm_Na_PiQO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124</th>\n",
       "      <td>93103</td>\n",
       "      <td>ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...</td>\n",
       "      <td>ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2125</th>\n",
       "      <td>503422</td>\n",
       "      <td>pUrvavaddakziNe_BAge_viprAnzoxaSa_saMsTitAn</td>\n",
       "      <td>pUrvavat_dakziNe_BAge_viprAn_zoxaSa_saMsTitAn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>18673</td>\n",
       "      <td>aDiSIFsTAsAM_karma</td>\n",
       "      <td>aDi_SIF_sTA_AsAm_karma</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>505255 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         UID                                              input  \\\n",
       "0     306690        medasvino_viSoDyA_ye_ca_narAH_kuzWamehArtAH   \n",
       "1     135745      smarantI_tasya_mandasya_kaM_vA_sAdyopatizWati   \n",
       "2      19559                              te_vA_ete_yajQakramAH   \n",
       "3     593287         imaM_navajvare_dadyAnmAzamAtraM_rasasya_tu   \n",
       "4     100331      lokayAtrA_hi_kArtsnyena_trizvetezu_pratizWitA   \n",
       "...      ...                                                ...   \n",
       "2122  233756            ganDatAlakasinDUtTaM_cUlIwaFkaNaBUKagam   \n",
       "2123   15920                         PANwAhRtimimatAByAM_NaPiQO   \n",
       "2124   93103  ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...   \n",
       "2125  503422        pUrvavaddakziNe_BAge_viprAnzoxaSa_saMsTitAn   \n",
       "2126   18673                                 aDiSIFsTAsAM_karma   \n",
       "\n",
       "                                                 output  \n",
       "0      medasvinaH_viSoDyAH_ye_ca_narAH_kuzWa_meha_ArtAH  \n",
       "1     smarantI_tasya_mandasya_kam_vA_sA_adya_upatizWati  \n",
       "2                                te_vA_ete_yajQa_kramAH  \n",
       "3          imam_navajvare_dadyAt_mAza_mAtram_rasasya_tu  \n",
       "4        lokayAtrA_hi_kArtsnyena_trizu_etezu_pratizWitA  \n",
       "...                                                 ...  \n",
       "2122        ganDa_tAlaka_sinDUtTam_cUlI_waFkaNa_BUKagam  \n",
       "2123                       PANwAhRti_mimatAByAm_Na_PiQO  \n",
       "2124  ayaM_soma_indra_tuByaM_sunva_A_tu_pra_yAhi_har...  \n",
       "2125      pUrvavat_dakziNe_BAge_viprAn_zoxaSa_saMsTitAn  \n",
       "2126                             aDi_SIF_sTA_AsAm_karma  \n",
       "\n",
       "[505255 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_df = concat_df.drop([\"Unnamed: 0\"],axis=1)\n",
    "concat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df.to_csv('/home/vp.shivasan/word_segmentation/working_dir/data/new_training_data/sentence_data/collection_1_2_cut_150.csv',index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collection3_test_df = pd.read_csv('/home/vp.shivasan/word_segmentation/working_dir/data/new_training_data/sentence_data/collection3_cut_150.csv')\n",
    "overall_test_df = pd.read_csv('/home/vp.shivasan/word_segmentation/working_dir/data/10K_test-v1/overall_test_sents_cut_150.csv')\n",
    "\n",
    "collection1_df = pd.read_csv('/home/vp.shivasan/word_segmentation/TransLIST-H22/H22-data/collection1_sent_Translist_cut_150.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>343679</td>\n",
       "      <td>tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH</td>\n",
       "      <td>tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>248181</td>\n",
       "      <td>ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet</td>\n",
       "      <td>iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>263479</td>\n",
       "      <td>tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret</td>\n",
       "      <td>tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>178206</td>\n",
       "      <td>evam_ukto_nataSirA_munir_novAca_kiMcana</td>\n",
       "      <td>evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>231495</td>\n",
       "      <td>rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH</td>\n",
       "      <td>rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8455</th>\n",
       "      <td>10745</td>\n",
       "      <td>-1</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8456</th>\n",
       "      <td>10746</td>\n",
       "      <td>-1</td>\n",
       "      <td>kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...</td>\n",
       "      <td>kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8457</th>\n",
       "      <td>10747</td>\n",
       "      <td>-1</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8458</th>\n",
       "      <td>10749</td>\n",
       "      <td>-1</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8459</th>\n",
       "      <td>10754</td>\n",
       "      <td>-1</td>\n",
       "      <td>arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...</td>\n",
       "      <td>arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>511752 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0     UID                                              input  \\\n",
       "0              0  343679           tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH   \n",
       "1              1  248181      ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet   \n",
       "2              2  263479      tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret   \n",
       "3              3  178206            evam_ukto_nataSirA_munir_novAca_kiMcana   \n",
       "4              4  231495        rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH   \n",
       "...          ...     ...                                                ...   \n",
       "8455       10745      -1  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...   \n",
       "8456       10746      -1  kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...   \n",
       "8457       10747      -1  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...   \n",
       "8458       10749      -1  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...   \n",
       "8459       10754      -1  arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...   \n",
       "\n",
       "                                                 output  \n",
       "0     tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH  \n",
       "1      iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_  \n",
       "2     tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_  \n",
       "3       evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana  \n",
       "4     rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_  \n",
       "...                                                 ...  \n",
       "8455  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...  \n",
       "8456  kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...  \n",
       "8457  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...  \n",
       "8458  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...  \n",
       "8459  arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...  \n",
       "\n",
       "[511752 rows x 4 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([collection1_df,overall_test_df])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>343679</td>\n",
       "      <td>tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH</td>\n",
       "      <td>tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>248181</td>\n",
       "      <td>ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet</td>\n",
       "      <td>iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>263479</td>\n",
       "      <td>tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret</td>\n",
       "      <td>tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>178206</td>\n",
       "      <td>evam_ukto_nataSirA_munir_novAca_kiMcana</td>\n",
       "      <td>evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231495</td>\n",
       "      <td>rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH</td>\n",
       "      <td>rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8455</th>\n",
       "      <td>-1</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8456</th>\n",
       "      <td>-1</td>\n",
       "      <td>kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...</td>\n",
       "      <td>kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8457</th>\n",
       "      <td>-1</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8458</th>\n",
       "      <td>-1</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8459</th>\n",
       "      <td>-1</td>\n",
       "      <td>arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...</td>\n",
       "      <td>arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>511752 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         UID                                              input  \\\n",
       "0     343679           tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH   \n",
       "1     248181      ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet   \n",
       "2     263479      tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret   \n",
       "3     178206            evam_ukto_nataSirA_munir_novAca_kiMcana   \n",
       "4     231495        rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH   \n",
       "...      ...                                                ...   \n",
       "8455      -1  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...   \n",
       "8456      -1  kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...   \n",
       "8457      -1  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...   \n",
       "8458      -1  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...   \n",
       "8459      -1  arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...   \n",
       "\n",
       "                                                 output  \n",
       "0     tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH  \n",
       "1      iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_  \n",
       "2     tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_  \n",
       "3       evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana  \n",
       "4     rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_  \n",
       "...                                                 ...  \n",
       "8455  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...  \n",
       "8456  kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...  \n",
       "8457  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...  \n",
       "8458  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...  \n",
       "8459  arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...  \n",
       "\n",
       "[511752 rows x 3 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(columns=[\"Unnamed: 0\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"/home/vp.shivasan/word_segmentation/TransLIST-H22/H22-data/overall_test.csv\",index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>343679</td>\n",
       "      <td>tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH</td>\n",
       "      <td>tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>248181</td>\n",
       "      <td>ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet</td>\n",
       "      <td>iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>263479</td>\n",
       "      <td>tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret</td>\n",
       "      <td>tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>178206</td>\n",
       "      <td>evam_ukto_nataSirA_munir_novAca_kiMcana</td>\n",
       "      <td>evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231495</td>\n",
       "      <td>rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH</td>\n",
       "      <td>rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8455</th>\n",
       "      <td>626032</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8456</th>\n",
       "      <td>626033</td>\n",
       "      <td>kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...</td>\n",
       "      <td>kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8457</th>\n",
       "      <td>626034</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8458</th>\n",
       "      <td>626035</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8459</th>\n",
       "      <td>626036</td>\n",
       "      <td>arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...</td>\n",
       "      <td>arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>511752 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         UID                                              input  \\\n",
       "0     343679           tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH   \n",
       "1     248181      ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet   \n",
       "2     263479      tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret   \n",
       "3     178206            evam_ukto_nataSirA_munir_novAca_kiMcana   \n",
       "4     231495        rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH   \n",
       "...      ...                                                ...   \n",
       "8455  626032  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...   \n",
       "8456  626033  kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...   \n",
       "8457  626034  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...   \n",
       "8458  626035  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...   \n",
       "8459  626036  arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...   \n",
       "\n",
       "                                                 output  \n",
       "0     tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH  \n",
       "1      iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_  \n",
       "2     tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_  \n",
       "3       evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana  \n",
       "4     rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_  \n",
       "...                                                 ...  \n",
       "8455  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...  \n",
       "8456  kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...  \n",
       "8457  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...  \n",
       "8458  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...  \n",
       "8459  arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...  \n",
       "\n",
       "[511752 rows x 3 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df[\"UID\"]==-1,\"UID\"] = list(range(620000,620000+6037))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6037"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(range(620000,620000+6037)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection_trans_df = pd.read_csv(\"/home/vp.shivasan/word_segmentation/TransLIST-H22/H22-data/collection1_sent_Translist_cut_150.csv\")\n",
    "overall_test = pd.read_csv(\"/home/vp.shivasan/word_segmentation/TransLIST-H22/H22-data/overall_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>343679</td>\n",
       "      <td>tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH</td>\n",
       "      <td>tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>248181</td>\n",
       "      <td>ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet</td>\n",
       "      <td>iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>263479</td>\n",
       "      <td>tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret</td>\n",
       "      <td>tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>178206</td>\n",
       "      <td>evam_ukto_nataSirA_munir_novAca_kiMcana</td>\n",
       "      <td>evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231495</td>\n",
       "      <td>rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH</td>\n",
       "      <td>rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511747</th>\n",
       "      <td>626032</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511748</th>\n",
       "      <td>626033</td>\n",
       "      <td>kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...</td>\n",
       "      <td>kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511749</th>\n",
       "      <td>626034</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511750</th>\n",
       "      <td>626035</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511751</th>\n",
       "      <td>626036</td>\n",
       "      <td>arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...</td>\n",
       "      <td>arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>511752 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           UID                                              input  \\\n",
       "0       343679           tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH   \n",
       "1       248181      ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet   \n",
       "2       263479      tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret   \n",
       "3       178206            evam_ukto_nataSirA_munir_novAca_kiMcana   \n",
       "4       231495        rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH   \n",
       "...        ...                                                ...   \n",
       "511747  626032  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...   \n",
       "511748  626033  kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...   \n",
       "511749  626034  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...   \n",
       "511750  626035  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...   \n",
       "511751  626036  arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...   \n",
       "\n",
       "                                                   output  \n",
       "0       tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH  \n",
       "1        iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_  \n",
       "2       tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_  \n",
       "3         evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana  \n",
       "4       rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_  \n",
       "...                                                   ...  \n",
       "511747  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...  \n",
       "511748  kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...  \n",
       "511749  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...  \n",
       "511750  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...  \n",
       "511751  arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...  \n",
       "\n",
       "[511752 rows x 3 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>343679</td>\n",
       "      <td>tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH</td>\n",
       "      <td>tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>248181</td>\n",
       "      <td>ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet</td>\n",
       "      <td>iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>263479</td>\n",
       "      <td>tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret</td>\n",
       "      <td>tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>178206</td>\n",
       "      <td>evam_ukto_nataSirA_munir_novAca_kiMcana</td>\n",
       "      <td>evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>231495</td>\n",
       "      <td>rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH</td>\n",
       "      <td>rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511747</th>\n",
       "      <td>626032</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "      <td>deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511748</th>\n",
       "      <td>626033</td>\n",
       "      <td>kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...</td>\n",
       "      <td>kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511749</th>\n",
       "      <td>626034</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "      <td>pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511750</th>\n",
       "      <td>626035</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "      <td>grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511751</th>\n",
       "      <td>626036</td>\n",
       "      <td>arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...</td>\n",
       "      <td>arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1015044 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           UID                                              input  \\\n",
       "0       343679           tarpaNo_bRMhaNo_balyo_vAtahRdroganASanaH   \n",
       "1       248181      ityevaM_syAnnirutTAnaM_sevyaM_vAritaraM_Bavet   \n",
       "2       263479      tIkzNEr_viriktaM_tadrUpEr_nirUhEH_samupAcaret   \n",
       "3       178206            evam_ukto_nataSirA_munir_novAca_kiMcana   \n",
       "4       231495        rasakABraM_kAntatAmre_BAgavRdDyA_DamettataH   \n",
       "...        ...                                                ...   \n",
       "511747  626032  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...   \n",
       "511748  626033  kroDaSokaBayaharzavizAderzyAByasUyAdEnyamAtsar...   \n",
       "511749  626034  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...   \n",
       "511750  626035  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...   \n",
       "511751  626036  arkAlarkakaraQjadvayanAgadantImayUrakaBArgIrAs...   \n",
       "\n",
       "                                                   output  \n",
       "0       tarpaNaH__bRMhaNaH__balyaH__vAta_hRd_roga_nASanaH  \n",
       "1        iti_evam_syAt_nirutTAnam_sevyam_vAritaram_Bavet_  \n",
       "2       tIkzNEH__viriktam_tad_rUpEH__nirUhEH_samupAcaret_  \n",
       "3         evam_uktaH__nata_SirAH__muniH__na_uvAca_kiMcana  \n",
       "4       rasaka_aBram_kAnta_tAmre_BAga_vRdDyA_Damet_tatas_  \n",
       "...                                                   ...  \n",
       "511747  deva_asura_ganDarva_yakza_rakzas_pitR_piSAca_n...  \n",
       "511748  kroDa_Soka_Baya_harza_vizAda_IrzyA_aByasUyA_dE...  \n",
       "511749  pravAta_nivAta_Atapa_CAyA_jyotsnA_tamas_SIta_u...  \n",
       "511750  grAma_nagara_devatAyatana_SmaSAna_catuzpaTa_sa...  \n",
       "511751  arka_alarka_karaQja_dvaya_nAgadantI_mayUraka_B...  \n",
       "\n",
       "[1015044 rows x 3 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "global_concat_df  =pd.concat([collection_trans_df,overall_test]).drop([\"Unnamed: 0\"],axis=1)\n",
    "global_concat_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ChartIE",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}