stock sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Load your data</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"stock.csv\",encoding=\"ISO-8859-1\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>some statastics</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=4101, step=1)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>A 'hindrance to operations': extracts from the...</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>Hughes' instant hit buoys Blues</td>\n",
       "      <td>Jack gets his skates on at ice-cold Alex</td>\n",
       "      <td>Chaos as Maracana builds up for United</td>\n",
       "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
       "      <td>Hungry Spurs sense rich pickings</td>\n",
       "      <td>Gunners so wide of an easy target</td>\n",
       "      <td>...</td>\n",
       "      <td>Flintoff injury piles on woe for England</td>\n",
       "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
       "      <td>Kohl's successor drawn into scandal</td>\n",
       "      <td>The difference between men and women</td>\n",
       "      <td>Sara Denver, nurse turned solicitor</td>\n",
       "      <td>Diana's landmine crusade put Tories in a panic</td>\n",
       "      <td>Yeltsin's resignation caught opposition flat-f...</td>\n",
       "      <td>Russian roulette</td>\n",
       "      <td>Sold out</td>\n",
       "      <td>Recovering a title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000-01-04</td>\n",
       "      <td>0</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>The best lake scene</td>\n",
       "      <td>Leader: German sleaze inquiry</td>\n",
       "      <td>Cheerio, boyo</td>\n",
       "      <td>The main recommendations</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>...</td>\n",
       "      <td>On the critical list</td>\n",
       "      <td>The timing of their lives</td>\n",
       "      <td>Dear doctor</td>\n",
       "      <td>Irish court halts IRA man's extradition to Nor...</td>\n",
       "      <td>Burundi peace initiative fades after rebels re...</td>\n",
       "      <td>PE points the way forward to the ECB</td>\n",
       "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
       "      <td>Jane Ratcliffe</td>\n",
       "      <td>Yet more things you wouldn't know without the ...</td>\n",
       "      <td>Millennium bug fails to bite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>Coventry caught on counter by Flo</td>\n",
       "      <td>United's rivals on the road to Rio</td>\n",
       "      <td>Thatcher issues defence before trial by video</td>\n",
       "      <td>Police help Smith lay down the law at Everton</td>\n",
       "      <td>Tale of Trautmann bears two more retellings</td>\n",
       "      <td>England on the rack</td>\n",
       "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
       "      <td>Cullinan continues his Cape monopoly</td>\n",
       "      <td>...</td>\n",
       "      <td>South Melbourne (Australia)</td>\n",
       "      <td>Necaxa (Mexico)</td>\n",
       "      <td>Real Madrid (Spain)</td>\n",
       "      <td>Raja Casablanca (Morocco)</td>\n",
       "      <td>Corinthians (Brazil)</td>\n",
       "      <td>Tony's pet project</td>\n",
       "      <td>Al Nassr (Saudi Arabia)</td>\n",
       "      <td>Ideal Holmes show</td>\n",
       "      <td>Pinochet leaves hospital after tests</td>\n",
       "      <td>Useful links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000-01-06</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilgrim knows how to progress</td>\n",
       "      <td>Thatcher facing ban</td>\n",
       "      <td>McIlroy calls for Irish fighting spirit</td>\n",
       "      <td>Leicester bin stadium blueprint</td>\n",
       "      <td>United braced for Mexican wave</td>\n",
       "      <td>Auntie back in fashion, even if the dress look...</td>\n",
       "      <td>Shoaib appeal goes to the top</td>\n",
       "      <td>Hussain hurt by 'shambles' but lays blame on e...</td>\n",
       "      <td>...</td>\n",
       "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
       "      <td>BBC worst hit as digital TV begins to bite</td>\n",
       "      <td>How much can you pay for...</td>\n",
       "      <td>Christmas glitches</td>\n",
       "      <td>Upending a table, Chopping a line and Scoring ...</td>\n",
       "      <td>Scientific evidence 'unreliable', defence claims</td>\n",
       "      <td>Fusco wins judicial review in extradition case</td>\n",
       "      <td>Rebels thwart Russian advance</td>\n",
       "      <td>Blair orders shake-up of failing NHS</td>\n",
       "      <td>Lessons of law's hard heart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>Hitches and Horlocks</td>\n",
       "      <td>Beckham off but United survive</td>\n",
       "      <td>Breast cancer screening</td>\n",
       "      <td>Alan Parker</td>\n",
       "      <td>Guardian readers: are you all whingers?</td>\n",
       "      <td>Hollywood Beyond</td>\n",
       "      <td>Ashes and diamonds</td>\n",
       "      <td>Whingers - a formidable minority</td>\n",
       "      <td>...</td>\n",
       "      <td>Most everywhere:  UDIs</td>\n",
       "      <td>Most wanted:  Chloe lunettes</td>\n",
       "      <td>Return of the cane 'completely off the agenda'</td>\n",
       "      <td>From Sleepy Hollow to Greeneland</td>\n",
       "      <td>Blunkett outlines vision for over 11s</td>\n",
       "      <td>Embattled Dobson attacks 'play now, pay later'...</td>\n",
       "      <td>Doom and the Dome</td>\n",
       "      <td>What is the north-south divide?</td>\n",
       "      <td>Aitken released from jail</td>\n",
       "      <td>Gone aloft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date  Label                                               Top1  \\\n",
       "0  2000-01-03      0  A 'hindrance to operations': extracts from the...   \n",
       "1  2000-01-04      0                                          Scorecard   \n",
       "2  2000-01-05      0                  Coventry caught on counter by Flo   \n",
       "3  2000-01-06      1                      Pilgrim knows how to progress   \n",
       "4  2000-01-07      1                               Hitches and Horlocks   \n",
       "\n",
       "                                 Top2  \\\n",
       "0                           Scorecard   \n",
       "1                 The best lake scene   \n",
       "2  United's rivals on the road to Rio   \n",
       "3                 Thatcher facing ban   \n",
       "4      Beckham off but United survive   \n",
       "\n",
       "                                            Top3  \\\n",
       "0                Hughes' instant hit buoys Blues   \n",
       "1                  Leader: German sleaze inquiry   \n",
       "2  Thatcher issues defence before trial by video   \n",
       "3        McIlroy calls for Irish fighting spirit   \n",
       "4                        Breast cancer screening   \n",
       "\n",
       "                                            Top4  \\\n",
       "0       Jack gets his skates on at ice-cold Alex   \n",
       "1                                  Cheerio, boyo   \n",
       "2  Police help Smith lay down the law at Everton   \n",
       "3                Leicester bin stadium blueprint   \n",
       "4                                    Alan Parker   \n",
       "\n",
       "                                          Top5  \\\n",
       "0       Chaos as Maracana builds up for United   \n",
       "1                     The main recommendations   \n",
       "2  Tale of Trautmann bears two more retellings   \n",
       "3               United braced for Mexican wave   \n",
       "4      Guardian readers: are you all whingers?   \n",
       "\n",
       "                                                Top6  \\\n",
       "0  Depleted Leicester prevail as Elliott spoils E...   \n",
       "1                             Has Cubie killed fees?   \n",
       "2                                England on the rack   \n",
       "3  Auntie back in fashion, even if the dress look...   \n",
       "4                                   Hollywood Beyond   \n",
       "\n",
       "                                              Top7  \\\n",
       "0                 Hungry Spurs sense rich pickings   \n",
       "1                           Has Cubie killed fees?   \n",
       "2  Pakistan retaliate with call for video of Walsh   \n",
       "3                    Shoaib appeal goes to the top   \n",
       "4                               Ashes and diamonds   \n",
       "\n",
       "                                                Top8  ...  \\\n",
       "0                  Gunners so wide of an easy target  ...   \n",
       "1                             Has Cubie killed fees?  ...   \n",
       "2               Cullinan continues his Cape monopoly  ...   \n",
       "3  Hussain hurt by 'shambles' but lays blame on e...  ...   \n",
       "4                   Whingers - a formidable minority  ...   \n",
       "\n",
       "                                               Top16  \\\n",
       "0           Flintoff injury piles on woe for England   \n",
       "1                               On the critical list   \n",
       "2                        South Melbourne (Australia)   \n",
       "3  Putin admits Yeltsin quit to give him a head s...   \n",
       "4                             Most everywhere:  UDIs   \n",
       "\n",
       "                                               Top17  \\\n",
       "0  Hunters threaten Jospin with new battle of the...   \n",
       "1                          The timing of their lives   \n",
       "2                                    Necaxa (Mexico)   \n",
       "3         BBC worst hit as digital TV begins to bite   \n",
       "4                       Most wanted:  Chloe lunettes   \n",
       "\n",
       "                                            Top18  \\\n",
       "0             Kohl's successor drawn into scandal   \n",
       "1                                     Dear doctor   \n",
       "2                             Real Madrid (Spain)   \n",
       "3                     How much can you pay for...   \n",
       "4  Return of the cane 'completely off the agenda'   \n",
       "\n",
       "                                               Top19  \\\n",
       "0               The difference between men and women   \n",
       "1  Irish court halts IRA man's extradition to Nor...   \n",
       "2                          Raja Casablanca (Morocco)   \n",
       "3                                 Christmas glitches   \n",
       "4                   From Sleepy Hollow to Greeneland   \n",
       "\n",
       "                                               Top20  \\\n",
       "0                Sara Denver, nurse turned solicitor   \n",
       "1  Burundi peace initiative fades after rebels re...   \n",
       "2                               Corinthians (Brazil)   \n",
       "3  Upending a table, Chopping a line and Scoring ...   \n",
       "4              Blunkett outlines vision for over 11s   \n",
       "\n",
       "                                               Top21  \\\n",
       "0     Diana's landmine crusade put Tories in a panic   \n",
       "1               PE points the way forward to the ECB   \n",
       "2                                 Tony's pet project   \n",
       "3   Scientific evidence 'unreliable', defence claims   \n",
       "4  Embattled Dobson attacks 'play now, pay later'...   \n",
       "\n",
       "                                               Top22  \\\n",
       "0  Yeltsin's resignation caught opposition flat-f...   \n",
       "1  Campaigners keep up pressure on Nazi war crime...   \n",
       "2                            Al Nassr (Saudi Arabia)   \n",
       "3     Fusco wins judicial review in extradition case   \n",
       "4                                  Doom and the Dome   \n",
       "\n",
       "                             Top23  \\\n",
       "0                 Russian roulette   \n",
       "1                   Jane Ratcliffe   \n",
       "2                Ideal Holmes show   \n",
       "3    Rebels thwart Russian advance   \n",
       "4  What is the north-south divide?   \n",
       "\n",
       "                                               Top24  \\\n",
       "0                                           Sold out   \n",
       "1  Yet more things you wouldn't know without the ...   \n",
       "2               Pinochet leaves hospital after tests   \n",
       "3               Blair orders shake-up of failing NHS   \n",
       "4                          Aitken released from jail   \n",
       "\n",
       "                          Top25  \n",
       "0            Recovering a title  \n",
       "1  Millennium bug fails to bite  \n",
       "2                  Useful links  \n",
       "3   Lessons of law's hard heart  \n",
       "4                    Gone aloft  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "headline = []\n",
    "for row in range(len(df.index)):\n",
    "    headline.append(''.join(str(x) for x in df.iloc[row,:25]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"2000-01-030A 'hindrance to operations': extracts from the leaked reportsScorecardHughes' instant hit buoys BluesJack gets his skates on at ice-cold AlexChaos as Maracana builds up for UnitedDepleted Leicester prevail as Elliott spoils Everton's partyHungry Spurs sense rich pickingsGunners so wide of an easy targetDerby raise a glass to Strupar's debut doubleSouthgate strikes, Leeds pay the penaltyHammers hand Robson a youthful lessonSaints party like it's 1999Wear wolves have turned into lambsStump mike catches testy Gough's tauntLanger escapes to hit 167Flintoff injury piles on woe for EnglandHunters threaten Jospin with new battle of the SommeKohl's successor drawn into scandalThe difference between men and womenSara Denver, nurse turned solicitorDiana's landmine crusade put Tories in a panicYeltsin's resignation caught opposition flat-footedRussian roulette\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headline[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4101 entries, 0 to 4100\n",
      "Data columns (total 27 columns):\n",
      "Date     4101 non-null object\n",
      "Label    4101 non-null int64\n",
      "Top1     4101 non-null object\n",
      "Top2     4101 non-null object\n",
      "Top3     4101 non-null object\n",
      "Top4     4101 non-null object\n",
      "Top5     4101 non-null object\n",
      "Top6     4101 non-null object\n",
      "Top7     4101 non-null object\n",
      "Top8     4101 non-null object\n",
      "Top9     4101 non-null object\n",
      "Top10    4101 non-null object\n",
      "Top11    4101 non-null object\n",
      "Top12    4101 non-null object\n",
      "Top13    4101 non-null object\n",
      "Top14    4101 non-null object\n",
      "Top15    4101 non-null object\n",
      "Top16    4101 non-null object\n",
      "Top17    4101 non-null object\n",
      "Top18    4101 non-null object\n",
      "Top19    4101 non-null object\n",
      "Top20    4101 non-null object\n",
      "Top21    4101 non-null object\n",
      "Top22    4101 non-null object\n",
      "Top23    4100 non-null object\n",
      "Top24    4098 non-null object\n",
      "Top25    4098 non-null object\n",
      "dtypes: int64(1), object(26)\n",
      "memory usage: 865.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Divide it in to train and test</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df[df['Date']<'20150101']\n",
    "test = df[df['Date']>'20141231']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>1</td>\n",
       "      <td>Most cases of cancer are the result of sheer b...</td>\n",
       "      <td>Iran dismissed United States efforts to fight ...</td>\n",
       "      <td>Poll: One in 8 Germans would join anti-Muslim ...</td>\n",
       "      <td>UK royal family's Prince Andrew named in US la...</td>\n",
       "      <td>Some 40 asylum-seekers refused to leave the bu...</td>\n",
       "      <td>Pakistani boat blows self up after India navy ...</td>\n",
       "      <td>Sweden hit by third mosque arson attack in a week</td>\n",
       "      <td>940 cars set alight during French New Year</td>\n",
       "      <td>...</td>\n",
       "      <td>Ukrainian minister threatens TV channel with c...</td>\n",
       "      <td>Palestinian President Mahmoud Abbas has entere...</td>\n",
       "      <td>Israeli security center publishes names of 50 ...</td>\n",
       "      <td>The year 2014 was the deadliest year yet in Sy...</td>\n",
       "      <td>A Secret underground complex built by the Nazi...</td>\n",
       "      <td>Restrictions on Web Freedom a Major Global Iss...</td>\n",
       "      <td>Austrian journalist Erich Mchel delivered a pr...</td>\n",
       "      <td>Thousands of Ukraine nationalists march in Kiev</td>\n",
       "      <td>Chinas New Years Resolution: No More Harvestin...</td>\n",
       "      <td>Authorities Pull Plug on Russia's Last Politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>2015-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>Moscow-&amp;gt;Beijing high speed train will reduc...</td>\n",
       "      <td>Two ancient tombs were discovered in Egypt on ...</td>\n",
       "      <td>China complains to Pyongyang after N Korean so...</td>\n",
       "      <td>Scotland Headed Towards Being Fossil Fuel-Free...</td>\n",
       "      <td>Prime Minister Shinzo Abe said Monday he will ...</td>\n",
       "      <td>Sex slave at centre of Prince Andrew scandal f...</td>\n",
       "      <td>Gay relative of Hamas founder faces deportatio...</td>\n",
       "      <td>The number of female drug addicts in Iran has ...</td>\n",
       "      <td>...</td>\n",
       "      <td>The Islamic State has approved a 2015 budget o...</td>\n",
       "      <td>Iceland To Withdraw EU Application, Lift Capit...</td>\n",
       "      <td>Blackfield Capital Founder Goes Missing: The v...</td>\n",
       "      <td>Rocket stage crashes back to Earth in rural Ch...</td>\n",
       "      <td>2 Dead as Aircraft Bombs Greek Tanker in Libya...</td>\n",
       "      <td>Belgian murderer Frank Van Den Bleeken to die ...</td>\n",
       "      <td>Czech President criticizes Ukrainian PM; says ...</td>\n",
       "      <td>3 Vietnamese jets join search for 16 missing F...</td>\n",
       "      <td>France seeks end to Russia sanctions over Ukraine</td>\n",
       "      <td>China scraps rare earths caps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>2015-01-06</td>\n",
       "      <td>0</td>\n",
       "      <td>US oil falls below $50 a barrel</td>\n",
       "      <td>Toyota gives away 5,680 fuel cell patents to b...</td>\n",
       "      <td>Young Indian couple who had been granted polic...</td>\n",
       "      <td>A senior figure in Islamic States self-declare...</td>\n",
       "      <td>Fukushima rice passes radiation tests for 1st ...</td>\n",
       "      <td>Nearly all Spanish parties guilty of financial...</td>\n",
       "      <td>King Abdullah to abdicate Saudi Throne</td>\n",
       "      <td>Taliban Commander Caught Networking On LinkedIn</td>\n",
       "      <td>...</td>\n",
       "      <td>Thousands of Indians have fled from their home...</td>\n",
       "      <td>Turkey sacks judges who oversaw Erdogan corrup...</td>\n",
       "      <td>SpaceX Falcon 9 launch and recovery has been a...</td>\n",
       "      <td>CNN: Americans charged in botched Gambia coup</td>\n",
       "      <td>Islamic State 'Police' Official Beheaded.</td>\n",
       "      <td>Libya bans Palestinians from country to preven...</td>\n",
       "      <td>A judicial inquiry was opened in France on Mon...</td>\n",
       "      <td>Video has captured the moment a cameraman was ...</td>\n",
       "      <td>Syria has complained to the United Nations tha...</td>\n",
       "      <td>Tests over, India set to make the iris of bigg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>2015-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>'Shots fired' at French magazine HQ</td>\n",
       "      <td>90% of Bibi Netanyahus Campaign Funds From U.S.</td>\n",
       "      <td>ISIS behead street magician for entertaining c...</td>\n",
       "      <td>Denmark sets new world record in renewable ene...</td>\n",
       "      <td>ISIS Closes Schools In Syria, Leaving 670000 C...</td>\n",
       "      <td>The European Union wants Turkey to explain how...</td>\n",
       "      <td>10 people have died in a shooting at the headq...</td>\n",
       "      <td>Most Earth-like planet ever discovered could b...</td>\n",
       "      <td>...</td>\n",
       "      <td>Discovery of witnesss mutilated body feeds cla...</td>\n",
       "      <td>Radical Islam a growing threat to sub-Saharan ...</td>\n",
       "      <td>Tony Blair may 'face war crime charges' over I...</td>\n",
       "      <td>Benjamin Netanyahu says settlements unlikely t...</td>\n",
       "      <td>Tasmania moves to allow corporations to sue pr...</td>\n",
       "      <td>Indonesia search chief: Tail section of AirAsi...</td>\n",
       "      <td>North Korean Defector: \"Bureau 121\" Hackers lo...</td>\n",
       "      <td>Female suicide bomber attacks police station i...</td>\n",
       "      <td>IS to turn Syria schools into religious indoct...</td>\n",
       "      <td>Ebola vaccine trials in Halifax so far a success</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>2015-01-08</td>\n",
       "      <td>1</td>\n",
       "      <td>New Charlie Hebdo issue to come out next week:...</td>\n",
       "      <td>Ahmed Merabet, Cop Killed In Paris Attacks, Wa...</td>\n",
       "      <td>Youngest Suspect in Charlie Hebdo Attack Turns...</td>\n",
       "      <td>Grenades thrown at a mosque in Le Mans, west o...</td>\n",
       "      <td>Danish newspaper to print Charlie Hebdo cartoo...</td>\n",
       "      <td>Muslim politician from India offers $8M to Cha...</td>\n",
       "      <td>Vladimir Putin has condemned a deadly terroris...</td>\n",
       "      <td>Turkey is trying to hide any evidence that hol...</td>\n",
       "      <td>...</td>\n",
       "      <td>Thousands in Europe hold up Pens in Solidarity...</td>\n",
       "      <td>Journalists race to show solidarity with 'Char...</td>\n",
       "      <td>Antibiotics: US discovery labelled 'game-chang...</td>\n",
       "      <td>The Charlie Hebdo attack condemned by Saudi Ar...</td>\n",
       "      <td>Brisbane man wearing \"I'm with stupid\" t-shirt...</td>\n",
       "      <td>Fourth Russian General Commits Suicide in Less...</td>\n",
       "      <td>Meteorite turns night into day over Bucharest,...</td>\n",
       "      <td>Israeli doctors implant Syrian man with titani...</td>\n",
       "      <td>Unusual Metal Recovered from Ancient Greek Shi...</td>\n",
       "      <td>Young mother let terrorists into Charlie Hebdo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date  Label                                               Top1  \\\n",
       "3723  2015-01-02      1  Most cases of cancer are the result of sheer b...   \n",
       "3724  2015-01-05      0  Moscow-&gt;Beijing high speed train will reduc...   \n",
       "3725  2015-01-06      0                    US oil falls below $50 a barrel   \n",
       "3726  2015-01-07      1                'Shots fired' at French magazine HQ   \n",
       "3727  2015-01-08      1  New Charlie Hebdo issue to come out next week:...   \n",
       "\n",
       "                                                   Top2  \\\n",
       "3723  Iran dismissed United States efforts to fight ...   \n",
       "3724  Two ancient tombs were discovered in Egypt on ...   \n",
       "3725  Toyota gives away 5,680 fuel cell patents to b...   \n",
       "3726    90% of Bibi Netanyahus Campaign Funds From U.S.   \n",
       "3727  Ahmed Merabet, Cop Killed In Paris Attacks, Wa...   \n",
       "\n",
       "                                                   Top3  \\\n",
       "3723  Poll: One in 8 Germans would join anti-Muslim ...   \n",
       "3724  China complains to Pyongyang after N Korean so...   \n",
       "3725  Young Indian couple who had been granted polic...   \n",
       "3726  ISIS behead street magician for entertaining c...   \n",
       "3727  Youngest Suspect in Charlie Hebdo Attack Turns...   \n",
       "\n",
       "                                                   Top4  \\\n",
       "3723  UK royal family's Prince Andrew named in US la...   \n",
       "3724  Scotland Headed Towards Being Fossil Fuel-Free...   \n",
       "3725  A senior figure in Islamic States self-declare...   \n",
       "3726  Denmark sets new world record in renewable ene...   \n",
       "3727  Grenades thrown at a mosque in Le Mans, west o...   \n",
       "\n",
       "                                                   Top5  \\\n",
       "3723  Some 40 asylum-seekers refused to leave the bu...   \n",
       "3724  Prime Minister Shinzo Abe said Monday he will ...   \n",
       "3725  Fukushima rice passes radiation tests for 1st ...   \n",
       "3726  ISIS Closes Schools In Syria, Leaving 670000 C...   \n",
       "3727  Danish newspaper to print Charlie Hebdo cartoo...   \n",
       "\n",
       "                                                   Top6  \\\n",
       "3723  Pakistani boat blows self up after India navy ...   \n",
       "3724  Sex slave at centre of Prince Andrew scandal f...   \n",
       "3725  Nearly all Spanish parties guilty of financial...   \n",
       "3726  The European Union wants Turkey to explain how...   \n",
       "3727  Muslim politician from India offers $8M to Cha...   \n",
       "\n",
       "                                                   Top7  \\\n",
       "3723  Sweden hit by third mosque arson attack in a week   \n",
       "3724  Gay relative of Hamas founder faces deportatio...   \n",
       "3725             King Abdullah to abdicate Saudi Throne   \n",
       "3726  10 people have died in a shooting at the headq...   \n",
       "3727  Vladimir Putin has condemned a deadly terroris...   \n",
       "\n",
       "                                                   Top8  ...  \\\n",
       "3723         940 cars set alight during French New Year  ...   \n",
       "3724  The number of female drug addicts in Iran has ...  ...   \n",
       "3725    Taliban Commander Caught Networking On LinkedIn  ...   \n",
       "3726  Most Earth-like planet ever discovered could b...  ...   \n",
       "3727  Turkey is trying to hide any evidence that hol...  ...   \n",
       "\n",
       "                                                  Top16  \\\n",
       "3723  Ukrainian minister threatens TV channel with c...   \n",
       "3724  The Islamic State has approved a 2015 budget o...   \n",
       "3725  Thousands of Indians have fled from their home...   \n",
       "3726  Discovery of witnesss mutilated body feeds cla...   \n",
       "3727  Thousands in Europe hold up Pens in Solidarity...   \n",
       "\n",
       "                                                  Top17  \\\n",
       "3723  Palestinian President Mahmoud Abbas has entere...   \n",
       "3724  Iceland To Withdraw EU Application, Lift Capit...   \n",
       "3725  Turkey sacks judges who oversaw Erdogan corrup...   \n",
       "3726  Radical Islam a growing threat to sub-Saharan ...   \n",
       "3727  Journalists race to show solidarity with 'Char...   \n",
       "\n",
       "                                                  Top18  \\\n",
       "3723  Israeli security center publishes names of 50 ...   \n",
       "3724  Blackfield Capital Founder Goes Missing: The v...   \n",
       "3725  SpaceX Falcon 9 launch and recovery has been a...   \n",
       "3726  Tony Blair may 'face war crime charges' over I...   \n",
       "3727  Antibiotics: US discovery labelled 'game-chang...   \n",
       "\n",
       "                                                  Top19  \\\n",
       "3723  The year 2014 was the deadliest year yet in Sy...   \n",
       "3724  Rocket stage crashes back to Earth in rural Ch...   \n",
       "3725      CNN: Americans charged in botched Gambia coup   \n",
       "3726  Benjamin Netanyahu says settlements unlikely t...   \n",
       "3727  The Charlie Hebdo attack condemned by Saudi Ar...   \n",
       "\n",
       "                                                  Top20  \\\n",
       "3723  A Secret underground complex built by the Nazi...   \n",
       "3724  2 Dead as Aircraft Bombs Greek Tanker in Libya...   \n",
       "3725          Islamic State 'Police' Official Beheaded.   \n",
       "3726  Tasmania moves to allow corporations to sue pr...   \n",
       "3727  Brisbane man wearing \"I'm with stupid\" t-shirt...   \n",
       "\n",
       "                                                  Top21  \\\n",
       "3723  Restrictions on Web Freedom a Major Global Iss...   \n",
       "3724  Belgian murderer Frank Van Den Bleeken to die ...   \n",
       "3725  Libya bans Palestinians from country to preven...   \n",
       "3726  Indonesia search chief: Tail section of AirAsi...   \n",
       "3727  Fourth Russian General Commits Suicide in Less...   \n",
       "\n",
       "                                                  Top22  \\\n",
       "3723  Austrian journalist Erich Mchel delivered a pr...   \n",
       "3724  Czech President criticizes Ukrainian PM; says ...   \n",
       "3725  A judicial inquiry was opened in France on Mon...   \n",
       "3726  North Korean Defector: \"Bureau 121\" Hackers lo...   \n",
       "3727  Meteorite turns night into day over Bucharest,...   \n",
       "\n",
       "                                                  Top23  \\\n",
       "3723    Thousands of Ukraine nationalists march in Kiev   \n",
       "3724  3 Vietnamese jets join search for 16 missing F...   \n",
       "3725  Video has captured the moment a cameraman was ...   \n",
       "3726  Female suicide bomber attacks police station i...   \n",
       "3727  Israeli doctors implant Syrian man with titani...   \n",
       "\n",
       "                                                  Top24  \\\n",
       "3723  Chinas New Years Resolution: No More Harvestin...   \n",
       "3724  France seeks end to Russia sanctions over Ukraine   \n",
       "3725  Syria has complained to the United Nations tha...   \n",
       "3726  IS to turn Syria schools into religious indoct...   \n",
       "3727  Unusual Metal Recovered from Ancient Greek Shi...   \n",
       "\n",
       "                                                  Top25  \n",
       "3723  Authorities Pull Plug on Russia's Last Politic...  \n",
       "3724                      China scraps rare earths caps  \n",
       "3725  Tests over, India set to make the iris of bigg...  \n",
       "3726   Ebola vaccine trials in Halifax so far a success  \n",
       "3727  Young mother let terrorists into Charlie Hebdo...  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9.217264081931237, 96.92757863935626)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "percent_test = (len(test)/len(df))*100 \n",
    "percent_train = (len(train)/len(df))*100 \n",
    "percent_test,percent_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Removing Punctuation and converting it into lower case</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vishal/anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>A  hindrance to operations   extracts from the...</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>Hughes  instant hit buoys Blues</td>\n",
       "      <td>Jack gets his skates on at ice cold Alex</td>\n",
       "      <td>Chaos as Maracana builds up for United</td>\n",
       "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
       "      <td>Hungry Spurs sense rich pickings</td>\n",
       "      <td>Gunners so wide of an easy target</td>\n",
       "      <td>...</td>\n",
       "      <td>Flintoff injury piles on woe for England</td>\n",
       "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
       "      <td>Kohl s successor drawn into scandal</td>\n",
       "      <td>The difference between men and women</td>\n",
       "      <td>Sara Denver  nurse turned solicitor</td>\n",
       "      <td>Diana s landmine crusade put Tories in a panic</td>\n",
       "      <td>Yeltsin s resignation caught opposition flat f...</td>\n",
       "      <td>Russian roulette</td>\n",
       "      <td>Sold out</td>\n",
       "      <td>Recovering a title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000-01-04</td>\n",
       "      <td>0</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>The best lake scene</td>\n",
       "      <td>Leader  German sleaze inquiry</td>\n",
       "      <td>Cheerio  boyo</td>\n",
       "      <td>The main recommendations</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>...</td>\n",
       "      <td>On the critical list</td>\n",
       "      <td>The timing of their lives</td>\n",
       "      <td>Dear doctor</td>\n",
       "      <td>Irish court halts IRA man s extradition to Nor...</td>\n",
       "      <td>Burundi peace initiative fades after rebels re...</td>\n",
       "      <td>PE points the way forward to the ECB</td>\n",
       "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
       "      <td>Jane Ratcliffe</td>\n",
       "      <td>Yet more things you wouldn t know without the ...</td>\n",
       "      <td>Millennium bug fails to bite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>Coventry caught on counter by Flo</td>\n",
       "      <td>United s rivals on the road to Rio</td>\n",
       "      <td>Thatcher issues defence before trial by video</td>\n",
       "      <td>Police help Smith lay down the law at Everton</td>\n",
       "      <td>Tale of Trautmann bears two more retellings</td>\n",
       "      <td>England on the rack</td>\n",
       "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
       "      <td>Cullinan continues his Cape monopoly</td>\n",
       "      <td>...</td>\n",
       "      <td>South Melbourne  Australia</td>\n",
       "      <td>Necaxa  Mexico</td>\n",
       "      <td>Real Madrid  Spain</td>\n",
       "      <td>Raja Casablanca  Morocco</td>\n",
       "      <td>Corinthians  Brazil</td>\n",
       "      <td>Tony s pet project</td>\n",
       "      <td>Al Nassr  Saudi Arabia</td>\n",
       "      <td>Ideal Holmes show</td>\n",
       "      <td>Pinochet leaves hospital after tests</td>\n",
       "      <td>Useful links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000-01-06</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilgrim knows how to progress</td>\n",
       "      <td>Thatcher facing ban</td>\n",
       "      <td>McIlroy calls for Irish fighting spirit</td>\n",
       "      <td>Leicester bin stadium blueprint</td>\n",
       "      <td>United braced for Mexican wave</td>\n",
       "      <td>Auntie back in fashion  even if the dress look...</td>\n",
       "      <td>Shoaib appeal goes to the top</td>\n",
       "      <td>Hussain hurt by  shambles  but lays blame on e...</td>\n",
       "      <td>...</td>\n",
       "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
       "      <td>BBC worst hit as digital TV begins to bite</td>\n",
       "      <td>How much can you pay for</td>\n",
       "      <td>Christmas glitches</td>\n",
       "      <td>Upending a table  Chopping a line and Scoring ...</td>\n",
       "      <td>Scientific evidence  unreliable   defence claims</td>\n",
       "      <td>Fusco wins judicial review in extradition case</td>\n",
       "      <td>Rebels thwart Russian advance</td>\n",
       "      <td>Blair orders shake up of failing NHS</td>\n",
       "      <td>Lessons of law s hard heart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>Hitches and Horlocks</td>\n",
       "      <td>Beckham off but United survive</td>\n",
       "      <td>Breast cancer screening</td>\n",
       "      <td>Alan Parker</td>\n",
       "      <td>Guardian readers  are you all whingers</td>\n",
       "      <td>Hollywood Beyond</td>\n",
       "      <td>Ashes and diamonds</td>\n",
       "      <td>Whingers   a formidable minority</td>\n",
       "      <td>...</td>\n",
       "      <td>Most everywhere   UDIs</td>\n",
       "      <td>Most wanted   Chloe lunettes</td>\n",
       "      <td>Return of the cane  completely off the agenda</td>\n",
       "      <td>From Sleepy Hollow to Greeneland</td>\n",
       "      <td>Blunkett outlines vision for over   s</td>\n",
       "      <td>Embattled Dobson attacks  play now  pay later ...</td>\n",
       "      <td>Doom and the Dome</td>\n",
       "      <td>What is the north south divide</td>\n",
       "      <td>Aitken released from jail</td>\n",
       "      <td>Gone aloft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date  Label                                               Top1  \\\n",
       "0  2000-01-03      0  A  hindrance to operations   extracts from the...   \n",
       "1  2000-01-04      0                                          Scorecard   \n",
       "2  2000-01-05      0                  Coventry caught on counter by Flo   \n",
       "3  2000-01-06      1                      Pilgrim knows how to progress   \n",
       "4  2000-01-07      1                               Hitches and Horlocks   \n",
       "\n",
       "                                 Top2  \\\n",
       "0                           Scorecard   \n",
       "1                 The best lake scene   \n",
       "2  United s rivals on the road to Rio   \n",
       "3                 Thatcher facing ban   \n",
       "4      Beckham off but United survive   \n",
       "\n",
       "                                            Top3  \\\n",
       "0                Hughes  instant hit buoys Blues   \n",
       "1                  Leader  German sleaze inquiry   \n",
       "2  Thatcher issues defence before trial by video   \n",
       "3        McIlroy calls for Irish fighting spirit   \n",
       "4                        Breast cancer screening   \n",
       "\n",
       "                                            Top4  \\\n",
       "0       Jack gets his skates on at ice cold Alex   \n",
       "1                                  Cheerio  boyo   \n",
       "2  Police help Smith lay down the law at Everton   \n",
       "3                Leicester bin stadium blueprint   \n",
       "4                                    Alan Parker   \n",
       "\n",
       "                                          Top5  \\\n",
       "0       Chaos as Maracana builds up for United   \n",
       "1                     The main recommendations   \n",
       "2  Tale of Trautmann bears two more retellings   \n",
       "3               United braced for Mexican wave   \n",
       "4      Guardian readers  are you all whingers    \n",
       "\n",
       "                                                Top6  \\\n",
       "0  Depleted Leicester prevail as Elliott spoils E...   \n",
       "1                             Has Cubie killed fees    \n",
       "2                                England on the rack   \n",
       "3  Auntie back in fashion  even if the dress look...   \n",
       "4                                   Hollywood Beyond   \n",
       "\n",
       "                                              Top7  \\\n",
       "0                 Hungry Spurs sense rich pickings   \n",
       "1                           Has Cubie killed fees    \n",
       "2  Pakistan retaliate with call for video of Walsh   \n",
       "3                    Shoaib appeal goes to the top   \n",
       "4                               Ashes and diamonds   \n",
       "\n",
       "                                                Top8  ...  \\\n",
       "0                  Gunners so wide of an easy target  ...   \n",
       "1                             Has Cubie killed fees   ...   \n",
       "2               Cullinan continues his Cape monopoly  ...   \n",
       "3  Hussain hurt by  shambles  but lays blame on e...  ...   \n",
       "4                   Whingers   a formidable minority  ...   \n",
       "\n",
       "                                               Top16  \\\n",
       "0           Flintoff injury piles on woe for England   \n",
       "1                               On the critical list   \n",
       "2                        South Melbourne  Australia    \n",
       "3  Putin admits Yeltsin quit to give him a head s...   \n",
       "4                             Most everywhere   UDIs   \n",
       "\n",
       "                                               Top17  \\\n",
       "0  Hunters threaten Jospin with new battle of the...   \n",
       "1                          The timing of their lives   \n",
       "2                                    Necaxa  Mexico    \n",
       "3         BBC worst hit as digital TV begins to bite   \n",
       "4                       Most wanted   Chloe lunettes   \n",
       "\n",
       "                                            Top18  \\\n",
       "0             Kohl s successor drawn into scandal   \n",
       "1                                     Dear doctor   \n",
       "2                             Real Madrid  Spain    \n",
       "3                     How much can you pay for      \n",
       "4  Return of the cane  completely off the agenda    \n",
       "\n",
       "                                               Top19  \\\n",
       "0               The difference between men and women   \n",
       "1  Irish court halts IRA man s extradition to Nor...   \n",
       "2                          Raja Casablanca  Morocco    \n",
       "3                                 Christmas glitches   \n",
       "4                   From Sleepy Hollow to Greeneland   \n",
       "\n",
       "                                               Top20  \\\n",
       "0                Sara Denver  nurse turned solicitor   \n",
       "1  Burundi peace initiative fades after rebels re...   \n",
       "2                               Corinthians  Brazil    \n",
       "3  Upending a table  Chopping a line and Scoring ...   \n",
       "4              Blunkett outlines vision for over   s   \n",
       "\n",
       "                                               Top21  \\\n",
       "0     Diana s landmine crusade put Tories in a panic   \n",
       "1               PE points the way forward to the ECB   \n",
       "2                                 Tony s pet project   \n",
       "3   Scientific evidence  unreliable   defence claims   \n",
       "4  Embattled Dobson attacks  play now  pay later ...   \n",
       "\n",
       "                                               Top22  \\\n",
       "0  Yeltsin s resignation caught opposition flat f...   \n",
       "1  Campaigners keep up pressure on Nazi war crime...   \n",
       "2                            Al Nassr  Saudi Arabia    \n",
       "3     Fusco wins judicial review in extradition case   \n",
       "4                                  Doom and the Dome   \n",
       "\n",
       "                             Top23  \\\n",
       "0                 Russian roulette   \n",
       "1                   Jane Ratcliffe   \n",
       "2                Ideal Holmes show   \n",
       "3    Rebels thwart Russian advance   \n",
       "4  What is the north south divide    \n",
       "\n",
       "                                               Top24  \\\n",
       "0                                           Sold out   \n",
       "1  Yet more things you wouldn t know without the ...   \n",
       "2               Pinochet leaves hospital after tests   \n",
       "3               Blair orders shake up of failing NHS   \n",
       "4                          Aitken released from jail   \n",
       "\n",
       "                          Top25  \n",
       "0            Recovering a title  \n",
       "1  Millennium bug fails to bite  \n",
       "2                  Useful links  \n",
       "3   Lessons of law s hard heart  \n",
       "4                    Gone aloft  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined = [train,test]\n",
    "\n",
    "for data in combined:\n",
    "    data1 = data.iloc[:,2:27]\n",
    "    data1.replace('[^A-Za-z]',\" \",regex=True,inplace=True)\n",
    "    data.loc[:,2:27] = data1.loc[:,:]\n",
    "    \n",
    "train.head()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for data in combined:\n",
    "    data1 = data.iloc[:,2:27]\n",
    "    for col in data1.columns:\n",
    "        data1[col] = data1[col].str.lower()\n",
    "    data.loc[:,2:27] = data1.loc[:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "headline1= []\n",
    "for row in range(len(train.index)):\n",
    "    headline1.append(''.join(str(x) for x in train.iloc[row,:]))\n",
    "    \n",
    "headline2= []\n",
    "for row in range(len(test.index)):\n",
    "    headline2.append(''.join(str(x) for x in test.iloc[row,:]))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>a  hindrance to operations   extracts from the...</td>\n",
       "      <td>scorecard</td>\n",
       "      <td>hughes  instant hit buoys blues</td>\n",
       "      <td>jack gets his skates on at ice cold alex</td>\n",
       "      <td>chaos as maracana builds up for united</td>\n",
       "      <td>depleted leicester prevail as elliott spoils e...</td>\n",
       "      <td>hungry spurs sense rich pickings</td>\n",
       "      <td>gunners so wide of an easy target</td>\n",
       "      <td>...</td>\n",
       "      <td>flintoff injury piles on woe for england</td>\n",
       "      <td>hunters threaten jospin with new battle of the...</td>\n",
       "      <td>kohl s successor drawn into scandal</td>\n",
       "      <td>the difference between men and women</td>\n",
       "      <td>sara denver  nurse turned solicitor</td>\n",
       "      <td>diana s landmine crusade put tories in a panic</td>\n",
       "      <td>yeltsin s resignation caught opposition flat f...</td>\n",
       "      <td>russian roulette</td>\n",
       "      <td>sold out</td>\n",
       "      <td>recovering a title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000-01-04</td>\n",
       "      <td>0</td>\n",
       "      <td>scorecard</td>\n",
       "      <td>the best lake scene</td>\n",
       "      <td>leader  german sleaze inquiry</td>\n",
       "      <td>cheerio  boyo</td>\n",
       "      <td>the main recommendations</td>\n",
       "      <td>has cubie killed fees</td>\n",
       "      <td>has cubie killed fees</td>\n",
       "      <td>has cubie killed fees</td>\n",
       "      <td>...</td>\n",
       "      <td>on the critical list</td>\n",
       "      <td>the timing of their lives</td>\n",
       "      <td>dear doctor</td>\n",
       "      <td>irish court halts ira man s extradition to nor...</td>\n",
       "      <td>burundi peace initiative fades after rebels re...</td>\n",
       "      <td>pe points the way forward to the ecb</td>\n",
       "      <td>campaigners keep up pressure on nazi war crime...</td>\n",
       "      <td>jane ratcliffe</td>\n",
       "      <td>yet more things you wouldn t know without the ...</td>\n",
       "      <td>millennium bug fails to bite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>coventry caught on counter by flo</td>\n",
       "      <td>united s rivals on the road to rio</td>\n",
       "      <td>thatcher issues defence before trial by video</td>\n",
       "      <td>police help smith lay down the law at everton</td>\n",
       "      <td>tale of trautmann bears two more retellings</td>\n",
       "      <td>england on the rack</td>\n",
       "      <td>pakistan retaliate with call for video of walsh</td>\n",
       "      <td>cullinan continues his cape monopoly</td>\n",
       "      <td>...</td>\n",
       "      <td>south melbourne  australia</td>\n",
       "      <td>necaxa  mexico</td>\n",
       "      <td>real madrid  spain</td>\n",
       "      <td>raja casablanca  morocco</td>\n",
       "      <td>corinthians  brazil</td>\n",
       "      <td>tony s pet project</td>\n",
       "      <td>al nassr  saudi arabia</td>\n",
       "      <td>ideal holmes show</td>\n",
       "      <td>pinochet leaves hospital after tests</td>\n",
       "      <td>useful links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000-01-06</td>\n",
       "      <td>1</td>\n",
       "      <td>pilgrim knows how to progress</td>\n",
       "      <td>thatcher facing ban</td>\n",
       "      <td>mcilroy calls for irish fighting spirit</td>\n",
       "      <td>leicester bin stadium blueprint</td>\n",
       "      <td>united braced for mexican wave</td>\n",
       "      <td>auntie back in fashion  even if the dress look...</td>\n",
       "      <td>shoaib appeal goes to the top</td>\n",
       "      <td>hussain hurt by  shambles  but lays blame on e...</td>\n",
       "      <td>...</td>\n",
       "      <td>putin admits yeltsin quit to give him a head s...</td>\n",
       "      <td>bbc worst hit as digital tv begins to bite</td>\n",
       "      <td>how much can you pay for</td>\n",
       "      <td>christmas glitches</td>\n",
       "      <td>upending a table  chopping a line and scoring ...</td>\n",
       "      <td>scientific evidence  unreliable   defence claims</td>\n",
       "      <td>fusco wins judicial review in extradition case</td>\n",
       "      <td>rebels thwart russian advance</td>\n",
       "      <td>blair orders shake up of failing nhs</td>\n",
       "      <td>lessons of law s hard heart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>hitches and horlocks</td>\n",
       "      <td>beckham off but united survive</td>\n",
       "      <td>breast cancer screening</td>\n",
       "      <td>alan parker</td>\n",
       "      <td>guardian readers  are you all whingers</td>\n",
       "      <td>hollywood beyond</td>\n",
       "      <td>ashes and diamonds</td>\n",
       "      <td>whingers   a formidable minority</td>\n",
       "      <td>...</td>\n",
       "      <td>most everywhere   udis</td>\n",
       "      <td>most wanted   chloe lunettes</td>\n",
       "      <td>return of the cane  completely off the agenda</td>\n",
       "      <td>from sleepy hollow to greeneland</td>\n",
       "      <td>blunkett outlines vision for over   s</td>\n",
       "      <td>embattled dobson attacks  play now  pay later ...</td>\n",
       "      <td>doom and the dome</td>\n",
       "      <td>what is the north south divide</td>\n",
       "      <td>aitken released from jail</td>\n",
       "      <td>gone aloft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date  Label                                               Top1  \\\n",
       "0  2000-01-03      0  a  hindrance to operations   extracts from the...   \n",
       "1  2000-01-04      0                                          scorecard   \n",
       "2  2000-01-05      0                  coventry caught on counter by flo   \n",
       "3  2000-01-06      1                      pilgrim knows how to progress   \n",
       "4  2000-01-07      1                               hitches and horlocks   \n",
       "\n",
       "                                 Top2  \\\n",
       "0                           scorecard   \n",
       "1                 the best lake scene   \n",
       "2  united s rivals on the road to rio   \n",
       "3                 thatcher facing ban   \n",
       "4      beckham off but united survive   \n",
       "\n",
       "                                            Top3  \\\n",
       "0                hughes  instant hit buoys blues   \n",
       "1                  leader  german sleaze inquiry   \n",
       "2  thatcher issues defence before trial by video   \n",
       "3        mcilroy calls for irish fighting spirit   \n",
       "4                        breast cancer screening   \n",
       "\n",
       "                                            Top4  \\\n",
       "0       jack gets his skates on at ice cold alex   \n",
       "1                                  cheerio  boyo   \n",
       "2  police help smith lay down the law at everton   \n",
       "3                leicester bin stadium blueprint   \n",
       "4                                    alan parker   \n",
       "\n",
       "                                          Top5  \\\n",
       "0       chaos as maracana builds up for united   \n",
       "1                     the main recommendations   \n",
       "2  tale of trautmann bears two more retellings   \n",
       "3               united braced for mexican wave   \n",
       "4      guardian readers  are you all whingers    \n",
       "\n",
       "                                                Top6  \\\n",
       "0  depleted leicester prevail as elliott spoils e...   \n",
       "1                             has cubie killed fees    \n",
       "2                                england on the rack   \n",
       "3  auntie back in fashion  even if the dress look...   \n",
       "4                                   hollywood beyond   \n",
       "\n",
       "                                              Top7  \\\n",
       "0                 hungry spurs sense rich pickings   \n",
       "1                           has cubie killed fees    \n",
       "2  pakistan retaliate with call for video of walsh   \n",
       "3                    shoaib appeal goes to the top   \n",
       "4                               ashes and diamonds   \n",
       "\n",
       "                                                Top8  ...  \\\n",
       "0                  gunners so wide of an easy target  ...   \n",
       "1                             has cubie killed fees   ...   \n",
       "2               cullinan continues his cape monopoly  ...   \n",
       "3  hussain hurt by  shambles  but lays blame on e...  ...   \n",
       "4                   whingers   a formidable minority  ...   \n",
       "\n",
       "                                               Top16  \\\n",
       "0           flintoff injury piles on woe for england   \n",
       "1                               on the critical list   \n",
       "2                        south melbourne  australia    \n",
       "3  putin admits yeltsin quit to give him a head s...   \n",
       "4                             most everywhere   udis   \n",
       "\n",
       "                                               Top17  \\\n",
       "0  hunters threaten jospin with new battle of the...   \n",
       "1                          the timing of their lives   \n",
       "2                                    necaxa  mexico    \n",
       "3         bbc worst hit as digital tv begins to bite   \n",
       "4                       most wanted   chloe lunettes   \n",
       "\n",
       "                                            Top18  \\\n",
       "0             kohl s successor drawn into scandal   \n",
       "1                                     dear doctor   \n",
       "2                             real madrid  spain    \n",
       "3                     how much can you pay for      \n",
       "4  return of the cane  completely off the agenda    \n",
       "\n",
       "                                               Top19  \\\n",
       "0               the difference between men and women   \n",
       "1  irish court halts ira man s extradition to nor...   \n",
       "2                          raja casablanca  morocco    \n",
       "3                                 christmas glitches   \n",
       "4                   from sleepy hollow to greeneland   \n",
       "\n",
       "                                               Top20  \\\n",
       "0                sara denver  nurse turned solicitor   \n",
       "1  burundi peace initiative fades after rebels re...   \n",
       "2                               corinthians  brazil    \n",
       "3  upending a table  chopping a line and scoring ...   \n",
       "4              blunkett outlines vision for over   s   \n",
       "\n",
       "                                               Top21  \\\n",
       "0     diana s landmine crusade put tories in a panic   \n",
       "1               pe points the way forward to the ecb   \n",
       "2                                 tony s pet project   \n",
       "3   scientific evidence  unreliable   defence claims   \n",
       "4  embattled dobson attacks  play now  pay later ...   \n",
       "\n",
       "                                               Top22  \\\n",
       "0  yeltsin s resignation caught opposition flat f...   \n",
       "1  campaigners keep up pressure on nazi war crime...   \n",
       "2                            al nassr  saudi arabia    \n",
       "3     fusco wins judicial review in extradition case   \n",
       "4                                  doom and the dome   \n",
       "\n",
       "                             Top23  \\\n",
       "0                 russian roulette   \n",
       "1                   jane ratcliffe   \n",
       "2                ideal holmes show   \n",
       "3    rebels thwart russian advance   \n",
       "4  what is the north south divide    \n",
       "\n",
       "                                               Top24  \\\n",
       "0                                           sold out   \n",
       "1  yet more things you wouldn t know without the ...   \n",
       "2               pinochet leaves hospital after tests   \n",
       "3               blair orders shake up of failing nhs   \n",
       "4                          aitken released from jail   \n",
       "\n",
       "                          Top25  \n",
       "0            recovering a title  \n",
       "1  millennium bug fails to bite  \n",
       "2                  useful links  \n",
       "3   lessons of law s hard heart  \n",
       "4                    gone aloft  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>1</td>\n",
       "      <td>most cases of cancer are the result of sheer b...</td>\n",
       "      <td>iran dismissed united states efforts to fight ...</td>\n",
       "      <td>poll  one in   germans would join anti muslim ...</td>\n",
       "      <td>uk royal family s prince andrew named in us la...</td>\n",
       "      <td>some    asylum seekers refused to leave the bu...</td>\n",
       "      <td>pakistani boat blows self up after india navy ...</td>\n",
       "      <td>sweden hit by third mosque arson attack in a week</td>\n",
       "      <td>cars set alight during french new year</td>\n",
       "      <td>...</td>\n",
       "      <td>ukrainian minister threatens tv channel with c...</td>\n",
       "      <td>palestinian president mahmoud abbas has entere...</td>\n",
       "      <td>israeli security center publishes names of    ...</td>\n",
       "      <td>the year      was the deadliest year yet in sy...</td>\n",
       "      <td>a secret underground complex built by the nazi...</td>\n",
       "      <td>restrictions on web freedom a major global iss...</td>\n",
       "      <td>austrian journalist erich mchel delivered a pr...</td>\n",
       "      <td>thousands of ukraine nationalists march in kiev</td>\n",
       "      <td>chinas new years resolution  no more harvestin...</td>\n",
       "      <td>authorities pull plug on russia s last politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>2015-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>moscow  gt beijing high speed train will reduc...</td>\n",
       "      <td>two ancient tombs were discovered in egypt on ...</td>\n",
       "      <td>china complains to pyongyang after n korean so...</td>\n",
       "      <td>scotland headed towards being fossil fuel free...</td>\n",
       "      <td>prime minister shinzo abe said monday he will ...</td>\n",
       "      <td>sex slave at centre of prince andrew scandal f...</td>\n",
       "      <td>gay relative of hamas founder faces deportatio...</td>\n",
       "      <td>the number of female drug addicts in iran has ...</td>\n",
       "      <td>...</td>\n",
       "      <td>the islamic state has approved a      budget o...</td>\n",
       "      <td>iceland to withdraw eu application  lift capit...</td>\n",
       "      <td>blackfield capital founder goes missing  the v...</td>\n",
       "      <td>rocket stage crashes back to earth in rural ch...</td>\n",
       "      <td>dead as aircraft bombs greek tanker in libya...</td>\n",
       "      <td>belgian murderer frank van den bleeken to die ...</td>\n",
       "      <td>czech president criticizes ukrainian pm  says ...</td>\n",
       "      <td>vietnamese jets join search for    missing f...</td>\n",
       "      <td>france seeks end to russia sanctions over ukraine</td>\n",
       "      <td>china scraps rare earths caps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>2015-01-06</td>\n",
       "      <td>0</td>\n",
       "      <td>us oil falls below     a barrel</td>\n",
       "      <td>toyota gives away       fuel cell patents to b...</td>\n",
       "      <td>young indian couple who had been granted polic...</td>\n",
       "      <td>a senior figure in islamic states self declare...</td>\n",
       "      <td>fukushima rice passes radiation tests for  st ...</td>\n",
       "      <td>nearly all spanish parties guilty of financial...</td>\n",
       "      <td>king abdullah to abdicate saudi throne</td>\n",
       "      <td>taliban commander caught networking on linkedin</td>\n",
       "      <td>...</td>\n",
       "      <td>thousands of indians have fled from their home...</td>\n",
       "      <td>turkey sacks judges who oversaw erdogan corrup...</td>\n",
       "      <td>spacex falcon   launch and recovery has been a...</td>\n",
       "      <td>cnn  americans charged in botched gambia coup</td>\n",
       "      <td>islamic state  police  official beheaded</td>\n",
       "      <td>libya bans palestinians from country to preven...</td>\n",
       "      <td>a judicial inquiry was opened in france on mon...</td>\n",
       "      <td>video has captured the moment a cameraman was ...</td>\n",
       "      <td>syria has complained to the united nations tha...</td>\n",
       "      <td>tests over  india set to make the iris of bigg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>2015-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>shots fired  at french magazine hq</td>\n",
       "      <td>of bibi netanyahus campaign funds from u s</td>\n",
       "      <td>isis behead street magician for entertaining c...</td>\n",
       "      <td>denmark sets new world record in renewable ene...</td>\n",
       "      <td>isis closes schools in syria  leaving        c...</td>\n",
       "      <td>the european union wants turkey to explain how...</td>\n",
       "      <td>people have died in a shooting at the headq...</td>\n",
       "      <td>most earth like planet ever discovered could b...</td>\n",
       "      <td>...</td>\n",
       "      <td>discovery of witnesss mutilated body feeds cla...</td>\n",
       "      <td>radical islam a growing threat to sub saharan ...</td>\n",
       "      <td>tony blair may  face war crime charges  over i...</td>\n",
       "      <td>benjamin netanyahu says settlements unlikely t...</td>\n",
       "      <td>tasmania moves to allow corporations to sue pr...</td>\n",
       "      <td>indonesia search chief  tail section of airasi...</td>\n",
       "      <td>north korean defector   bureau      hackers lo...</td>\n",
       "      <td>female suicide bomber attacks police station i...</td>\n",
       "      <td>is to turn syria schools into religious indoct...</td>\n",
       "      <td>ebola vaccine trials in halifax so far a success</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>2015-01-08</td>\n",
       "      <td>1</td>\n",
       "      <td>new charlie hebdo issue to come out next week ...</td>\n",
       "      <td>ahmed merabet  cop killed in paris attacks  wa...</td>\n",
       "      <td>youngest suspect in charlie hebdo attack turns...</td>\n",
       "      <td>grenades thrown at a mosque in le mans  west o...</td>\n",
       "      <td>danish newspaper to print charlie hebdo cartoo...</td>\n",
       "      <td>muslim politician from india offers   m to cha...</td>\n",
       "      <td>vladimir putin has condemned a deadly terroris...</td>\n",
       "      <td>turkey is trying to hide any evidence that hol...</td>\n",
       "      <td>...</td>\n",
       "      <td>thousands in europe hold up pens in solidarity...</td>\n",
       "      <td>journalists race to show solidarity with  char...</td>\n",
       "      <td>antibiotics  us discovery labelled  game chang...</td>\n",
       "      <td>the charlie hebdo attack condemned by saudi ar...</td>\n",
       "      <td>brisbane man wearing  i m with stupid  t shirt...</td>\n",
       "      <td>fourth russian general commits suicide in less...</td>\n",
       "      <td>meteorite turns night into day over bucharest ...</td>\n",
       "      <td>israeli doctors implant syrian man with titani...</td>\n",
       "      <td>unusual metal recovered from ancient greek shi...</td>\n",
       "      <td>young mother let terrorists into charlie hebdo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date  Label                                               Top1  \\\n",
       "3723  2015-01-02      1  most cases of cancer are the result of sheer b...   \n",
       "3724  2015-01-05      0  moscow  gt beijing high speed train will reduc...   \n",
       "3725  2015-01-06      0                    us oil falls below     a barrel   \n",
       "3726  2015-01-07      1                 shots fired  at french magazine hq   \n",
       "3727  2015-01-08      1  new charlie hebdo issue to come out next week ...   \n",
       "\n",
       "                                                   Top2  \\\n",
       "3723  iran dismissed united states efforts to fight ...   \n",
       "3724  two ancient tombs were discovered in egypt on ...   \n",
       "3725  toyota gives away       fuel cell patents to b...   \n",
       "3726        of bibi netanyahus campaign funds from u s    \n",
       "3727  ahmed merabet  cop killed in paris attacks  wa...   \n",
       "\n",
       "                                                   Top3  \\\n",
       "3723  poll  one in   germans would join anti muslim ...   \n",
       "3724  china complains to pyongyang after n korean so...   \n",
       "3725  young indian couple who had been granted polic...   \n",
       "3726  isis behead street magician for entertaining c...   \n",
       "3727  youngest suspect in charlie hebdo attack turns...   \n",
       "\n",
       "                                                   Top4  \\\n",
       "3723  uk royal family s prince andrew named in us la...   \n",
       "3724  scotland headed towards being fossil fuel free...   \n",
       "3725  a senior figure in islamic states self declare...   \n",
       "3726  denmark sets new world record in renewable ene...   \n",
       "3727  grenades thrown at a mosque in le mans  west o...   \n",
       "\n",
       "                                                   Top5  \\\n",
       "3723  some    asylum seekers refused to leave the bu...   \n",
       "3724  prime minister shinzo abe said monday he will ...   \n",
       "3725  fukushima rice passes radiation tests for  st ...   \n",
       "3726  isis closes schools in syria  leaving        c...   \n",
       "3727  danish newspaper to print charlie hebdo cartoo...   \n",
       "\n",
       "                                                   Top6  \\\n",
       "3723  pakistani boat blows self up after india navy ...   \n",
       "3724  sex slave at centre of prince andrew scandal f...   \n",
       "3725  nearly all spanish parties guilty of financial...   \n",
       "3726  the european union wants turkey to explain how...   \n",
       "3727  muslim politician from india offers   m to cha...   \n",
       "\n",
       "                                                   Top7  \\\n",
       "3723  sweden hit by third mosque arson attack in a week   \n",
       "3724  gay relative of hamas founder faces deportatio...   \n",
       "3725             king abdullah to abdicate saudi throne   \n",
       "3726     people have died in a shooting at the headq...   \n",
       "3727  vladimir putin has condemned a deadly terroris...   \n",
       "\n",
       "                                                   Top8  ...  \\\n",
       "3723             cars set alight during french new year  ...   \n",
       "3724  the number of female drug addicts in iran has ...  ...   \n",
       "3725    taliban commander caught networking on linkedin  ...   \n",
       "3726  most earth like planet ever discovered could b...  ...   \n",
       "3727  turkey is trying to hide any evidence that hol...  ...   \n",
       "\n",
       "                                                  Top16  \\\n",
       "3723  ukrainian minister threatens tv channel with c...   \n",
       "3724  the islamic state has approved a      budget o...   \n",
       "3725  thousands of indians have fled from their home...   \n",
       "3726  discovery of witnesss mutilated body feeds cla...   \n",
       "3727  thousands in europe hold up pens in solidarity...   \n",
       "\n",
       "                                                  Top17  \\\n",
       "3723  palestinian president mahmoud abbas has entere...   \n",
       "3724  iceland to withdraw eu application  lift capit...   \n",
       "3725  turkey sacks judges who oversaw erdogan corrup...   \n",
       "3726  radical islam a growing threat to sub saharan ...   \n",
       "3727  journalists race to show solidarity with  char...   \n",
       "\n",
       "                                                  Top18  \\\n",
       "3723  israeli security center publishes names of    ...   \n",
       "3724  blackfield capital founder goes missing  the v...   \n",
       "3725  spacex falcon   launch and recovery has been a...   \n",
       "3726  tony blair may  face war crime charges  over i...   \n",
       "3727  antibiotics  us discovery labelled  game chang...   \n",
       "\n",
       "                                                  Top19  \\\n",
       "3723  the year      was the deadliest year yet in sy...   \n",
       "3724  rocket stage crashes back to earth in rural ch...   \n",
       "3725      cnn  americans charged in botched gambia coup   \n",
       "3726  benjamin netanyahu says settlements unlikely t...   \n",
       "3727  the charlie hebdo attack condemned by saudi ar...   \n",
       "\n",
       "                                                  Top20  \\\n",
       "3723  a secret underground complex built by the nazi...   \n",
       "3724    dead as aircraft bombs greek tanker in libya...   \n",
       "3725          islamic state  police  official beheaded    \n",
       "3726  tasmania moves to allow corporations to sue pr...   \n",
       "3727  brisbane man wearing  i m with stupid  t shirt...   \n",
       "\n",
       "                                                  Top21  \\\n",
       "3723  restrictions on web freedom a major global iss...   \n",
       "3724  belgian murderer frank van den bleeken to die ...   \n",
       "3725  libya bans palestinians from country to preven...   \n",
       "3726  indonesia search chief  tail section of airasi...   \n",
       "3727  fourth russian general commits suicide in less...   \n",
       "\n",
       "                                                  Top22  \\\n",
       "3723  austrian journalist erich mchel delivered a pr...   \n",
       "3724  czech president criticizes ukrainian pm  says ...   \n",
       "3725  a judicial inquiry was opened in france on mon...   \n",
       "3726  north korean defector   bureau      hackers lo...   \n",
       "3727  meteorite turns night into day over bucharest ...   \n",
       "\n",
       "                                                  Top23  \\\n",
       "3723    thousands of ukraine nationalists march in kiev   \n",
       "3724    vietnamese jets join search for    missing f...   \n",
       "3725  video has captured the moment a cameraman was ...   \n",
       "3726  female suicide bomber attacks police station i...   \n",
       "3727  israeli doctors implant syrian man with titani...   \n",
       "\n",
       "                                                  Top24  \\\n",
       "3723  chinas new years resolution  no more harvestin...   \n",
       "3724  france seeks end to russia sanctions over ukraine   \n",
       "3725  syria has complained to the united nations tha...   \n",
       "3726  is to turn syria schools into religious indoct...   \n",
       "3727  unusual metal recovered from ancient greek shi...   \n",
       "\n",
       "                                                  Top25  \n",
       "3723  authorities pull plug on russia s last politic...  \n",
       "3724                      china scraps rare earths caps  \n",
       "3725  tests over  india set to make the iris of bigg...  \n",
       "3726   ebola vaccine trials in halifax so far a success  \n",
       "3727  young mother let terrorists into charlie hebdo...  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer as cv\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>practice of CountVectorize</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']\n",
      "[[0 1 1 1 0 0 1 0 1]\n",
      " [0 2 0 1 0 1 1 0 1]\n",
      " [1 0 0 1 1 0 1 1 1]\n",
      " [0 1 1 1 0 0 1 0 1]]\n"
     ]
    }
   ],
   "source": [
    " \n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "corpus = [\n",
    "     'This is the first document.',\n",
    "    'This document is the second document.',\n",
    "    'And this is the third one.',\n",
    "     'Is this the first document?']\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "print(vectorizer.get_feature_names())\n",
    "#['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']\n",
    "print(X.toarray())  # doctest: +NORMALIZE_WHITESPACE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['and this', 'document is', 'first document', 'is the', 'is this', 'second document', 'the first', 'the second', 'the third', 'third one', 'this document', 'this is', 'this the']\n",
      "[[0 0 1 1 0 0 1 0 0 0 0 1 0]\n",
      " [0 1 0 1 0 1 0 1 0 0 1 0 0]\n",
      " [1 0 0 1 0 0 0 0 1 1 0 1 0]\n",
      " [0 0 1 0 1 0 1 0 0 0 0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "corpus = [\n",
    "     'This is the first document.',\n",
    "    'This document is the second document.',\n",
    "    'And this is the third one.',\n",
    "     'Is this the first document?']\n",
    "\n",
    "vectorizer2 = CountVectorizer(ngram_range=(2,2))\n",
    "X = vectorizer2.fit_transform(corpus)\n",
    "print(vectorizer2.get_feature_names())\n",
    "#['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']\n",
    "print(X.toarray())  # doctest: +NORMALIZE_WHITESPACE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Classification using RandomForestClassifier</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3975"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(headline1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=200,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector = cv(ngram_range=(2,2))\n",
    "traindataset = count_vector.fit_transform(headline1)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=200,criterion='entropy')\n",
    "clf.fit(traindataset,train['Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = count_vector.transform(headline2)\n",
    "\n",
    "prediction = clf.predict(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[139  47]\n",
      " [ 11 181]]\n",
      "0.8465608465608465\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.75      0.83       186\n",
      "           1       0.79      0.94      0.86       192\n",
      "\n",
      "    accuracy                           0.85       378\n",
      "   macro avg       0.86      0.85      0.84       378\n",
      "weighted avg       0.86      0.85      0.84       378\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score\n",
    "\n",
    "matrix = confusion_matrix(test['Label'],prediction)\n",
    "print(matrix)\n",
    "\n",
    "score = accuracy_score(test['Label'],prediction)\n",
    "print(score)\n",
    "\n",
    "report = classification_report(test['Label'],prediction)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}