Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import language_check\n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('./datas/training_set_rel3.tsv',sep='\\t', encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>essay_set</th>\n",
       "      <th>essay</th>\n",
       "      <th>rater1_domain1</th>\n",
       "      <th>rater2_domain1</th>\n",
       "      <th>rater3_domain1</th>\n",
       "      <th>domain1_score</th>\n",
       "      <th>rater1_domain2</th>\n",
       "      <th>rater2_domain2</th>\n",
       "      <th>domain2_score</th>\n",
       "      <th>...</th>\n",
       "      <th>rater2_trait3</th>\n",
       "      <th>rater2_trait4</th>\n",
       "      <th>rater2_trait5</th>\n",
       "      <th>rater2_trait6</th>\n",
       "      <th>rater3_trait1</th>\n",
       "      <th>rater3_trait2</th>\n",
       "      <th>rater3_trait3</th>\n",
       "      <th>rater3_trait4</th>\n",
       "      <th>rater3_trait5</th>\n",
       "      <th>rater3_trait6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>Dear @CAPS1 @CAPS2, I believe that using compu...</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   essay_id  essay_set                                              essay  \\\n",
       "0         1          1  Dear local newspaper, I think effects computer...   \n",
       "1         2          1  Dear @CAPS1 @CAPS2, I believe that using compu...   \n",
       "\n",
       "   rater1_domain1  rater2_domain1  rater3_domain1  domain1_score  \\\n",
       "0               4               4             NaN              8   \n",
       "1               5               4             NaN              9   \n",
       "\n",
       "   rater1_domain2  rater2_domain2  domain2_score      ...        \\\n",
       "0             NaN             NaN            NaN      ...         \n",
       "1             NaN             NaN            NaN      ...         \n",
       "\n",
       "   rater2_trait3  rater2_trait4  rater2_trait5  rater2_trait6  rater3_trait1  \\\n",
       "0            NaN            NaN            NaN            NaN            NaN   \n",
       "1            NaN            NaN            NaN            NaN            NaN   \n",
       "\n",
       "   rater3_trait2  rater3_trait3  rater3_trait4  rater3_trait5  rater3_trait6  \n",
       "0            NaN            NaN            NaN            NaN            NaN  \n",
       "1            NaN            NaN            NaN            NaN            NaN  \n",
       "\n",
       "[2 rows x 28 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_to_keep =['essay','domain1_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_data = data[col_to_keep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12976\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay</th>\n",
       "      <th>domain1_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dear @CAPS1 @CAPS2, I believe that using compu...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dear Local Newspaper, @CAPS1 I have found that...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dear @LOCATION1, I know having computers has a...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               essay  domain1_score\n",
       "0  Dear local newspaper, I think effects computer...              8\n",
       "1  Dear @CAPS1 @CAPS2, I believe that using compu...              9\n",
       "2  Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...              7\n",
       "3  Dear Local Newspaper, @CAPS1 I have found that...             10\n",
       "4  Dear @LOCATION1, I know having computers has a...              8"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(filter_data))\n",
    "filter_data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_counting(x):\n",
    "    return (len(TextBlob(x).words))\n",
    "filter_data['word_length'] = filter_data['essay'].apply(word_counting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_counting(x):\n",
    "    sentence_len = len([len(sentence.split(' ')) for sentence in TextBlob(x).sentences])\n",
    "    return sentence_len\n",
    "filter_data['no_of_sentence'] = filter_data['essay'].apply(sentence_counting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_sentence_sentiment(x):\n",
    "    sentiment_essay = TextBlob(x).sentiment.polarity\n",
    "    return sentiment_essay\n",
    "filter_data['sentiment_essay'] = filter_data['essay'].apply(avg_sentence_sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_length_of_words(x):\n",
    "    word_len = [len(word) for word in TextBlob(x).words]\n",
    "    return ( sum(word_len)/len(word_len) )\n",
    "filter_data['avg_word_len']  = filter_data['essay'].apply(avg_length_of_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay</th>\n",
       "      <th>domain1_score</th>\n",
       "      <th>word_length</th>\n",
       "      <th>no_of_sentence</th>\n",
       "      <th>sentiment_essay</th>\n",
       "      <th>avg_word_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8</td>\n",
       "      <td>343</td>\n",
       "      <td>16</td>\n",
       "      <td>0.310471</td>\n",
       "      <td>4.358601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dear @CAPS1 @CAPS2, I believe that using compu...</td>\n",
       "      <td>9</td>\n",
       "      <td>422</td>\n",
       "      <td>20</td>\n",
       "      <td>0.274000</td>\n",
       "      <td>4.331754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...</td>\n",
       "      <td>7</td>\n",
       "      <td>283</td>\n",
       "      <td>14</td>\n",
       "      <td>0.340393</td>\n",
       "      <td>4.356890</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               essay  domain1_score  \\\n",
       "0  Dear local newspaper, I think effects computer...              8   \n",
       "1  Dear @CAPS1 @CAPS2, I believe that using compu...              9   \n",
       "2  Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...              7   \n",
       "\n",
       "   word_length  no_of_sentence  sentiment_essay  avg_word_len  \n",
       "0          343              16         0.310471      4.358601  \n",
       "1          422              20         0.274000      4.331754  \n",
       "2          283              14         0.340393      4.356890  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grammar_check(x):\n",
    "    tool = language_check.LanguageTool('en-US')\n",
    "    matches = tool.check(x)\n",
    "    return len(matches)\n",
    "filter_data['Grammar_check'] = filter_data['essay'].apply(grammar_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_word_mis_spell(x):\n",
    "    print('next')\n",
    "    every = 0\n",
    "    print(len(TextBlob(x).words))\n",
    "    for wor in TextBlob(x).words:\n",
    "#         w = Word(wor)\n",
    "        print(wor)\n",
    "        if wor.lower() not in word_list:\n",
    "            every += 1\n",
    "            print(every)\n",
    "        print(every)\n",
    "    return every    \n",
    "filter_data['mis_spell_word'] = filter_data['essay'].apply(num_word_mis_spell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_data.to_csv('filter_data.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_data['Grammar_check'] = filter_data['Grammar_check']/5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay</th>\n",
       "      <th>domain1_score</th>\n",
       "      <th>word_length</th>\n",
       "      <th>no_of_sentence</th>\n",
       "      <th>sentiment_essay</th>\n",
       "      <th>avg_word_len</th>\n",
       "      <th>Grammar_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8</td>\n",
       "      <td>343</td>\n",
       "      <td>16</td>\n",
       "      <td>0.310471</td>\n",
       "      <td>4.358601</td>\n",
       "      <td>2.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dear @CAPS1 @CAPS2, I believe that using compu...</td>\n",
       "      <td>9</td>\n",
       "      <td>422</td>\n",
       "      <td>20</td>\n",
       "      <td>0.274000</td>\n",
       "      <td>4.331754</td>\n",
       "      <td>3.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...</td>\n",
       "      <td>7</td>\n",
       "      <td>283</td>\n",
       "      <td>14</td>\n",
       "      <td>0.340393</td>\n",
       "      <td>4.356890</td>\n",
       "      <td>1.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dear Local Newspaper, @CAPS1 I have found that...</td>\n",
       "      <td>10</td>\n",
       "      <td>527</td>\n",
       "      <td>27</td>\n",
       "      <td>0.266828</td>\n",
       "      <td>4.851992</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dear @LOCATION1, I know having computers has a...</td>\n",
       "      <td>8</td>\n",
       "      <td>470</td>\n",
       "      <td>30</td>\n",
       "      <td>0.199684</td>\n",
       "      <td>4.378723</td>\n",
       "      <td>3.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               essay  domain1_score  \\\n",
       "0  Dear local newspaper, I think effects computer...              8   \n",
       "1  Dear @CAPS1 @CAPS2, I believe that using compu...              9   \n",
       "2  Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...              7   \n",
       "3  Dear Local Newspaper, @CAPS1 I have found that...             10   \n",
       "4  Dear @LOCATION1, I know having computers has a...              8   \n",
       "\n",
       "   word_length  no_of_sentence  sentiment_essay  avg_word_len  Grammar_check  \n",
       "0          343              16         0.310471      4.358601            2.2  \n",
       "1          422              20         0.274000      4.331754            3.8  \n",
       "2          283              14         0.340393      4.356890            1.8  \n",
       "3          527              27         0.266828      4.851992            7.0  \n",
       "4          470              30         0.199684      4.378723            3.4  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_data = pd.read_csv('filter_data.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay</th>\n",
       "      <th>domain1_score</th>\n",
       "      <th>word_length</th>\n",
       "      <th>no_of_sentence</th>\n",
       "      <th>sentiment_essay</th>\n",
       "      <th>avg_word_len</th>\n",
       "      <th>Grammar_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dear local newspaper, I think effects computer...</td>\n",
       "      <td>8</td>\n",
       "      <td>343</td>\n",
       "      <td>16</td>\n",
       "      <td>0.310471</td>\n",
       "      <td>4.358601</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dear @CAPS1 @CAPS2, I believe that using compu...</td>\n",
       "      <td>9</td>\n",
       "      <td>422</td>\n",
       "      <td>20</td>\n",
       "      <td>0.274000</td>\n",
       "      <td>4.331754</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...</td>\n",
       "      <td>7</td>\n",
       "      <td>283</td>\n",
       "      <td>14</td>\n",
       "      <td>0.340393</td>\n",
       "      <td>4.356890</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               essay  domain1_score  \\\n",
       "0  Dear local newspaper, I think effects computer...              8   \n",
       "1  Dear @CAPS1 @CAPS2, I believe that using compu...              9   \n",
       "2  Dear, @CAPS1 @CAPS2 @CAPS3 More and more peopl...              7   \n",
       "\n",
       "   word_length  no_of_sentence  sentiment_essay  avg_word_len  Grammar_check  \n",
       "0          343              16         0.310471      4.358601             11  \n",
       "1          422              20         0.274000      4.331754             19  \n",
       "2          283              14         0.340393      4.356890              9  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_data.drop('Unnamed: 0',axis=1, inplace=True)\n",
    "filter_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>word_length</th>\n",
       "      <th>no_of_sentence</th>\n",
       "      <th>sentiment_essay</th>\n",
       "      <th>avg_word_len</th>\n",
       "      <th>Grammar_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.318990</td>\n",
       "      <td>0.157895</td>\n",
       "      <td>0.655235</td>\n",
       "      <td>0.484731</td>\n",
       "      <td>0.085938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.392891</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.637000</td>\n",
       "      <td>0.475809</td>\n",
       "      <td>0.148438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.116667</td>\n",
       "      <td>0.262862</td>\n",
       "      <td>0.136842</td>\n",
       "      <td>0.670196</td>\n",
       "      <td>0.484163</td>\n",
       "      <td>0.070312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.491113</td>\n",
       "      <td>0.273684</td>\n",
       "      <td>0.633414</td>\n",
       "      <td>0.648698</td>\n",
       "      <td>0.273438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.437792</td>\n",
       "      <td>0.305263</td>\n",
       "      <td>0.599842</td>\n",
       "      <td>0.491419</td>\n",
       "      <td>0.132812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      score  word_length  no_of_sentence  sentiment_essay  avg_word_len  \\\n",
       "0  0.133333     0.318990        0.157895         0.655235      0.484731   \n",
       "1  0.150000     0.392891        0.200000         0.637000      0.475809   \n",
       "2  0.116667     0.262862        0.136842         0.670196      0.484163   \n",
       "3  0.166667     0.491113        0.273684         0.633414      0.648698   \n",
       "4  0.133333     0.437792        0.305263         0.599842      0.491419   \n",
       "\n",
       "   Grammar_check  \n",
       "0       0.085938  \n",
       "1       0.148438  \n",
       "2       0.070312  \n",
       "3       0.273438  \n",
       "4       0.132812  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# normalized_df=(df-df.mean())/df.std()\n",
    "from sklearn import preprocessing\n",
    "\n",
    "# data = {'score': [234,24,14,27,-74,46,73,-18,59,160]}\n",
    "# df = pd.DataFrame(data)\n",
    "# df\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "np_scaled = min_max_scaler.fit_transform(filter_data.drop('essay',axis=1))\n",
    "df_normalized = pd.DataFrame(np_scaled, columns = ['score','word_length','no_of_sentence','sentiment_essay','avg_word_len','Grammar_check'])\n",
    "df_normalized.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = df_normalized['score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_normalized.drop('score',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word_length</th>\n",
       "      <th>no_of_sentence</th>\n",
       "      <th>sentiment_essay</th>\n",
       "      <th>avg_word_len</th>\n",
       "      <th>Grammar_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.318990</td>\n",
       "      <td>0.157895</td>\n",
       "      <td>0.655235</td>\n",
       "      <td>0.484731</td>\n",
       "      <td>0.085938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.392891</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.637000</td>\n",
       "      <td>0.475809</td>\n",
       "      <td>0.148438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.262862</td>\n",
       "      <td>0.136842</td>\n",
       "      <td>0.670196</td>\n",
       "      <td>0.484163</td>\n",
       "      <td>0.070312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   word_length  no_of_sentence  sentiment_essay  avg_word_len  Grammar_check\n",
       "0     0.318990        0.157895         0.655235      0.484731       0.085938\n",
       "1     0.392891        0.200000         0.637000      0.475809       0.148438\n",
       "2     0.262862        0.136842         0.670196      0.484163       0.070312"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(y, x, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import linear_model\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma='auto',\n",
      "  kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma='auto',\n",
      "  kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False) :     0.5088554660112241\n",
      "SGDRegressor(alpha=0.0001, average=False, epsilon=0.1, eta0=0.01,\n",
      "       fit_intercept=True, l1_ratio=0.15, learning_rate='invscaling',\n",
      "       loss='squared_loss', max_iter=None, n_iter=None, penalty='l2',\n",
      "       power_t=0.25, random_state=None, shuffle=True, tol=None, verbose=0,\n",
      "       warm_start=False)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sushil\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDRegressor'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGDRegressor(alpha=0.0001, average=False, epsilon=0.1, eta0=0.01,\n",
      "       fit_intercept=True, l1_ratio=0.15, learning_rate='invscaling',\n",
      "       loss='squared_loss', max_iter=None, n_iter=None, penalty='l2',\n",
      "       power_t=0.25, random_state=None, shuffle=True, tol=None, verbose=0,\n",
      "       warm_start=False) :     0.4184124752748861\n",
      "BayesianRidge(alpha_1=1e-06, alpha_2=1e-06, compute_score=False, copy_X=True,\n",
      "       fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06, n_iter=300,\n",
      "       normalize=False, tol=0.001, verbose=False)\n",
      "BayesianRidge(alpha_1=1e-06, alpha_2=1e-06, compute_score=False, copy_X=True,\n",
      "       fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06, n_iter=300,\n",
      "       normalize=False, tol=0.001, verbose=False) :     0.47782025610749496\n",
      "LassoLars(alpha=1.0, copy_X=True, eps=2.220446049250313e-16,\n",
      "     fit_intercept=True, fit_path=True, max_iter=500, normalize=True,\n",
      "     positive=False, precompute='auto', verbose=False)\n",
      "LassoLars(alpha=1.0, copy_X=True, eps=2.220446049250313e-16,\n",
      "     fit_intercept=True, fit_path=True, max_iter=500, normalize=True,\n",
      "     positive=False, precompute='auto', verbose=False) :     -0.00083156872616863\n",
      "ARDRegression(alpha_1=1e-06, alpha_2=1e-06, compute_score=False, copy_X=True,\n",
      "       fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06, n_iter=300,\n",
      "       normalize=False, threshold_lambda=10000.0, tol=0.001, verbose=False)\n"
     ]
    }
   ],
   "source": [
    "# import numpy as np\n",
    "# from sklearn import linear_model\n",
    "# from sklearn import svm\n",
    "\n",
    "classifiers = [\n",
    "    svm.SVR(),\n",
    "    linear_model.SGDRegressor(),\n",
    "    linear_model.BayesianRidge(),\n",
    "    linear_model.LassoLars(),\n",
    "    linear_model.ARDRegression(),\n",
    "    linear_model.PassiveAggressiveRegressor(),\n",
    "    linear_model.TheilSenRegressor(),\n",
    "    linear_model.LinearRegression()\n",
    "]\n",
    "\n",
    "# trainingData    = np.array([ [2.3, 4.3, 2.5],  [1.3, 5.2, 5.2],  [3.3, 2.9, 0.8],  [3.1, 4.3, 4.0]  ])\n",
    "# trainingScores  = np.array( [3.4, 7.5, 4.5, 1.6] )\n",
    "# predictionData  = np.array([ [2.5, 2.4, 2.7],  [2.7, 3.2, 1.2] ])\n",
    "\n",
    "for item in classifiers:\n",
    "    print(item)\n",
    "    clf = item\n",
    "    clf.fit(x_train, y_train)\n",
    "#     print(clf.predict(predictionData),'\\n')\n",
    "    print(item,':    ',clf.score(x_test, y_test, sample_weight=None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn import model_selection\n",
    "# from sklearn.linear_model import LinearRegression\n",
    "# from sklearn.linear_model import Ridge\n",
    "# from sklearn.linear_model import ElasticNet\n",
    "# from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.base import TransformerMixin\n",
    "# from sklearn.datasets import make_regression\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",
    "from sklearn.linear_model import LinearRegression, Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RidgeTransformer(Ridge, TransformerMixin):\n",
    "\n",
    "    def transform(self, X, *_):\n",
    "        return self.predict(X)\n",
    "\n",
    "\n",
    "class RandomForestTransformer(RandomForestRegressor, TransformerMixin):\n",
    "\n",
    "    def transform(self, X, *_):\n",
    "        return self.predict(X)\n",
    "\n",
    "\n",
    "class KNeighborsTransformer(KNeighborsRegressor, TransformerMixin):\n",
    "\n",
    "    def transform(self, X, *_):\n",
    "        return self.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    ridge_transformer = Pipeline(steps=[\n",
    "        ('scaler', StandardScaler()),\n",
    "        ('poly_feats', PolynomialFeatures()),\n",
    "        ('ridge', RidgeTransformer())\n",
    "    ])\n",
    "\n",
    "    pred_union = FeatureUnion(\n",
    "        transformer_list=[\n",
    "            ('ridge', ridge_transformer),\n",
    "            ('rand_forest', RandomForestTransformer()),\n",
    "            ('knn', KNeighborsTransformer())\n",
    "        ],\n",
    "        n_jobs=2\n",
    "    )\n",
    "\n",
    "    model = Pipeline(steps=[\n",
    "        ('pred_union', pred_union),\n",
    "        ('lin_regr', LinearRegression())\n",
    "    ])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build and fit a model..\n"
     ]
    }
   ],
   "source": [
    "print('Build and fit a model..')\n",
    "\n",
    "model = build_model()\n",
    "\n",
    "# X, y = make_regression(n_features=10, n_targets=2/)\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.25)\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "score = model.score(X_test, y_test)\n",
    "\n",
    "print('Done. Score:', score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_normalized.to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}