SearchRelevanceDistanceStacking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "ddada558-c36b-4e17-890a-0d566e74306c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "search_df=pd.read_csv('SearchReltrain.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "c30f44a7-953c-4cee-8055-2d897c2dd4a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_test=pd.read_csv('SearchReltest.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "6724aeb9-3b3b-42a1-82dc-d4f2b43fb580",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "wireless mouse             113\n",
       "rachel ray cookware         91\n",
       "memory foam pillow          90\n",
       "bike lock                   84\n",
       "16 gb memory card           64\n",
       "                          ... \n",
       "longboard pads              13\n",
       "silicone toe separators     12\n",
       "8 ounce mason jars          10\n",
       "polo bear sweater           10\n",
       "dollhouse bathtub            8\n",
       "Name: query, Length: 261, dtype: int64"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_df['query'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "5cdd2cf1-415e-4bb9-a5b4-a92a49d014a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    6171\n",
       "3    1737\n",
       "2    1476\n",
       "1     774\n",
       "Name: median_relevance, dtype: int64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking target labels -\n",
    "search_df['median_relevance'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "746c124b-e874-4727-8b31-43e023a2e558",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting textdistance\n",
      "  Using cached textdistance-4.2.2-py3-none-any.whl (28 kB)\n",
      "Installing collected packages: textdistance\n",
      "Successfully installed textdistance-4.2.2\n",
      "\u001b[33mWARNING: Target directory /home/repl/.local/lib/python3.8/site-packages/textdistance-4.2.2.dist-info already exists. Specify --upgrade to force replacement.\u001b[0m\n",
      "\u001b[33mWARNING: Target directory /home/repl/.local/lib/python3.8/site-packages/textdistance already exists. Specify --upgrade to force replacement.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install textdistance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "9bf4107f-eac2-426a-ae38-2875b9bd8e27",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing numbers from and punctuations from the title field\n",
    "import re\n",
    "def clean_data(text):\n",
    "    text=re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?','NUMBER', text)\n",
    "    text=re.sub(r'\\W+',' ',text,flags=re.M)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "4426c47a-6352-4755-a8aa-fd0ef3c52b45",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_title=search_df['product_title'].apply(clean_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "be555934-7fbd-4281-a8f3-5364d9f8e476",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=search_df['median_relevance']\n",
    "search_df=search_df.drop(['median_relevance','relevance_variance'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "94a24e70-cbcf-4304-8140-510f2de6851a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = search_df.drop('id', axis=1)\n",
    "test = search_test.drop('id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "e7a3401c-7656-448d-a473-e1b1df95e938",
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata = list(train.apply(lambda x:'%s %s' % (x['query'],x['product_title']),axis=1))\n",
    "testdata = list(test.apply(lambda x:'%s %s' % (x['query'],x['product_title']),axis=1)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "708068db-2656-4a64-ab9b-bc3f949dfc7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bridal shower decorations Accent Pillow with Heart Design - Red/Black',\n",
       " 'led christmas lights Set of 10 Battery Operated Multi LED Train Christmas Lights - Clear Wire',\n",
       " 'projector ViewSonic Pro8200 DLP Multimedia Projector',\n",
       " 'wine rack Concept Housewares WR-44526 Solid-Wood Ceiling/Wall-Mount Wine Rack, Charcoal Grey, 6 Bottle',\n",
       " 'light bulb Wintergreen Lighting Christmas LED Light Bulb (Pack of 25)']"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traindata[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "65d353a9-d67e-407f-80ef-369c5ec3649c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_cnv=TfidfVectorizer(stop_words='english',min_df=3,analyzer='word',ngram_range=(1,5),token_pattern=r'\\w{1,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "c1cd0837-cb69-4d4f-8157-386c4ab2e644",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dimensionality Reduction as an Initialization step\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd=TruncatedSVD()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "452c9ff3-dcfb-49ad-88fc-b2dff111313f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scaling the training data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "std=StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "690e493f-4994-4a36-8874-c962000f8374",
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM Model\n",
    "from sklearn.svm import SVC\n",
    "svm_model=SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "66b5c362-9e4e-4058-8af8-158bfc5f7d95",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a scorer(evaluator) function\n",
    "def build_confusion_matrix(rater_a,rater_b,min_rating=None,max_rating=None):\n",
    "    assert(len(rater_a)==len(rater_b))\n",
    "    if min_rating == None:\n",
    "        min_rating=min(rater_a + rater_b)\n",
    "    if max_rating == None:\n",
    "        max_rating=max(rater_a + rater_b)\n",
    "    num_ratings=int(max_rating - min_rating +1)\n",
    "    print(min_rating)\n",
    "    conf_mat=[[0 for i in range(num_ratings)] for j in range(num_ratings)]\n",
    "    for a,b in zip(rater_a,rater_b):\n",
    "        conf_mat[a - min_rating][b - min_rating] += 1\n",
    "    return conf_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "2890b511-48f8-42d4-8f2e-2cbc1560af20",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Count of each type of rating user makes\n",
    "def histogram(ratings,min_rating=None,max_rating=None):\n",
    "    if min_rating == None:\n",
    "        min_rating=min(ratings)\n",
    "    if max_rating == None:\n",
    "        max_rating=max(ratings)\n",
    "    num_ratings = int(max_rating - min_rating + 1)\n",
    "    hist_ratings = [0 for i in range(num_ratings)]\n",
    "    for i in hist_ratings:\n",
    "        hist_ratings[r - min_rating] += 1\n",
    "    return hist_ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "306f6002-5933-479a-afac-f0657cd1d423",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The kappa statistic is frequently used to test interrater reliability. The importance of rater reliability lies in the fact that it represents the extent to which \n",
    "#the data collected in the study are correct representations of the variables measured. Measurement of the extent to which data collectors (raters) assign the same \n",
    "#score to the same variable is called interrater reliability. While there have been a variety of methods to measure interrater reliability, traditionally it was \n",
    "#measured as percent agreement, calculated as the number of agreement scores divided by the total number of scores.\n",
    "#For deatils on Kappa - link - https://www.kaggle.com/aroraaman/quadratic-kappa-metric-explained-in-5-simple-steps\n",
    "def quadratic_weighted_kappa(y, y_pred):\n",
    "    rater_a=y\n",
    "    rater_b=y_pred\n",
    "    min_rating=None\n",
    "    max_rating=None\n",
    "    rater_a = np.array(rater_a, dtype=int)                     #they were df object so converted them into array\n",
    "    rater_b = np.array(rater_b, dtype=int)\n",
    "    assert(len(rater_a)==len(rater_b))\n",
    "    if min_rating == None:\n",
    "        min_rating=min(min(rater_a),min(rater_b))\n",
    "    if max_rating == None:\n",
    "        max_rating=max(max(rater_a),max(rater_b))\n",
    "    conf_mat=build_confusion_matrix(rater_a,rater_b,min_rating,max_rating)          #Maps the predicted and actual o/p values\n",
    "    num_ratings=len(conf_mat)\n",
    "    num_scored_items=float(len(rater_a))\n",
    "    \n",
    "    hist_rater_a = histogram(rater_a, min_rating, max_rating)                       #Create vectors depicting which catagory contains how many no of instances for \n",
    "    hist_rater_b = histogram(rater_b, min_rating, max_rating)                       #both predicted and actual\n",
    "    \n",
    "    numerator = 0.0\n",
    "    denominator = 0.0\n",
    "\n",
    "    for i in range(num_ratings):\n",
    "        for j in range(num_ratings):\n",
    "            expected_count = (hist_rater_a[i] * hist_rater_b[j]   #expected matrix - outer product of the above derived vectors and normalized using num_scored_items\n",
    "                              / num_scored_items)\n",
    "            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)       #Assigning a weight to be applied on each element of confusion matrix\n",
    "            numerator += d * conf_mat[i][j] / num_scored_items    #weight*confusion matrix along with normalization\n",
    "            denominator += d * expected_count / num_scored_items  #weight*expected matrix elemenr along with normalization\n",
    "\n",
    "    return (1.0 - numerator / denominator)                        #kappa coefficient formula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "3213e74e-5a4d-49c3-9da2-da9ef7939d81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting validation params for GridSearchCV\n",
    "param_grid={'svd__n_components':[200,400],\n",
    "            'svm__C':[10,12]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "7de0113e-582a-4df7-a38c-d5e4aa837b6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting model scorer\n",
    "from sklearn import metrics\n",
    "kappa_scorer = metrics.make_scorer(quadratic_weighted_kappa, greater_is_better = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "46e148d5-8a6a-469a-b182-e8959ef01bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import textdistance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "5b97ae8b-320e-4b27-b592-f25bcdb056da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.54899485, 0.50069396, 0.4114756 , ..., 0.52868053, 0.3441236 ,\n",
       "       0.56077215])"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Finding out the cosine distance of query and product-title\n",
    "import numpy as np\n",
    "dist=np.zeros([len(search_df)])\n",
    "idx=0\n",
    "for i,j in zip(search_df['query'],search_df['product_title']):\n",
    "    dist[idx]=(textdistance.cosine(i,j))\n",
    "    idx+=1\n",
    "dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "19a3eb76-82eb-42ed-aaf8-5219abcca591",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10158,)"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "61c7e87b-fbce-45f2-92a8-0b49d67d4ee2",
   "metadata": {},
   "outputs": [],
   "source": [
    "features=pd.DataFrame(traindata)\n",
    "features=features.rename(columns={0:'text'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "815da930-d508-4eb2-91c0-699382ad67a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_cnv=np.reshape(len(dist),1)\n",
    "features['distance']=pd.DataFrame(dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "db258f3e-44d5-4184-ae96-7e7cccda2bd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.62861856],\n",
       "       [0.49102862],\n",
       "       [0.71205164],\n",
       "       ...,\n",
       "       [0.36514837],\n",
       "       [0.62401997],\n",
       "       [0.40546538]])"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dist=np.zeros((len(testdata),1))\n",
    "idx=0\n",
    "for i,j in zip(search_test['query'],search_test['product_title']):\n",
    "    test_dist[idx]=textdistance.cosine(i,j)\n",
    "    idx+=1\n",
    "test_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "237ae0c9-3be9-4c4b-9197-5c1542312142",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>electric griddle Star-Max 48 in Electric Griddle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>phillips coffee maker Philips SENSEO HD7810 WH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>san francisco 49ers 2013 San Francisco 49ers C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aveeno shampoo AVEENO       10.5FLOZ NRSH SHIN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>flea and tick control for dogs Merial Frontlin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0   electric griddle Star-Max 48 in Electric Griddle\n",
       "1  phillips coffee maker Philips SENSEO HD7810 WH...\n",
       "2  san francisco 49ers 2013 San Francisco 49ers C...\n",
       "3  aveeno shampoo AVEENO       10.5FLOZ NRSH SHIN...\n",
       "4  flea and tick control for dogs Merial Frontlin..."
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_test=pd.DataFrame(testdata)\n",
    "feature_test=feature_test.rename(columns={0:'text'})\n",
    "feature_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "7fd079df-b4c3-41af-a289-bc4653f58789",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22513, 1)"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dist.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "192917ac-c510-45bd-a7cc-6b581e63d04b",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_test['distance']=pd.DataFrame(test_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "6b623b79-6dc4-4cd5-94ca-a43d806f32d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>electric griddle Star-Max 48 in Electric Griddle</td>\n",
       "      <td>0.628619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>phillips coffee maker Philips SENSEO HD7810 WH...</td>\n",
       "      <td>0.491029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>san francisco 49ers 2013 San Francisco 49ers C...</td>\n",
       "      <td>0.712052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aveeno shampoo AVEENO       10.5FLOZ NRSH SHIN...</td>\n",
       "      <td>0.045175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>flea and tick control for dogs Merial Frontlin...</td>\n",
       "      <td>0.578829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22508</th>\n",
       "      <td>seagate 2tb hard disk Seagate Backup Plus STDU...</td>\n",
       "      <td>0.370970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22509</th>\n",
       "      <td>storage ottoman Chevron Storage Ottoman, Gray ...</td>\n",
       "      <td>0.578829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22510</th>\n",
       "      <td>gym gloves Lion Martial Arts Large Red Grappli...</td>\n",
       "      <td>0.365148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22511</th>\n",
       "      <td>wreck it ralph Wreck-It Ralph (Original Score)</td>\n",
       "      <td>0.624020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22512</th>\n",
       "      <td>hair accessories Goody Slide proof Claw Clip, ...</td>\n",
       "      <td>0.405465</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22513 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  distance\n",
       "0       electric griddle Star-Max 48 in Electric Griddle  0.628619\n",
       "1      phillips coffee maker Philips SENSEO HD7810 WH...  0.491029\n",
       "2      san francisco 49ers 2013 San Francisco 49ers C...  0.712052\n",
       "3      aveeno shampoo AVEENO       10.5FLOZ NRSH SHIN...  0.045175\n",
       "4      flea and tick control for dogs Merial Frontlin...  0.578829\n",
       "...                                                  ...       ...\n",
       "22508  seagate 2tb hard disk Seagate Backup Plus STDU...  0.370970\n",
       "22509  storage ottoman Chevron Storage Ottoman, Gray ...  0.578829\n",
       "22510  gym gloves Lion Martial Arts Large Red Grappli...  0.365148\n",
       "22511     wreck it ralph Wreck-It Ralph (Original Score)  0.624020\n",
       "22512  hair accessories Goody Slide proof Claw Clip, ...  0.405465\n",
       "\n",
       "[22513 rows x 2 columns]"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "cc9f7ec2-9d4f-4f03-8c95-c68546ae1e6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "feature_conv=ColumnTransformer([\n",
    "    ('tf_idf',tfidf_cnv,'text')],\n",
    "    remainder='passthrough'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "170330b3-3ce9-4f0d-8a36-a3daf4669c7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_new=feature_conv.fit_transform(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "2360b50e-7866-4078-823f-eb1dd751991d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a pipeline to initialize, scale and Train the model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "80dd8526-e7b6-46a2-8240-53baba5a436a",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_model2=Pipeline([\n",
    "    ('svd',svd),\n",
    "    ('std',std),\n",
    "    ('svm',svm_model)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cf0e6958-c57e-437f-be72-d6022c0613aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_clf=GridSearchCV(estimator=clf_model2,param_grid=param_grid,scoring=kappa_scorer,verbose=10, n_jobs=-1, refit=True, cv=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "537280df-4591-4c14-aba3-5ea692f6f9a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=Pipeline(steps=[('svd', TruncatedSVD()),\n",
       "                                       ('std', StandardScaler()),\n",
       "                                       ('svm', SVC())]),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'svd__n_components': [200, 400], 'svm__C': [10, 12]},\n",
       "             scoring=make_scorer(quadratic_weighted_kappa), verbose=10)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "e3efeae0-b4b5-4349-b4a9-1e427c3b9698",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_new=feature_conv.transform(feature_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9e122b10-be85-4f2a-b253-f6768144f399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('std', StandardScaler()), ('svm', SVC(C=10))])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1c22696a-2fff-4d96-932e-bea52953a8dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2=grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "50c2800e-e91d-40df-8843-08057ff519e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('std', StandardScaler()), ('svm', SVC(C=10))])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7e8f233c-2ced-4f77-9b5d-f390ea9d01cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7843079346328017"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred_model2=model_2.predict(X_train_new)\n",
    "accuracy_score(y_train, y_pred_model2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "cc23a10c-7e02-4e31-97dd-ba54def44d1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_model2=model_2.predict(X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "26455dde-d409-44dc-a56d-180e9a201ae6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1, 2, 3, 4]), array([ 1050,  2269,  1728, 17466]))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_test_model2,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "943284fd-9c36-4f8a-8d40-b324889a3c87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    13680\n",
       "3     3849\n",
       "2     3270\n",
       "1     1714\n",
       "Name: prediction, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Copying the test output label\n",
    "test_label=pd.read_csv('SearchRelevanceSubmission.csv')\n",
    "test_label['prediction'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a9fe10e8-b9ec-4b54-bc57-c6173821bedc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7254031004308622"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(test_label['prediction'],y_test_model2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "144b3f10-3be3-4649-b3aa-670037d6bae1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#RandomForestClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest=RandomForestClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "88edc436-7b3c-4a4f-b70c-f6e05dfe0f5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_2=Pipeline([\n",
    "    ('svd',svd),\n",
    "#    ('std',std),\n",
    "    ('model',forest)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "289f022e-3542-4f27-989d-880c8d06efd2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['memory', 'steps', 'verbose', 'svd', 'model', 'svd__algorithm', 'svd__n_components', 'svd__n_iter', 'svd__random_state', 'svd__tol', 'model__bootstrap', 'model__ccp_alpha', 'model__class_weight', 'model__criterion', 'model__max_depth', 'model__max_features', 'model__max_leaf_nodes', 'model__max_samples', 'model__min_impurity_decrease', 'model__min_impurity_split', 'model__min_samples_leaf', 'model__min_samples_split', 'model__min_weight_fraction_leaf', 'model__n_estimators', 'model__n_jobs', 'model__oob_score', 'model__random_state', 'model__verbose', 'model__warm_start'])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_2.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "b84820b6-8a88-42db-b91c-0b9e51545d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={'svd__n_components':[200,400],\n",
    "            'model__n_estimators':[10,12,15],\n",
    "            'model__max_depth':[20,30]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "b753dffd-8f28-4b6b-b1d7-15ec3e2c9713",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_clf=GridSearchCV(estimator=clf_2,param_grid=param_grid,scoring=kappa_scorer,verbose=10, n_jobs=-1, refit=True, cv=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "8ef15f3a-89c6-432a-a07d-9757acd5bb57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 12 candidates, totalling 48 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4,\n",
       "             estimator=Pipeline(steps=[('svd', TruncatedSVD()),\n",
       "                                       ('model',\n",
       "                                        RandomForestClassifier(random_state=42))]),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'model__max_depth': [20, 30],\n",
       "                         'model__n_estimators': [10, 12, 15],\n",
       "                         'svd__n_components': [200, 400]},\n",
       "             scoring=make_scorer(quadratic_weighted_kappa), verbose=10)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "ed0e7abb-91ff-4033-b726-1013c8da1921",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('model',\n",
       "                 RandomForestClassifier(max_depth=20, n_estimators=10,\n",
       "                                        random_state=42))])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "264b325d-2bd1-4ef2-9301-040753f1482a",
   "metadata": {},
   "outputs": [],
   "source": [
    "forest_model=grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "b82f9bc5-b0a5-4dc1-8d56-6bf8cb273005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('model',\n",
       "                 RandomForestClassifier(max_depth=20, n_estimators=10,\n",
       "                                        random_state=42))])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_model.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "21efec39-6871-4e94-a94d-90bfd2bd05c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9392596967907069"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_forest=forest_model.predict(X_train_new)\n",
    "accuracy_score(y_train,y_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3f5e8c0b-7705-46ff-b31e-cdfc9b4c4a19",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_test=forest_model.predict(X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "d1244b0d-8c6d-4c3c-82dd-60f814ee05d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6732110336250167"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/4; 6/12] START model__max_depth=10, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 6/12] END model__max_depth=10, model__n_estimators=15, svd__n_components=400; total time=   6.9s\n",
      "[CV 1/4; 7/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 7/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.5s\n",
      "[CV 3/4; 7/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 7/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.3s\n",
      "[CV 1/4; 8/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 8/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.1s\n",
      "[CV 4/4; 8/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 8/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 2/4; 9/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 9/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 4/4; 9/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 9/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.4s\n",
      "[CV 2/4; 10/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 10/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.3s\n",
      "[CV 4/4; 10/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 10/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.3s\n",
      "[CV 2/4; 11/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 11/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.8s\n",
      "[CV 4/4; 11/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 11/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.7s\n",
      "[CV 2/4; 12/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 12/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.6s\n",
      "[CV 4/4; 12/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 12/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 2/4; 1/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 1/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.3s\n",
      "[CV 3/4; 1/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 1/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.2s\n",
      "[CV 1/4; 2/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 2/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   6.9s\n",
      "[CV 3/4; 2/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 2/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 1/4; 3/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 3/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.4s\n",
      "[CV 3/4; 3/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 3/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 1/4; 4/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 4/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.3s\n",
      "[CV 3/4; 4/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 4/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.2s\n",
      "[CV 1/4; 5/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 5/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.7s\n",
      "[CV 3/4; 5/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 5/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.7s\n",
      "[CV 1/4; 6/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 6/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 3/4; 6/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 6/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 1/4; 7/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 7/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=200; total time=   3.2s\n",
      "[CV 3/4; 7/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 7/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=200; total time=   3.4s\n",
      "[CV 1/4; 8/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 8/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 3/4; 8/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 8/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=400; total time=   7.2s\n",
      "[CV 1/4; 9/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 9/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 3/4; 9/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 9/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 1/4; 10/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 10/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=400; total time=   7.5s\n",
      "[CV 3/4; 10/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 10/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=400; total time=   7.7s\n",
      "[CV 1/4; 11/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 11/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=200; total time=   3.9s\n",
      "[CV 3/4; 11/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 11/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=200; total time=   3.9s\n",
      "[CV 1/4; 12/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 12/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=400; total time=   8.2s\n",
      "[CV 3/4; 12/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 12/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=400; total time=   8.1s\n",
      "[CV 4/4; 6/12] START model__max_depth=10, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 6/12] END model__max_depth=10, model__n_estimators=15, svd__n_components=400; total time=   6.9s\n",
      "[CV 2/4; 7/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 7/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.4s\n",
      "[CV 4/4; 7/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 7/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.2s\n",
      "[CV 2/4; 8/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 8/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 3/4; 8/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 8/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   6.9s\n",
      "[CV 1/4; 9/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 9/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.4s\n",
      "[CV 3/4; 9/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 9/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.4s\n",
      "[CV 1/4; 10/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 10/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.2s\n",
      "[CV 3/4; 10/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 10/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.3s\n",
      "[CV 1/4; 11/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 11/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.8s\n",
      "[CV 3/4; 11/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 3/4; 11/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.7s\n",
      "[CV 1/4; 12/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 1/4; 12/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.6s\n",
      "[CV 3/4; 12/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 3/4; 12/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 1/4; 1/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 1/4; 1/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.3s\n",
      "[CV 4/4; 1/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 1/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=200; total time=   3.2s\n",
      "[CV 2/4; 2/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 2/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 4/4; 2/12] START model__max_depth=20, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 2/12] END model__max_depth=20, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 2/4; 3/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 3/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 4/4; 3/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 3/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 2/4; 4/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 4/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.3s\n",
      "[CV 4/4; 4/12] START model__max_depth=20, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 4/12] END model__max_depth=20, model__n_estimators=12, svd__n_components=400; total time=   7.2s\n",
      "[CV 2/4; 5/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 5/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.8s\n",
      "[CV 4/4; 5/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 5/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=200; total time=   3.7s\n",
      "[CV 2/4; 6/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 6/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 4/4; 6/12] START model__max_depth=20, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 6/12] END model__max_depth=20, model__n_estimators=15, svd__n_components=400; total time=   7.7s\n",
      "[CV 2/4; 7/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 7/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=200; total time=   3.2s\n",
      "[CV 4/4; 7/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 7/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=200; total time=   3.3s\n",
      "[CV 2/4; 8/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 8/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=400; total time=   7.0s\n",
      "[CV 4/4; 8/12] START model__max_depth=30, model__n_estimators=10, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 8/12] END model__max_depth=30, model__n_estimators=10, svd__n_components=400; total time=   7.1s\n",
      "[CV 2/4; 9/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 9/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=200; total time=   3.5s\n",
      "[CV 4/4; 9/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 9/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=200; total time=   3.4s\n",
      "[CV 2/4; 10/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 10/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=400; total time=   7.5s\n",
      "[CV 4/4; 10/12] START model__max_depth=30, model__n_estimators=12, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 10/12] END model__max_depth=30, model__n_estimators=12, svd__n_components=400; total time=   7.7s\n",
      "[CV 2/4; 11/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 2/4; 11/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=200; total time=   3.9s\n",
      "[CV 4/4; 11/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=200\n",
      "1\n",
      "[CV 4/4; 11/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=200; total time=   3.8s\n",
      "[CV 2/4; 12/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 2/4; 12/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=400; total time=   8.2s\n",
      "[CV 4/4; 12/12] START model__max_depth=30, model__n_estimators=15, svd__n_components=400\n",
      "1\n",
      "[CV 4/4; 12/12] END model__max_depth=30, model__n_estimators=15, svd__n_components=400; total time=   8.2s\n"
     ]
    }
   ],
   "source": [
    "accuracy_score(test_label['prediction'],y_pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "3688fa0d-2579-44f1-9cf9-26c91984e68b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1, 2, 3, 4]), array([ 701, 1344, 1380, 6733]))"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_pred_forest,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "5bab6572-dd19-4ef1-87a9-b576c4d63abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Logistic Model building\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "log_clf=LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "d70f7577-5406-4e28-893a-6bd9ca0a5808",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating Pipeline for Logistic Regression\n",
    "clf_log=Pipeline([\n",
    "    ('svd',svd),\n",
    "    ('std',std),\n",
    "    ('model',log_clf)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "62355197-a839-4a9d-b9af-44e5177bc106",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['memory', 'steps', 'verbose', 'svd', 'std', 'model', 'svd__algorithm', 'svd__n_components', 'svd__n_iter', 'svd__random_state', 'svd__tol', 'std__copy', 'std__with_mean', 'std__with_std', 'model__C', 'model__class_weight', 'model__dual', 'model__fit_intercept', 'model__intercept_scaling', 'model__l1_ratio', 'model__max_iter', 'model__multi_class', 'model__n_jobs', 'model__penalty', 'model__random_state', 'model__solver', 'model__tol', 'model__verbose', 'model__warm_start'])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_log.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "f5e77450-ef74-470f-b9d9-0f503b956860",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={'svd__n_components':[200,400],\n",
    "            'model__C':[10,12]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "5ff7aca8-349e-423a-a02c-b2661e4bb01c",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_clf=GridSearchCV(estimator=clf_log,param_grid=param_grid,scoring=kappa_scorer,verbose=10, n_jobs=-1, refit=True, cv=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "c32e074d-a1a1-4dae-a4ef-6708b619cbd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 4 candidates, totalling 16 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4,\n",
       "             estimator=Pipeline(steps=[('svd', TruncatedSVD()),\n",
       "                                       ('std', StandardScaler()),\n",
       "                                       ('model', LogisticRegression())]),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'model__C': [10, 12], 'svd__n_components': [200, 400]},\n",
       "             scoring=make_scorer(quadratic_weighted_kappa), verbose=10)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "fae790b5-f59c-4899-a583-265d1b78c822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('std', StandardScaler()),\n",
       "                ('model', LogisticRegression(C=10))])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "d1b57d29-17ea-4f2a-b4c1-5857aec6f3d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_log=grid_search_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "5eeb8c75-8585-4312-84c3-b53125e1b04f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('svd', TruncatedSVD(n_components=200)),\n",
       "                ('std', StandardScaler()),\n",
       "                ('model', LogisticRegression(C=10))])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_log.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "2d635f0d-bdeb-489a-ba97-d3ab2b7be378",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6311281748375664"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_log_train=model_log.predict(X_train_new)\n",
    "accuracy_score(y_train,y_log_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "a515c41c-a0a5-4e18-921a-08db8f0cd44a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6471372096122241"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_log_test=model_log.predict(X_test_new)\n",
    "accuracy_score(test_label['prediction'],y_log_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "d5c11c88-0abc-4c51-a13e-7acaec70c53f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stacking models SVC and RandomForest using LogisticRegression\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "estimator=[\n",
    "#    ('svc',make_pipeline(TruncatedSVD(n_components=200, random_state=42),StandardScaler(),SVC(random_state=42))),\n",
    "#    ('forest',make_pipeline(TruncatedSVD(n_components=200, random_state=42),RandomForestClassifier(random_state=42,n_estimators=10,max_depth=20)))\n",
    "    ('SVC',model_2),\n",
    "    ('forest',forest_model)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "c552cd5f-ae5b-4cc2-8db3-45fcb9e84176",
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_clf=StackingClassifier(estimators=estimator,final_estimator=LogisticRegression())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "1e657c1f-66d3-448a-bc7b-0b4df8138538",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StackingClassifier(estimators=[('SVC',\n",
       "                                Pipeline(steps=[('svd',\n",
       "                                                 TruncatedSVD(n_components=200)),\n",
       "                                                ('std', StandardScaler()),\n",
       "                                                ('svm', SVC(C=10))])),\n",
       "                               ('forest',\n",
       "                                Pipeline(steps=[('svd',\n",
       "                                                 TruncatedSVD(n_components=200)),\n",
       "                                                ('model',\n",
       "                                                 RandomForestClassifier(max_depth=20,\n",
       "                                                                        n_estimators=10,\n",
       "                                                                        random_state=42))]))],\n",
       "                   final_estimator=LogisticRegression())"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack_clf.fit(X_train_new,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "d8688d10-8183-4d98-b420-17d909784497",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_final_pred=stack_clf.predict(X_train_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "5721d60c-8300-4af4-9197-938707da8ad4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8041937389249852"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_train,y_final_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "d26bfdf8-910f-42a4-a077-978457119b9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_final_test=stack_clf.predict(X_test_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "cf800fce-e4ec-44b3-9b9c-1dcbe564e054",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7059476746768534"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(test_label['prediction'],y_final_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "e6e2d1e1-219d-4eba-8de8-a1eb1d15146c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22508</th>\n",
       "      <td>32665</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22509</th>\n",
       "      <td>32667</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22510</th>\n",
       "      <td>32669</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22511</th>\n",
       "      <td>32670</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22512</th>\n",
       "      <td>32671</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22513 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  prediction\n",
       "0          3           4\n",
       "1          6           3\n",
       "2          9           4\n",
       "3         11           4\n",
       "4         12           4\n",
       "...      ...         ...\n",
       "22508  32665           4\n",
       "22509  32667           4\n",
       "22510  32669           1\n",
       "22511  32670           4\n",
       "22512  32671           4\n",
       "\n",
       "[22513 rows x 2 columns]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subm=pd.DataFrame(search_test['id'])\n",
    "subm['prediction']=y_final_test\n",
    "subm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "91723c71-91d0-46a1-a3e1-c2bb465e1e0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    18690\n",
       "2     2141\n",
       "1     1155\n",
       "3      527\n",
       "Name: prediction, dtype: int64"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subm['prediction'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "896de20f-97ec-4ba6-a951-ebf61aa1071c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}