Classic_ML_Model.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import copy\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.experimental import enable_halving_search_cv\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV, HalvingGridSearchCV,RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "seed = 42\n",
    "pca = None\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "df = pd.read_csv('dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>(no genres listed)</th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Documentary</th>\n",
       "      <th>Drama</th>\n",
       "      <th>...</th>\n",
       "      <th>world politics</th>\n",
       "      <th>world war i</th>\n",
       "      <th>world war ii</th>\n",
       "      <th>writer's life</th>\n",
       "      <th>writers</th>\n",
       "      <th>writing</th>\n",
       "      <th>wuxia</th>\n",
       "      <th>wwii</th>\n",
       "      <th>zombie</th>\n",
       "      <th>zombies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.893708</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.04050</td>\n",
       "      <td>0.01425</td>\n",
       "      <td>0.03050</td>\n",
       "      <td>0.03500</td>\n",
       "      <td>0.14125</td>\n",
       "      <td>0.05775</td>\n",
       "      <td>0.03900</td>\n",
       "      <td>0.02975</td>\n",
       "      <td>0.08475</td>\n",
       "      <td>0.02200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.251527</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05250</td>\n",
       "      <td>0.01575</td>\n",
       "      <td>0.01250</td>\n",
       "      <td>0.02000</td>\n",
       "      <td>0.12225</td>\n",
       "      <td>0.03275</td>\n",
       "      <td>0.02100</td>\n",
       "      <td>0.01100</td>\n",
       "      <td>0.10525</td>\n",
       "      <td>0.01975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.142028</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06275</td>\n",
       "      <td>0.01950</td>\n",
       "      <td>0.02225</td>\n",
       "      <td>0.02300</td>\n",
       "      <td>0.12200</td>\n",
       "      <td>0.03475</td>\n",
       "      <td>0.01700</td>\n",
       "      <td>0.01800</td>\n",
       "      <td>0.09100</td>\n",
       "      <td>0.01775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.853547</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05325</td>\n",
       "      <td>0.02800</td>\n",
       "      <td>0.01675</td>\n",
       "      <td>0.03875</td>\n",
       "      <td>0.18200</td>\n",
       "      <td>0.07050</td>\n",
       "      <td>0.01625</td>\n",
       "      <td>0.01425</td>\n",
       "      <td>0.08850</td>\n",
       "      <td>0.01500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.058434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05350</td>\n",
       "      <td>0.02050</td>\n",
       "      <td>0.01425</td>\n",
       "      <td>0.02550</td>\n",
       "      <td>0.19225</td>\n",
       "      <td>0.02675</td>\n",
       "      <td>0.01625</td>\n",
       "      <td>0.01300</td>\n",
       "      <td>0.08700</td>\n",
       "      <td>0.01600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1149 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     rating  (no genres listed)  Action  Adventure  Animation  Children   \n",
       "0  3.893708                   0       0          1          1         1  \\\n",
       "1  3.251527                   0       0          1          0         1   \n",
       "2  3.142028                   0       0          0          0         0   \n",
       "3  2.853547                   0       0          0          0         0   \n",
       "4  3.058434                   0       0          0          0         0   \n",
       "\n",
       "   Comedy  Crime  Documentary  Drama  ...  world politics  world war i   \n",
       "0       1      0            0      0  ...         0.04050      0.01425  \\\n",
       "1       0      0            0      0  ...         0.05250      0.01575   \n",
       "2       1      0            0      0  ...         0.06275      0.01950   \n",
       "3       1      0            0      1  ...         0.05325      0.02800   \n",
       "4       1      0            0      0  ...         0.05350      0.02050   \n",
       "\n",
       "   world war ii  writer's life  writers  writing    wuxia     wwii   zombie   \n",
       "0       0.03050        0.03500  0.14125  0.05775  0.03900  0.02975  0.08475  \\\n",
       "1       0.01250        0.02000  0.12225  0.03275  0.02100  0.01100  0.10525   \n",
       "2       0.02225        0.02300  0.12200  0.03475  0.01700  0.01800  0.09100   \n",
       "3       0.01675        0.03875  0.18200  0.07050  0.01625  0.01425  0.08850   \n",
       "4       0.01425        0.02550  0.19225  0.02675  0.01625  0.01300  0.08700   \n",
       "\n",
       "   zombies  \n",
       "0  0.02200  \n",
       "1  0.01975  \n",
       "2  0.01775  \n",
       "3  0.01500  \n",
       "4  0.01600  \n",
       "\n",
       "[5 rows x 1149 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#splitting dataframe df into train and test\n",
    "X=df.drop(['rating'],axis=1)\n",
    "y=df['rating']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training samples: (9946, 1148)\n",
      "Number of testing samples: (2764, 1148)\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of training samples: {X_train.shape}')\n",
    "print(f'Number of testing samples: {X_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PCA\n",
    "pca = PCA(n_components=0.95)\n",
    "pca.fit(X_train)\n",
    "X_train = pca.transform(X_train)\n",
    "X_val = pca.transform(X_val)\n",
    "X_test = pca.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear Regression R2 score: 0.9707843273740225\n",
      "Linear Regression MSE: 0.006476124027323298\n"
     ]
    }
   ],
   "source": [
    "# Linear Regression\n",
    "model = LinearRegression()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "print(f'Linear Regression R2 score: {model.score(X_test, y_test)}')\n",
    "print(f'Linear Regression MSE: {mean_squared_error(y_test, model.predict(X_test))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_model_hyperparams(grid,model_name):\n",
    "    writer = SummaryWriter('run/PCA/{}'.format(model_name))\n",
    "    for i in range(len(grid.cv_results_['params'])):\n",
    "        writer.add_hparams(\n",
    "            {f'{k}': v for k, v in grid.cv_results_['params'][i].items()},\n",
    "            {f'{model_name}_mean_test_score': grid.cv_results_['mean_test_score'][i]}\n",
    "        )\n",
    "    writer.flush()\n",
    "    writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 10 candidates, totalling 20 fits\n",
      "[CV 1/2] END max_depth=10, max_features=sqrt, min_samples_leaf=3, min_samples_split=2, n_estimators=50;, score=0.537 total time=   6.1s\n",
      "[CV 2/2] END max_depth=10, max_features=sqrt, min_samples_leaf=3, min_samples_split=2, n_estimators=50;, score=0.515 total time=   5.8s\n",
      "[CV 1/2] END max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=100;, score=0.745 total time= 2.4min\n",
      "[CV 2/2] END max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=100;, score=0.743 total time= 2.4min\n",
      "[CV 1/2] END max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=50;, score=0.507 total time=   5.6s\n",
      "[CV 2/2] END max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=50;, score=0.523 total time=   5.7s\n",
      "[CV 1/2] END max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=200;, score=0.747 total time= 4.7min\n",
      "[CV 2/2] END max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=200;, score=0.743 total time= 4.7min\n",
      "[CV 1/2] END max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=2, n_estimators=50;, score=0.282 total time=   3.2s\n",
      "[CV 2/2] END max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=2, n_estimators=50;, score=0.318 total time=   3.3s\n",
      "[CV 1/2] END max_depth=10, max_features=log2, min_samples_leaf=2, min_samples_split=4, n_estimators=100;, score=0.307 total time=   4.8s\n",
      "[CV 2/2] END max_depth=10, max_features=log2, min_samples_leaf=2, min_samples_split=4, n_estimators=100;, score=0.299 total time=   4.8s\n",
      "[CV 1/2] END max_depth=10, max_features=log2, min_samples_leaf=3, min_samples_split=4, n_estimators=200;, score=0.308 total time=   9.6s\n",
      "[CV 2/2] END max_depth=10, max_features=log2, min_samples_leaf=3, min_samples_split=4, n_estimators=200;, score=0.328 total time=   9.5s\n",
      "[CV 1/2] END max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=2, n_estimators=100;, score=0.748 total time= 2.6min\n",
      "[CV 2/2] END max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=2, n_estimators=100;, score=0.742 total time= 2.4min\n",
      "[CV 1/2] END max_depth=5, max_features=auto, min_samples_leaf=3, min_samples_split=4, n_estimators=100;, score=0.748 total time= 2.3min\n",
      "[CV 2/2] END max_depth=5, max_features=auto, min_samples_leaf=3, min_samples_split=4, n_estimators=100;, score=0.742 total time= 2.4min\n",
      "[CV 1/2] END max_depth=10, max_features=sqrt, min_samples_leaf=3, min_samples_split=2, n_estimators=200;, score=0.510 total time=  23.1s\n",
      "[CV 2/2] END max_depth=10, max_features=sqrt, min_samples_leaf=3, min_samples_split=2, n_estimators=200;, score=0.522 total time=  22.7s\n",
      "\n",
      "Best R2: 0.745117\n",
      "Best parameters: {'n_estimators': 100, 'min_samples_split': 4, 'min_samples_leaf': 3, 'max_features': 'auto', 'max_depth': 5}\n",
      "Random Forest Regressor R2 score: 0.7534747767024224\n",
      "Random Forest Regressor MSE: 0.05464628325959238\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#hyperparameters tuning for random forest regressor\n",
    "model=RandomForestRegressor()\n",
    "\n",
    "#generate random number\n",
    "rng = np.random.RandomState(0)\n",
    "param_grid = {\n",
    "    'n_estimators': [50,100,200],\n",
    "    'max_depth': [5,10],\n",
    "    'min_samples_split': [2, 4],\n",
    "    'min_samples_leaf': [1, 2, 3],\n",
    "    'max_features': ['auto', 'sqrt', 'log2']\n",
    "}\n",
    "\n",
    "hgs = RandomizedSearchCV(\n",
    "    estimator=model, param_distributions=param_grid, random_state=rng, verbose=3, cv=2\n",
    ")\n",
    "hgs.fit(X_train, y_train)\n",
    "plot_model_hyperparams(hgs, 'RFR_Bis')\n",
    "\n",
    "\n",
    "print(\"\\nBest R2: {:.6f}\".format(hgs.best_score_))\n",
    "print(\"Best parameters: {}\".format(hgs.best_params_))\n",
    "\n",
    "print(f'Random Forest Regressor R2 score: {hgs.score(X_test, y_test)}')\n",
    "print(f'Random Forest Regressor MSE: {mean_squared_error(y_test, hgs.predict(X_test))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.52586566, 0.7439882 , 0.51488839, 0.74504078, 0.30006974,\n",
       "       0.30271001, 0.31790306, 0.74501493, 0.74511704, 0.51596005])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hgs.cv_results_['mean_test_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_iterations: 4\n",
      "n_required_iterations: 4\n",
      "n_possible_iterations: 4\n",
      "min_resources_: 368\n",
      "max_resources_: 9946\n",
      "aggressive_elimination: False\n",
      "factor: 3\n",
      "----------\n",
      "iter: 0\n",
      "n_candidates: 72\n",
      "n_resources: 368\n",
      "Fitting 2 folds for each of 72 candidates, totalling 144 fits\n",
      "[CV 1/2] END C=0.001, epsilon=0.001, kernel=linear;, score=(train=0.551, test=0.457) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.001, kernel=linear;, score=(train=0.455, test=0.446) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.001, kernel=poly;, score=(train=0.005, test=-0.070) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.001, kernel=poly;, score=(train=-0.004, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.001, kernel=rbf;, score=(train=0.009, test=-0.060) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.001, kernel=rbf;, score=(train=-0.003, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.01, kernel=linear;, score=(train=0.548, test=0.454) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.01, kernel=linear;, score=(train=0.448, test=0.437) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.01, kernel=poly;, score=(train=0.005, test=-0.070) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.01, kernel=poly;, score=(train=-0.005, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.01, kernel=rbf;, score=(train=0.008, test=-0.062) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.01, kernel=rbf;, score=(train=-0.005, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.1, kernel=linear;, score=(train=0.539, test=0.450) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.1, kernel=linear;, score=(train=0.468, test=0.458) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.1, kernel=poly;, score=(train=-0.004, test=-0.098) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.1, kernel=poly;, score=(train=-0.013, test=-0.022) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=0.1, kernel=rbf;, score=(train=-0.001, test=-0.088) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=0.1, kernel=rbf;, score=(train=-0.012, test=-0.017) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=1, kernel=linear;, score=(train=-0.179, test=-0.067) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=1, kernel=linear;, score=(train=-0.130, test=-0.142) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=1, kernel=poly;, score=(train=-0.236, test=-0.117) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=1, kernel=poly;, score=(train=-0.150, test=-0.162) total time=   0.0s\n",
      "[CV 1/2] END C=0.001, epsilon=1, kernel=rbf;, score=(train=-0.237, test=-0.117) total time=   0.0s\n",
      "[CV 2/2] END C=0.001, epsilon=1, kernel=rbf;, score=(train=-0.145, test=-0.155) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.933, test=0.844) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.903, test=0.819) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=poly;, score=(train=0.112, test=-0.009) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=poly;, score=(train=0.102, test=0.062) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=rbf;, score=(train=0.151, test=0.094) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=rbf;, score=(train=0.137, test=0.122) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.932, test=0.843) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.903, test=0.820) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=poly;, score=(train=0.111, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=poly;, score=(train=0.100, test=0.061) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=rbf;, score=(train=0.149, test=0.088) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=rbf;, score=(train=0.135, test=0.120) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.1, kernel=linear;, score=(train=0.911, test=0.819) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.1, kernel=linear;, score=(train=0.878, test=0.806) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.1, kernel=poly;, score=(train=0.095, test=-0.046) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.1, kernel=poly;, score=(train=0.094, test=0.055) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.1, kernel=rbf;, score=(train=0.135, test=0.044) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.1, kernel=rbf;, score=(train=0.133, test=0.118) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=1, kernel=linear;, score=(train=-0.243, test=-0.153) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=1, kernel=linear;, score=(train=-0.161, test=-0.182) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=1, kernel=poly;, score=(train=-0.239, test=-0.126) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=1, kernel=poly;, score=(train=-0.174, test=-0.195) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=1, kernel=rbf;, score=(train=-0.236, test=-0.115) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=1, kernel=rbf;, score=(train=-0.138, test=-0.151) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.920) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=0.999, test=0.905) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.001, kernel=poly;, score=(train=0.520, test=0.259) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.001, kernel=poly;, score=(train=0.431, test=0.254) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.001, kernel=rbf;, score=(train=0.756, test=0.659) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.001, kernel=rbf;, score=(train=0.747, test=0.644) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.999, test=0.920) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.999, test=0.907) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=poly;, score=(train=0.518, test=0.255) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=poly;, score=(train=0.432, test=0.255) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=rbf;, score=(train=0.755, test=0.657) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=rbf;, score=(train=0.746, test=0.644) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.979, test=0.897) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.973, test=0.898) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.1, kernel=poly;, score=(train=0.491, test=0.223) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.1, kernel=poly;, score=(train=0.434, test=0.270) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.1, kernel=rbf;, score=(train=0.731, test=0.632) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.1, kernel=rbf;, score=(train=0.713, test=0.619) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=1, kernel=linear;, score=(train=-0.278, test=-0.187) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=1, kernel=linear;, score=(train=-0.161, test=-0.182) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=1, kernel=poly;, score=(train=-0.268, test=-0.174) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=1, kernel=poly;, score=(train=-0.261, test=-0.302) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=1, kernel=rbf;, score=(train=-0.217, test=-0.098) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=1, kernel=rbf;, score=(train=-0.133, test=-0.153) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.918) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.905) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.001, kernel=poly;, score=(train=0.926, test=0.564) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.001, kernel=poly;, score=(train=0.918, test=0.325) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.001, kernel=rbf;, score=(train=0.998, test=0.875) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.853) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.919) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.908) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.01, kernel=poly;, score=(train=0.926, test=0.563) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.01, kernel=poly;, score=(train=0.918, test=0.325) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.01, kernel=rbf;, score=(train=0.998, test=0.873) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.01, kernel=rbf;, score=(train=0.999, test=0.851) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.979, test=0.897) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.973, test=0.898) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=poly;, score=(train=0.900, test=0.545) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=poly;, score=(train=0.890, test=0.344) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=rbf;, score=(train=0.972, test=0.851) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.827) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=1, kernel=linear;, score=(train=-0.278, test=-0.187) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=1, kernel=linear;, score=(train=-0.161, test=-0.182) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=1, kernel=poly;, score=(train=-0.383, test=-0.312) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=1, kernel=poly;, score=(train=-0.317, test=-0.369) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=1, kernel=rbf;, score=(train=-0.276, test=-0.168) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=1, kernel=rbf;, score=(train=-0.137, test=-0.158) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.918) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.905) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.001, kernel=poly;, score=(train=0.973, test=0.596) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.001, kernel=poly;, score=(train=0.967, test=0.234) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.878) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.855) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.919) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.908) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.01, kernel=poly;, score=(train=0.973, test=0.595) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.01, kernel=poly;, score=(train=0.967, test=0.242) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.876) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.853) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.979, test=0.897) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.973, test=0.898) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=poly;, score=(train=0.947, test=0.578) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=poly;, score=(train=0.933, test=0.285) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=rbf;, score=(train=0.973, test=0.852) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.827) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=1, kernel=linear;, score=(train=-0.278, test=-0.187) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=1, kernel=linear;, score=(train=-0.161, test=-0.182) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=1, kernel=poly;, score=(train=-0.398, test=-0.329) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=1, kernel=poly;, score=(train=-0.317, test=-0.369) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=1, kernel=rbf;, score=(train=-0.276, test=-0.168) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=1, kernel=rbf;, score=(train=-0.137, test=-0.158) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.918) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.001, kernel=linear;, score=(train=1.000, test=0.905) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.001, kernel=poly;, score=(train=1.000, test=0.609) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.001, kernel=poly;, score=(train=1.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.878) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.855) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.919) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.908) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.01, kernel=poly;, score=(train=1.000, test=0.607) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.01, kernel=poly;, score=(train=0.999, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.876) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.853) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.979, test=0.897) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.973, test=0.898) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=poly;, score=(train=0.971, test=0.589) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=poly;, score=(train=0.964, test=0.144) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=rbf;, score=(train=0.973, test=0.852) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.827) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=1, kernel=linear;, score=(train=-0.278, test=-0.187) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=1, kernel=linear;, score=(train=-0.161, test=-0.182) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=1, kernel=poly;, score=(train=-0.398, test=-0.329) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=1, kernel=poly;, score=(train=-0.317, test=-0.369) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=1, kernel=rbf;, score=(train=-0.276, test=-0.168) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=1, kernel=rbf;, score=(train=-0.137, test=-0.158) total time=   0.0s\n",
      "----------\n",
      "iter: 1\n",
      "n_candidates: 24\n",
      "n_resources: 1104\n",
      "Fitting 2 folds for each of 24 candidates, totalling 48 fits\n",
      "[CV 1/2] END C=0.01, epsilon=0.1, kernel=linear;, score=(train=0.949, test=0.916) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.1, kernel=linear;, score=(train=0.939, test=0.909) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.965, test=0.932) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.955, test=0.927) total time=   0.0s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.966, test=0.932) total time=   0.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.956, test=0.927) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=rbf;, score=(train=0.971, test=0.888) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.875) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=rbf;, score=(train=0.972, test=0.889) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.875) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=rbf;, score=(train=0.972, test=0.889) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=rbf;, score=(train=0.969, test=0.875) total time=   0.0s\n",
      "[CV 1/2] END C=1, epsilon=0.01, kernel=rbf;, score=(train=0.999, test=0.914) total time=   0.1s\n",
      "[CV 2/2] END C=1, epsilon=0.01, kernel=rbf;, score=(train=0.999, test=0.897) total time=   0.1s\n",
      "[CV 1/2] END C=1, epsilon=0.001, kernel=rbf;, score=(train=0.999, test=0.916) total time=   0.1s\n",
      "[CV 2/2] END C=1, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.898) total time=   0.1s\n",
      "[CV 1/2] END C=2, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.916) total time=   0.1s\n",
      "[CV 2/2] END C=2, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.898) total time=   0.1s\n",
      "[CV 1/2] END C=10, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.916) total time=   0.1s\n",
      "[CV 2/2] END C=10, epsilon=0.01, kernel=rbf;, score=(train=1.000, test=0.898) total time=   0.1s\n",
      "[CV 1/2] END C=2, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.917) total time=   0.1s\n",
      "[CV 2/2] END C=2, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.899) total time=   0.1s\n",
      "[CV 1/2] END C=10, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.917) total time=   0.1s\n",
      "[CV 2/2] END C=10, epsilon=0.001, kernel=rbf;, score=(train=1.000, test=0.899) total time=   0.1s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.978, test=0.943) total time=   0.0s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.975, test=0.941) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.978, test=0.943) total time=   0.0s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.975, test=0.941) total time=   0.0s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.978, test=0.943) total time=   0.0s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.975, test=0.941) total time=   0.0s\n",
      "[CV 1/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.978, test=0.943) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.974, test=0.941) total time=   0.0s\n",
      "[CV 1/2] END C=10, epsilon=0.001, kernel=linear;, score=(train=0.999, test=0.412) total time=  22.3s\n",
      "[CV 2/2] END C=10, epsilon=0.001, kernel=linear;, score=(train=0.996, test=0.330) total time=  14.8s\n",
      "[CV 1/2] END C=2, epsilon=0.001, kernel=linear;, score=(train=0.998, test=0.734) total time=   2.5s\n",
      "[CV 2/2] END C=2, epsilon=0.001, kernel=linear;, score=(train=0.995, test=0.752) total time=   1.6s\n",
      "[CV 1/2] END C=1, epsilon=0.001, kernel=linear;, score=(train=0.997, test=0.823) total time=   1.4s\n",
      "[CV 2/2] END C=1, epsilon=0.001, kernel=linear;, score=(train=0.994, test=0.827) total time=   0.9s\n",
      "[CV 1/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=0.993, test=0.935) total time=   0.1s\n",
      "[CV 2/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=0.989, test=0.933) total time=   0.1s\n",
      "[CV 1/2] END C=2, epsilon=0.01, kernel=linear;, score=(train=0.999, test=0.772) total time=   2.7s\n",
      "[CV 2/2] END C=2, epsilon=0.01, kernel=linear;, score=(train=0.997, test=0.773) total time=   1.4s\n",
      "[CV 1/2] END C=1, epsilon=0.01, kernel=linear;, score=(train=0.998, test=0.829) total time=   1.3s\n",
      "[CV 2/2] END C=1, epsilon=0.01, kernel=linear;, score=(train=0.996, test=0.852) total time=   0.7s\n",
      "[CV 1/2] END C=10, epsilon=0.01, kernel=linear;, score=(train=1.000, test=0.595) total time=  16.4s\n",
      "[CV 2/2] END C=10, epsilon=0.01, kernel=linear;, score=(train=0.998, test=0.418) total time=  12.8s\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.994, test=0.937) total time=   0.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.990, test=0.937) total time=   0.0s\n",
      "----------\n",
      "iter: 2\n",
      "n_candidates: 8\n",
      "n_resources: 3312\n",
      "Fitting 2 folds for each of 8 candidates, totalling 16 fits\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.974, test=0.963) total time=   1.2s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.974, test=0.965) total time=   1.2s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.975, test=0.963) total time=   1.0s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.974, test=0.965) total time=   1.1s\n",
      "[CV 1/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=0.980, test=0.962) total time=   2.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.001, kernel=linear;, score=(train=0.980, test=0.963) total time=   1.9s\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.981, test=0.963) total time=   1.7s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.981, test=0.964) total time=   1.7s\n",
      "[CV 1/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.978, test=0.960) total time=   0.3s\n",
      "[CV 2/2] END C=0.1, epsilon=0.1, kernel=linear;, score=(train=0.979, test=0.962) total time=   0.3s\n",
      "[CV 1/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.948) total time=   5.5s\n",
      "[CV 2/2] END C=2, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.951) total time=   6.3s\n",
      "[CV 1/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.946) total time=  36.1s\n",
      "[CV 2/2] END C=10, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.950) total time=  39.5s\n",
      "[CV 1/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.950) total time=   2.7s\n",
      "[CV 2/2] END C=1, epsilon=0.1, kernel=linear;, score=(train=0.977, test=0.953) total time=   2.7s\n",
      "----------\n",
      "iter: 3\n",
      "n_candidates: 3\n",
      "n_resources: 9936\n",
      "Fitting 2 folds for each of 3 candidates, totalling 6 fits\n",
      "[CV 1/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.978, test=0.970) total time=  16.0s\n",
      "[CV 2/2] END C=0.1, epsilon=0.01, kernel=linear;, score=(train=0.976, test=0.972) total time=  15.5s\n",
      "[CV 1/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.977, test=0.970) total time=  13.1s\n",
      "[CV 2/2] END C=0.01, epsilon=0.001, kernel=linear;, score=(train=0.975, test=0.972) total time=  13.2s\n",
      "[CV 1/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.977, test=0.970) total time=  11.9s\n",
      "[CV 2/2] END C=0.01, epsilon=0.01, kernel=linear;, score=(train=0.975, test=0.972) total time=  11.8s\n",
      "\n",
      "Best MSE: 0.970983\n",
      "Best parameters: {'C': 0.01, 'epsilon': 0.01, 'kernel': 'linear'}\n",
      "SVM R2 score: 0.9705957061952064\n",
      "SVM MSE: 0.006517934947230283\n"
     ]
    }
   ],
   "source": [
    "# use GridSearchCV to find the best parameters for SVM\n",
    "model = SVR()\n",
    "\n",
    "\n",
    "rng = np.random.RandomState(0)\n",
    "param_grid = {\n",
    "    'kernel': ['linear', 'poly', 'rbf'],\n",
    "    'C': [0.001,0.01,0.1, 1,2,10],\n",
    "    'epsilon': [0.001,0.01, 0.1, 1]\n",
    "}\n",
    "\n",
    "rgs = HalvingGridSearchCV(\n",
    "    estimator=model, param_grid=param_grid, random_state=rng, verbose=3, cv=2\n",
    ")\n",
    "rgs.fit(X_train, y_train)\n",
    "plot_model_hyperparams(rgs, 'SVR')\n",
    "\n",
    "print(\"\\nBest MSE: {:.6f}\".format(rgs.best_score_))\n",
    "print(\"Best parameters: {}\".format(rgs.best_params_))\n",
    "\n",
    "\n",
    "\n",
    "print(f'SVM R2 score: {rgs.score(X_test, y_test)}')\n",
    "print(f'SVM MSE: {mean_squared_error(y_test, rgs.predict(X_test))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_iterations: 3\n",
      "n_required_iterations: 3\n",
      "n_possible_iterations: 3\n",
      "min_resources_: 1103\n",
      "max_resources_: 9934\n",
      "aggressive_elimination: False\n",
      "factor: 3\n",
      "----------\n",
      "iter: 0\n",
      "n_candidates: 15\n",
      "n_resources: 1103\n",
      "Fitting 2 folds for each of 15 candidates, totalling 30 fits\n",
      "[CV 1/2] END alpha=0.0001, max_iter=100;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=100;, score=(train=-0.002, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.002, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.002, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, max_iter=100;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, max_iter=100;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, max_iter=1000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, max_iter=1000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, max_iter=10000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, max_iter=10000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, max_iter=100;, score=(train=-0.037, test=-0.040) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, max_iter=100;, score=(train=-0.036, test=-0.042) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, max_iter=1000;, score=(train=-0.037, test=-0.040) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, max_iter=1000;, score=(train=-0.036, test=-0.042) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, max_iter=10000;, score=(train=-0.037, test=-0.040) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, max_iter=10000;, score=(train=-0.036, test=-0.042) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, max_iter=100;, score=(train=-0.105, test=-0.101) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=100;, score=(train=-0.109, test=-0.112) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, max_iter=1000;, score=(train=-0.105, test=-0.101) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=1000;, score=(train=-0.109, test=-0.112) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, max_iter=10000;, score=(train=-0.105, test=-0.101) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=10000;, score=(train=-0.109, test=-0.112) total time=   0.0s\n",
      "[CV 1/2] END alpha=1, max_iter=100;, score=(train=-0.225, test=-0.239) total time=   0.0s\n",
      "[CV 2/2] END alpha=1, max_iter=100;, score=(train=-0.243, test=-0.219) total time=   0.0s\n",
      "[CV 1/2] END alpha=1, max_iter=1000;, score=(train=-0.225, test=-0.239) total time=   0.0s\n",
      "[CV 2/2] END alpha=1, max_iter=1000;, score=(train=-0.243, test=-0.219) total time=   0.0s\n",
      "[CV 1/2] END alpha=1, max_iter=10000;, score=(train=-0.225, test=-0.239) total time=   0.0s\n",
      "[CV 2/2] END alpha=1, max_iter=10000;, score=(train=-0.243, test=-0.219) total time=   0.0s\n",
      "----------\n",
      "iter: 1\n",
      "n_candidates: 5\n",
      "n_resources: 3309\n",
      "Fitting 2 folds for each of 5 candidates, totalling 10 fits\n",
      "[CV 1/2] END alpha=0.001, max_iter=1000;, score=(train=-0.011, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, max_iter=1000;, score=(train=-0.011, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, max_iter=10000;, score=(train=-0.011, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, max_iter=10000;, score=(train=-0.011, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=100;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=100;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "----------\n",
      "iter: 2\n",
      "n_candidates: 2\n",
      "n_resources: 9927\n",
      "Fitting 2 folds for each of 2 candidates, totalling 4 fits\n",
      "[CV 1/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=1000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, max_iter=10000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "\n",
      "Best MSE: 0.007035\n",
      "Best parameters: {'alpha': 0.0001, 'max_iter': 1000}\n",
      "Lasso Regression R2 score: 0.9708857228615794\n",
      "Lasso Regression MSE: 0.0065670025239887445\n"
     ]
    }
   ],
   "source": [
    "# use GridSearchCV to find the best parameters for Lasso Regression\n",
    "model = Lasso()\n",
    "\n",
    "\n",
    "rng = np.random.RandomState(0)\n",
    "param_grid = {\n",
    "    'alpha': [0.0001,0.001,0.01, 0.1,1],\n",
    "    'max_iter': [100, 1000, 10000]\n",
    "}\n",
    "\n",
    "lgs = HalvingGridSearchCV(\n",
    "    estimator=model, param_grid=param_grid, random_state=rng, verbose=3, cv=2,scoring='neg_mean_squared_error'\n",
    ")\n",
    "lgs.fit(X_train, y_train)\n",
    "\n",
    "print(\"\\nBest MSE: {:.6f}\".format(-lgs.best_score_))\n",
    "print(\"Best parameters: {}\".format(lgs.best_params_))\n",
    "\n",
    "#print the r2 results for the best parameters\n",
    "model = Lasso(\n",
    "    alpha=lgs.best_params_['alpha'],\n",
    "    max_iter=lgs.best_params_['max_iter']\n",
    ")\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "plot_model_hyperparams(lgs, 'Lasso')\n",
    "\n",
    "print(f'Lasso Regression R2 score: {model.score(X_test, y_test)}')\n",
    "print(f'Lasso Regression MSE: {mean_squared_error(y_test, model.predict(X_test))}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_iterations: 3\n",
      "n_required_iterations: 3\n",
      "n_possible_iterations: 3\n",
      "min_resources_: 1103\n",
      "max_resources_: 9934\n",
      "aggressive_elimination: False\n",
      "factor: 3\n",
      "----------\n",
      "iter: 0\n",
      "n_candidates: 15\n",
      "n_resources: 1103\n",
      "Fitting 2 folds for each of 15 candidates, totalling 30 fits\n",
      "[CV 1/2] END alpha=0.1, max_iter=100;, score=(train=-0.001, test=-0.022) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=100;, score=(train=-0.001, test=-0.023) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, max_iter=1000;, score=(train=-0.001, test=-0.022) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=1000;, score=(train=-0.001, test=-0.023) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, max_iter=10000;, score=(train=-0.001, test=-0.022) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, max_iter=10000;, score=(train=-0.001, test=-0.023) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.5, max_iter=100;, score=(train=-0.001, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.5, max_iter=100;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.5, max_iter=1000;, score=(train=-0.001, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.5, max_iter=1000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.5, max_iter=10000;, score=(train=-0.001, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.5, max_iter=10000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=1.0, max_iter=100;, score=(train=-0.002, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=1.0, max_iter=100;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=1.0, max_iter=1000;, score=(train=-0.002, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=1.0, max_iter=1000;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=1.0, max_iter=10000;, score=(train=-0.002, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=1.0, max_iter=10000;, score=(train=-0.002, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=100;, score=(train=-0.003, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=100;, score=(train=-0.003, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=1000;, score=(train=-0.003, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=1000;, score=(train=-0.003, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=10000;, score=(train=-0.003, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=10000;, score=(train=-0.003, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, max_iter=100;, score=(train=-0.007, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, max_iter=100;, score=(train=-0.007, test=-0.015) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, max_iter=1000;, score=(train=-0.007, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, max_iter=1000;, score=(train=-0.007, test=-0.015) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, max_iter=10000;, score=(train=-0.007, test=-0.013) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, max_iter=10000;, score=(train=-0.007, test=-0.015) total time=   0.0s\n",
      "----------\n",
      "iter: 1\n",
      "n_candidates: 5\n",
      "n_resources: 3309\n",
      "Fitting 2 folds for each of 5 candidates, totalling 10 fits\n",
      "[CV 1/2] END alpha=1.0, max_iter=1000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=1.0, max_iter=1000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=1.0, max_iter=10000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=1.0, max_iter=10000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=100;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=100;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=1000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=1000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=10000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=10000;, score=(train=-0.004, test=-0.008) total time=   0.0s\n",
      "----------\n",
      "iter: 2\n",
      "n_candidates: 2\n",
      "n_resources: 9927\n",
      "Fitting 2 folds for each of 2 candidates, totalling 4 fits\n",
      "[CV 1/2] END alpha=2.5, max_iter=1000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=1000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=2.5, max_iter=10000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=2.5, max_iter=10000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "\n",
      "Best MSE: 0.006789\n",
      "Best parameters: {'alpha': 2.5, 'max_iter': 1000}\n",
      "Ridge Regression R2 score: 0.9719177197405217\n",
      "Ridge Regression MSE: 0.006334225798104697\n"
     ]
    }
   ],
   "source": [
    "# use GridSearchCV to find the best parameters for Ridge Regression\n",
    "model = Ridge()\n",
    "\n",
    "\n",
    "rng = np.random.RandomState(0)\n",
    "param_grid = {\n",
    "    'alpha': [0.1,0.5,1.0,2.5,5,6,10],\n",
    "    'max_iter': [100, 1000, 10000]\n",
    "}\n",
    "\n",
    "hgs = HalvingGridSearchCV(\n",
    "    estimator=model, param_grid=param_grid, random_state=rng, verbose=3, cv=2, scoring='neg_mean_squared_error'\n",
    ")\n",
    "\n",
    "hgs.fit(X_train, y_train)\n",
    "\n",
    "print(\"\\nBest MSE: {:.6f}\".format(-hgs.best_score_))\n",
    "print(\"Best parameters: {}\".format(hgs.best_params_))\n",
    "\n",
    "#print the r2 results for the best parameters\n",
    "model = Ridge(\n",
    "    alpha=hgs.best_params_['alpha'],\n",
    "    max_iter=hgs.best_params_['max_iter']\n",
    ")\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "plot_model_hyperparams(hgs, 'Ridge')\n",
    "\n",
    "print(f'Ridge Regression R2 score: {model.score(X_test, y_test)}')\n",
    "print(f'Ridge Regression MSE: {mean_squared_error(y_test, model.predict(X_test))}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_iterations: 4\n",
      "n_required_iterations: 4\n",
      "n_possible_iterations: 4\n",
      "min_resources_: 367\n",
      "max_resources_: 9934\n",
      "aggressive_elimination: False\n",
      "factor: 3\n",
      "----------\n",
      "iter: 0\n",
      "n_candidates: 75\n",
      "n_resources: 367\n",
      "Fitting 2 folds for each of 75 candidates, totalling 150 fits\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.021) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.018) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.485e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.363e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.019) total time=   0.1s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.1, max_iter=100;, score=(train=-0.000, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.1, max_iter=100;, score=(train=-0.000, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.000, test=-0.021) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.000, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.000, test=-0.021) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.386e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 2.678e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.000, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.5, max_iter=100;, score=(train=-0.000, test=-0.023) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.5, max_iter=100;, score=(train=-0.000, test=-0.022) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.9, max_iter=100;, score=(train=-0.000, test=-0.023) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.9, max_iter=100;, score=(train=-0.000, test=-0.022) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.426e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 5.524e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 4.176e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 4.296e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=1, max_iter=100;, score=(train=-0.000, test=-0.023) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=1, max_iter=100;, score=(train=-0.000, test=-0.022) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=1, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=1, max_iter=1000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=1, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=1, max_iter=10000;, score=(train=-0.000, test=-0.024) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 4.059e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 4.457e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 4.097e-02, tolerance: 4.123e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.806e-02, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 5.886e-03, tolerance: 4.708e-03\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.000, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.002, test=-0.019) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=100;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=100;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=100;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=100;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=1000;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=1000;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=10000;, score=(train=-0.005, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=10000;, score=(train=-0.004, test=-0.018) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.002, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.002, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.002, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.002, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.002, test=-0.017) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.002, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=100;, score=(train=-0.007, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=100;, score=(train=-0.006, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=1000;, score=(train=-0.007, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=1000;, score=(train=-0.006, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=10000;, score=(train=-0.007, test=-0.018) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=10000;, score=(train=-0.006, test=-0.020) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.5, max_iter=100;, score=(train=-0.021, test=-0.033) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.5, max_iter=100;, score=(train=-0.020, test=-0.028) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.5, max_iter=1000;, score=(train=-0.021, test=-0.033) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.5, max_iter=1000;, score=(train=-0.020, test=-0.028) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.5, max_iter=10000;, score=(train=-0.021, test=-0.033) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.5, max_iter=10000;, score=(train=-0.020, test=-0.028) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.9, max_iter=100;, score=(train=-0.029, test=-0.041) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.9, max_iter=100;, score=(train=-0.032, test=-0.039) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.9, max_iter=1000;, score=(train=-0.029, test=-0.041) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.9, max_iter=1000;, score=(train=-0.032, test=-0.039) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.9, max_iter=10000;, score=(train=-0.029, test=-0.041) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.9, max_iter=10000;, score=(train=-0.032, test=-0.039) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=1, max_iter=100;, score=(train=-0.031, test=-0.043) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=1, max_iter=100;, score=(train=-0.035, test=-0.041) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=1, max_iter=1000;, score=(train=-0.031, test=-0.043) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=1, max_iter=1000;, score=(train=-0.035, test=-0.041) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=1, max_iter=10000;, score=(train=-0.031, test=-0.043) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=1, max_iter=10000;, score=(train=-0.035, test=-0.041) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.01, max_iter=100;, score=(train=-0.022, test=-0.035) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.01, max_iter=100;, score=(train=-0.021, test=-0.035) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.01, max_iter=1000;, score=(train=-0.022, test=-0.035) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.01, max_iter=1000;, score=(train=-0.021, test=-0.035) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.01, max_iter=10000;, score=(train=-0.022, test=-0.035) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.01, max_iter=10000;, score=(train=-0.021, test=-0.035) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.1, max_iter=100;, score=(train=-0.042, test=-0.052) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.1, max_iter=100;, score=(train=-0.047, test=-0.049) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.1, max_iter=1000;, score=(train=-0.042, test=-0.052) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.1, max_iter=1000;, score=(train=-0.047, test=-0.049) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.1, max_iter=10000;, score=(train=-0.042, test=-0.052) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.1, max_iter=10000;, score=(train=-0.047, test=-0.049) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.5, max_iter=100;, score=(train=-0.080, test=-0.082) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.5, max_iter=100;, score=(train=-0.098, test=-0.079) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.5, max_iter=1000;, score=(train=-0.080, test=-0.082) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.5, max_iter=1000;, score=(train=-0.098, test=-0.079) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.5, max_iter=10000;, score=(train=-0.080, test=-0.082) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.5, max_iter=10000;, score=(train=-0.098, test=-0.079) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.9, max_iter=100;, score=(train=-0.094, test=-0.096) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.9, max_iter=100;, score=(train=-0.118, test=-0.091) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.9, max_iter=1000;, score=(train=-0.094, test=-0.096) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.9, max_iter=1000;, score=(train=-0.118, test=-0.091) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=0.9, max_iter=10000;, score=(train=-0.094, test=-0.096) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=0.9, max_iter=10000;, score=(train=-0.118, test=-0.091) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=1, max_iter=100;, score=(train=-0.098, test=-0.099) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=1, max_iter=100;, score=(train=-0.121, test=-0.092) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=1, max_iter=1000;, score=(train=-0.098, test=-0.099) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=1, max_iter=1000;, score=(train=-0.121, test=-0.092) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.1, l1_ratio=1, max_iter=10000;, score=(train=-0.098, test=-0.099) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.1, l1_ratio=1, max_iter=10000;, score=(train=-0.121, test=-0.092) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.01, max_iter=100;, score=(train=-0.188, test=-0.202) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.01, max_iter=100;, score=(train=-0.214, test=-0.156) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.01, max_iter=1000;, score=(train=-0.188, test=-0.202) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.01, max_iter=1000;, score=(train=-0.214, test=-0.156) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.01, max_iter=10000;, score=(train=-0.188, test=-0.202) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.01, max_iter=10000;, score=(train=-0.214, test=-0.156) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.1, max_iter=100;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.1, max_iter=100;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.1, max_iter=1000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.1, max_iter=1000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.1, max_iter=10000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.1, max_iter=10000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.5, max_iter=100;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.5, max_iter=100;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.5, max_iter=1000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.5, max_iter=1000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.5, max_iter=10000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.5, max_iter=10000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.9, max_iter=100;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.9, max_iter=100;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.9, max_iter=1000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.9, max_iter=1000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=0.9, max_iter=10000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=0.9, max_iter=10000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=1, max_iter=100;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=1, max_iter=100;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=1, max_iter=1000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=1, max_iter=1000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "[CV 1/2] END alpha=10, l1_ratio=1, max_iter=10000;, score=(train=-0.225, test=-0.244) total time=   0.0s\n",
      "[CV 2/2] END alpha=10, l1_ratio=1, max_iter=10000;, score=(train=-0.257, test=-0.191) total time=   0.0s\n",
      "----------\n",
      "iter: 1\n",
      "n_candidates: 25\n",
      "n_resources: 1101\n",
      "Fitting 2 folds for each of 25 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 2.057e-01, tolerance: 1.275e-02\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.699e-01, tolerance: 1.343e-02\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 9.911e-02, tolerance: 1.275e-02\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "C:\\Users\\franc\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:631: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 7.720e-02, tolerance: 1.343e-02\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=100;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.1, max_iter=100;, score=(train=-0.001, test=-0.023) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.1, max_iter=100;, score=(train=-0.000, test=-0.024) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=10000;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=10000;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=1000;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=1000;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.1, max_iter=100;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.1, max_iter=100;, score=(train=-0.011, test=-0.016) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.006, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.005, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.005, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.005, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.005, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.005, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.005, test=-0.013) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.002, test=-0.012) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.0001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.0001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.000, test=-0.029) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=100;, score=(train=-0.009, test=-0.015) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=100;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=1000;, score=(train=-0.009, test=-0.015) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=1000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=1, max_iter=10000;, score=(train=-0.009, test=-0.015) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=1, max_iter=10000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=10000;, score=(train=-0.008, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=1000;, score=(train=-0.008, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.9, max_iter=100;, score=(train=-0.009, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.9, max_iter=100;, score=(train=-0.008, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=1000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=10000;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.01, max_iter=100;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.01, max_iter=100;, score=(train=-0.001, test=-0.014) total time=   0.0s\n",
      "----------\n",
      "iter: 2\n",
      "n_candidates: 9\n",
      "n_resources: 3303\n",
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.007, test=-0.010) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=100;, score=(train=-0.008, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.007, test=-0.010) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=1000;, score=(train=-0.008, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.007, test=-0.010) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.01, l1_ratio=0.01, max_iter=10000;, score=(train=-0.008, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.008, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=100;, score=(train=-0.009, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.008, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=10000;, score=(train=-0.009, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.008, test=-0.011) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.5, max_iter=1000;, score=(train=-0.009, test=-0.009) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.005, test=-0.008) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.005, test=-0.007) total time=   0.0s\n",
      "----------\n",
      "iter: 3\n",
      "n_candidates: 3\n",
      "n_resources: 9909\n",
      "Fitting 2 folds for each of 3 candidates, totalling 6 fits\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=10000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=100;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 1/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "[CV 2/2] END alpha=0.001, l1_ratio=0.1, max_iter=1000;, score=(train=-0.006, test=-0.007) total time=   0.0s\n",
      "\n",
      "Best MSE: 0.007151\n",
      "Best parameters: {'alpha': 0.001, 'l1_ratio': 0.1, 'max_iter': 10000}\n",
      "ElasticNet Regression R2 score: 0.9703232212467424\n",
      "ElasticNet Regression MSE: 0.006693880121080375\n"
     ]
    }
   ],
   "source": [
    "# use GridSearchCV to find the best parameters for ElasticNet Regression\n",
    "model = ElasticNet()\n",
    "\n",
    "\n",
    "rng = np.random.RandomState(0)\n",
    "param_grid = {\n",
    "    'alpha': [0.0001,0.001,0.01, 0.1,10],\n",
    "    'max_iter': [100, 1000, 10000],\n",
    "    'l1_ratio': [0.01, 0.1, 0.5, 0.9, 1]\n",
    "}\n",
    "\n",
    "hgs = HalvingGridSearchCV(\n",
    "    estimator=model, param_grid=param_grid, random_state=rng, verbose=3, cv=2, scoring='neg_mean_squared_error'\n",
    ")\n",
    "\n",
    "hgs.fit(X_train, y_train)\n",
    "\n",
    "print(\"\\nBest MSE: {:.6f}\".format(-hgs.best_score_))\n",
    "print(\"Best parameters: {}\".format(hgs.best_params_))\n",
    "\n",
    "#print the r2 results for the best parameters\n",
    "model = ElasticNet(\n",
    "    alpha=hgs.best_params_['alpha'],\n",
    "    max_iter=hgs.best_params_['max_iter'],\n",
    "    l1_ratio=hgs.best_params_['l1_ratio']\n",
    ")\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "plot_model_hyperparams(hgs, 'ElasticNet')\n",
    "\n",
    "print(f'ElasticNet Regression R2 score: {model.score(X_test, y_test)}')\n",
    "print(f'ElasticNet Regression MSE: {mean_squared_error(y_test, model.predict(X_test))}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}