1-scikit-learn-mlflow-monitor-model-training-and-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Init and import pkg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst radius  worst texture  worst perimeter  \\\n",
       "0                 0.07871  ...         25.38          17.33           184.60   \n",
       "1                 0.05667  ...         24.99          23.41           158.80   \n",
       "2                 0.05999  ...         23.57          25.53           152.50   \n",
       "3                 0.09744  ...         14.91          26.50            98.87   \n",
       "4                 0.05883  ...         22.54          16.67           152.20   \n",
       "\n",
       "   worst area  worst smoothness  worst compactness  worst concavity  \\\n",
       "0      2019.0            0.1622             0.6656           0.7119   \n",
       "1      1956.0            0.1238             0.1866           0.2416   \n",
       "2      1709.0            0.1444             0.4245           0.4504   \n",
       "3       567.7            0.2098             0.8663           0.6869   \n",
       "4      1575.0            0.1374             0.2050           0.4000   \n",
       "\n",
       "   worst concave points  worst symmetry  worst fractal dimension  \n",
       "0                0.2654          0.4601                  0.11890  \n",
       "1                0.1860          0.2750                  0.08902  \n",
       "2                0.2430          0.3613                  0.08758  \n",
       "3                0.2575          0.6638                  0.17300  \n",
       "4                0.1625          0.2364                  0.07678  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1    357\n",
       "0    212\n",
       "Name: target, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RepeatedStratifiedKFold\n",
    "import warnings\n",
    "from pandas import  DataFrame\n",
    "import mlflow\n",
    "\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# mlflow settings\n",
    "mlflow.set_tracking_uri(\"http://localhost:5000\")\n",
    "mlflow.set_experiment('Breast Cancer Classification')\n",
    "mlflow.sklearn.autolog()\n",
    "\n",
    "# Load the data and format it as a dataframe\n",
    "cancer = load_breast_cancer(as_frame=True)\n",
    "\n",
    "# print the first 5 rows of the data\n",
    "display(cancer.data.head())\n",
    "display(cancer.target.value_counts())\n",
    "\n",
    "\n",
    "def model_training(model, train_data: DataFrame, train_target: DataFrame):\n",
    "    \"\"\"\n",
    "        input: model, train_data, train_target\n",
    "        process: model training\n",
    "        output: model\n",
    "    \"\"\"\n",
    "    model.fit(train_data, train_target)\n",
    "    return model\n",
    "\n",
    "\n",
    "def model_evaluation(model, test_data: DataFrame, predict_target: DataFrame) -> float:\n",
    "    \"\"\"\n",
    "        input: model, test_data, predict_target\n",
    "        process: model evaluation\n",
    "        output: model score\n",
    "        \n",
    "    \"\"\"\n",
    "    print(f'Model Evaluation:{ model.score(test_data,predict_target)}\\n')\n",
    "    return model.score(test_data, predict_target)\n",
    "\n",
    "\n",
    "def print_classification_report(model, test_data: DataFrame, test_target: DataFrame):\n",
    "    \"\"\"\n",
    "        input: model, test_data, test_target\n",
    "        process: print classification report\n",
    "        output: None\n",
    "    \"\"\"\n",
    "    predict_target = model.predict(test_data)\n",
    "    print(\n",
    "        f'Classification Report:\\n{classification_report(y_true=test_target, y_pred=predict_target)}\\n')\n",
    "\n",
    "\n",
    "def decide_classifier(classifier_name: str):\n",
    "    \"\"\"\n",
    "        input: classifier_name\n",
    "        process: classifier callback\n",
    "        output: classifier\n",
    "    \"\"\"\n",
    "\n",
    "    return {\n",
    "        'RandomForest': RandomForestClassifier(n_jobs=-1),\n",
    "        'SVC': SVC()\n",
    "    }[classifier_name]\n",
    "\n",
    "\n",
    "def gen_param_grid(classifier_name: str) -> dict:\n",
    "    \"\"\"\n",
    "        input: classifier_name\n",
    "        process: generate parameter grid\n",
    "        output: parameter range dictionary\n",
    "    \"\"\"\n",
    "\n",
    "    return {\n",
    "        'RandomForest': {'n_estimators': range(10, 20, 10),\n",
    "                         'max_depth': range(1, 4, 2),\n",
    "                         'min_samples_split': [2, 5, 10],\n",
    "                         'min_samples_leaf': [1, 2, 4],\n",
    "                         },\n",
    "        'SVC': {'C': [0.1, 1, 10, 100, 1000],\n",
    "                'kernel': ['linear', 'rbf', 'poly', 'sigmoid'],\n",
    "                'gamma': [1, 0.1, 0.01, 0.001]\n",
    "                }\n",
    "    }[classifier_name]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>-1.446714</td>\n",
       "      <td>-0.455622</td>\n",
       "      <td>-1.365450</td>\n",
       "      <td>-1.149113</td>\n",
       "      <td>0.728073</td>\n",
       "      <td>0.699812</td>\n",
       "      <td>2.812359</td>\n",
       "      <td>-0.133216</td>\n",
       "      <td>1.092064</td>\n",
       "      <td>2.501626</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.232959</td>\n",
       "      <td>-0.492531</td>\n",
       "      <td>-1.242799</td>\n",
       "      <td>-0.976335</td>\n",
       "      <td>0.693374</td>\n",
       "      <td>1.158250</td>\n",
       "      <td>4.696536</td>\n",
       "      <td>0.918783</td>\n",
       "      <td>2.145302</td>\n",
       "      <td>1.857798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>1.975770</td>\n",
       "      <td>1.692697</td>\n",
       "      <td>2.087782</td>\n",
       "      <td>1.864406</td>\n",
       "      <td>1.261345</td>\n",
       "      <td>3.386663</td>\n",
       "      <td>2.005784</td>\n",
       "      <td>2.594677</td>\n",
       "      <td>2.128020</td>\n",
       "      <td>1.583827</td>\n",
       "      <td>...</td>\n",
       "      <td>2.154002</td>\n",
       "      <td>1.269517</td>\n",
       "      <td>2.060522</td>\n",
       "      <td>2.122424</td>\n",
       "      <td>0.732791</td>\n",
       "      <td>3.204183</td>\n",
       "      <td>1.945179</td>\n",
       "      <td>2.672867</td>\n",
       "      <td>1.935176</td>\n",
       "      <td>2.461299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>-1.405852</td>\n",
       "      <td>-1.262405</td>\n",
       "      <td>-1.348576</td>\n",
       "      <td>-1.119560</td>\n",
       "      <td>-1.361640</td>\n",
       "      <td>-0.318692</td>\n",
       "      <td>-0.362762</td>\n",
       "      <td>-0.698896</td>\n",
       "      <td>1.931042</td>\n",
       "      <td>0.967711</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.295029</td>\n",
       "      <td>-1.048967</td>\n",
       "      <td>-1.240121</td>\n",
       "      <td>-1.001978</td>\n",
       "      <td>-1.489486</td>\n",
       "      <td>-0.549555</td>\n",
       "      <td>-0.635058</td>\n",
       "      <td>-0.969633</td>\n",
       "      <td>0.616227</td>\n",
       "      <td>0.052830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>-0.986732</td>\n",
       "      <td>1.378819</td>\n",
       "      <td>-0.986010</td>\n",
       "      <td>-0.874898</td>\n",
       "      <td>0.014912</td>\n",
       "      <td>-0.605933</td>\n",
       "      <td>-0.815473</td>\n",
       "      <td>-0.844504</td>\n",
       "      <td>0.311449</td>\n",
       "      <td>0.069740</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.831572</td>\n",
       "      <td>1.547735</td>\n",
       "      <td>-0.871399</td>\n",
       "      <td>-0.746251</td>\n",
       "      <td>0.767829</td>\n",
       "      <td>-0.727517</td>\n",
       "      <td>-0.765436</td>\n",
       "      <td>-0.810046</td>\n",
       "      <td>0.821505</td>\n",
       "      <td>-0.137079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>-1.122939</td>\n",
       "      <td>-1.025253</td>\n",
       "      <td>-1.128403</td>\n",
       "      <td>-0.974639</td>\n",
       "      <td>1.211573</td>\n",
       "      <td>-0.449342</td>\n",
       "      <td>-0.977916</td>\n",
       "      <td>-0.928260</td>\n",
       "      <td>3.397431</td>\n",
       "      <td>0.963462</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.086060</td>\n",
       "      <td>-1.338574</td>\n",
       "      <td>-1.113047</td>\n",
       "      <td>-0.899230</td>\n",
       "      <td>-0.213232</td>\n",
       "      <td>-0.988995</td>\n",
       "      <td>-1.200764</td>\n",
       "      <td>-1.351180</td>\n",
       "      <td>1.060726</td>\n",
       "      <td>-0.207395</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "68     -1.446714     -0.455622       -1.365450  -1.149113         0.728073   \n",
       "181     1.975770      1.692697        2.087782   1.864406         1.261345   \n",
       "63     -1.405852     -1.262405       -1.348576  -1.119560        -1.361640   \n",
       "248    -0.986732      1.378819       -0.986010  -0.874898         0.014912   \n",
       "60     -1.122939     -1.025253       -1.128403  -0.974639         1.211573   \n",
       "\n",
       "     mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "68           0.699812        2.812359            -0.133216       1.092064   \n",
       "181          3.386663        2.005784             2.594677       2.128020   \n",
       "63          -0.318692       -0.362762            -0.698896       1.931042   \n",
       "248         -0.605933       -0.815473            -0.844504       0.311449   \n",
       "60          -0.449342       -0.977916            -0.928260       3.397431   \n",
       "\n",
       "     mean fractal dimension  ...  worst radius  worst texture  \\\n",
       "68                 2.501626  ...     -1.232959      -0.492531   \n",
       "181                1.583827  ...      2.154002       1.269517   \n",
       "63                 0.967711  ...     -1.295029      -1.048967   \n",
       "248                0.069740  ...     -0.831572       1.547735   \n",
       "60                 0.963462  ...     -1.086060      -1.338574   \n",
       "\n",
       "     worst perimeter  worst area  worst smoothness  worst compactness  \\\n",
       "68         -1.242799   -0.976335          0.693374           1.158250   \n",
       "181         2.060522    2.122424          0.732791           3.204183   \n",
       "63         -1.240121   -1.001978         -1.489486          -0.549555   \n",
       "248        -0.871399   -0.746251          0.767829          -0.727517   \n",
       "60         -1.113047   -0.899230         -0.213232          -0.988995   \n",
       "\n",
       "     worst concavity  worst concave points  worst symmetry  \\\n",
       "68          4.696536              0.918783        2.145302   \n",
       "181         1.945179              2.672867        1.935176   \n",
       "63         -0.635058             -0.969633        0.616227   \n",
       "248        -0.765436             -0.810046        0.821505   \n",
       "60         -1.200764             -1.351180        1.060726   \n",
       "\n",
       "     worst fractal dimension  \n",
       "68                  1.857798  \n",
       "181                 2.461299  \n",
       "63                  0.052830  \n",
       "248                -0.137079  \n",
       "60                 -0.207395  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# normalize data\n",
    "nor_cancer_data = (cancer.data - cancer.data.mean()) / cancer.data.std()\n",
    "\n",
    "# split the data into training and testing\n",
    "cancer_data_train, cancer_data_test, cancer_target_train, cancer_target_test = train_test_split(\n",
    "    nor_cancer_data, cancer.target, test_size=0.2, random_state=42)\n",
    "display(cancer_data_train.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tracking One Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======== SVM Model Training ========\n",
      "\n",
      "Model Evaluation:0.9736842105263158\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.95      0.96        43\n",
      "           1       0.97      0.99      0.98        71\n",
      "\n",
      "    accuracy                           0.97       114\n",
      "   macro avg       0.97      0.97      0.97       114\n",
      "weighted avg       0.97      0.97      0.97       114\n",
      "\n",
      "\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "single_model_tags = {'model': 'SVM',\n",
    "                     'tuning': 'False'\n",
    "                     }\n",
    "\n",
    "with mlflow.start_run() as run:\n",
    "\n",
    "    mlflow.set_tags(single_model_tags)\n",
    "# single model training\n",
    "    print(f'======== SVM Model Training ========\\n')\n",
    "\n",
    "    svc = model_training(model=decide_classifier(classifier_name='SVC'), train_data=cancer_data_train, train_target=cancer_target_train)\n",
    "    svc_result = model_evaluation(model=svc, test_data=cancer_data_test, predict_target=cancer_target_test)\n",
    "    svc_report = print_classification_report(model=svc, test_data=cancer_data_test, test_target=cancer_target_test)\n",
    "\n",
    "    print('====================================================================================================')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tracking Multiple Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======== SVM Model Training ========\n",
      "\n",
      "Model Evaluation:0.9736842105263158\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.95      0.96        43\n",
      "           1       0.97      0.99      0.98        71\n",
      "\n",
      "    accuracy                           0.97       114\n",
      "   macro avg       0.97      0.97      0.97       114\n",
      "weighted avg       0.97      0.97      0.97       114\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "======== RandomForest Model Training ========\n",
      "\n",
      "Model Evaluation:0.9649122807017544\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.93      0.95        43\n",
      "           1       0.96      0.99      0.97        71\n",
      "\n",
      "    accuracy                           0.96       114\n",
      "   macro avg       0.97      0.96      0.96       114\n",
      "weighted avg       0.97      0.96      0.96       114\n",
      "\n",
      "\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "svm_tags = {'model': 'SVM',\n",
    "                     'tuning': 'False'\n",
    "                     }\n",
    "\n",
    "with mlflow.start_run() as run:\n",
    "\n",
    "    mlflow.set_tags(single_model_tags)\n",
    "# SVM model training\n",
    "    print(f'======== SVM Model Training ========\\n')\n",
    "\n",
    "    svc = model_training(model=decide_classifier(classifier_name='SVC'), train_data=cancer_data_train, train_target=cancer_target_train)\n",
    "    svc_result = model_evaluation(model=svc, test_data=cancer_data_test, predict_target=cancer_target_test)\n",
    "    svc_report = print_classification_report(model=svc, test_data=cancer_data_test, test_target=cancer_target_test)\n",
    "\n",
    "    print('====================================================================================================')\n",
    "\n",
    "rf_tags = {'model': 'RandomForest',\n",
    "                     'tuning': 'False'\n",
    "                     }\n",
    "\n",
    "with mlflow.start_run() as run:\n",
    "\n",
    "    mlflow.set_tags(rf_tags)\n",
    "# random forest model training\n",
    "    print(f'======== RandomForest Model Training ========\\n')\n",
    "\n",
    "    rf = model_training(model=decide_classifier(classifier_name='RandomForest'), train_data=cancer_data_train, train_target=cancer_target_train)\n",
    "    rf_result = model_evaluation(model=rf, test_data=cancer_data_test, predict_target=cancer_target_test)\n",
    "    rf_report = print_classification_report(model=rf, test_data=cancer_data_test, test_target=cancer_target_test)\n",
    "\n",
    "    print('====================================================================================================')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tracking Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======== SVM Model Hyperparameters tuning ========\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022/12/12 01:12:13 INFO mlflow.sklearn.utils: Logging the 5 best runs, 75 runs will be omitted.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Evaluation:0.9824561403508771\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.95      0.98        43\n",
      "           1       0.97      1.00      0.99        71\n",
      "\n",
      "    accuracy                           0.98       114\n",
      "   macro avg       0.99      0.98      0.98       114\n",
      "weighted avg       0.98      0.98      0.98       114\n",
      "\n",
      "\n",
      "After tuning, performance had been updated 0.9%\n",
      "====================================================================================================\n",
      "======== RandomForest Model Hyperparameters tuning ========\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022/12/12 01:12:32 INFO mlflow.sklearn.utils: Logging the 5 best runs, 13 runs will be omitted.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Evaluation:0.9649122807017544\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.93      0.95        43\n",
      "           1       0.96      0.99      0.97        71\n",
      "\n",
      "    accuracy                           0.96       114\n",
      "   macro avg       0.97      0.96      0.96       114\n",
      "weighted avg       0.97      0.96      0.96       114\n",
      "\n",
      "\n",
      "After tuning, performance had been updated 0.0%\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "\n",
    "svm_tuning_model_tags = {'model': 'SVM',\n",
    "                     'tuning': 'True'\n",
    "                     }\n",
    "\n",
    "rf_tuning_model_tags = {'model': 'RandomForest',\n",
    "                     'tuning': 'True'\n",
    "                     }\n",
    "\n",
    "with mlflow.start_run() as run:\n",
    "\n",
    "    mlflow.set_tags(svm_tuning_model_tags)\n",
    "\n",
    "    print(f'======== SVM Model Hyperparameters tuning ========\\n')\n",
    "\n",
    "    svc_grid_search = GridSearchCV(svc, param_grid=gen_param_grid(classifier_name='SVC'), refit=True, verbose=0, cv=cv).fit(cancer_data_train, cancer_target_train)\n",
    "    svc_best_result = model_evaluation(model=svc_grid_search.best_estimator_, test_data=cancer_data_test, predict_target=cancer_target_test)\n",
    "    svc_grid_search_report = print_classification_report(model=svc_grid_search.best_estimator_, test_data=cancer_data_test, test_target=cancer_target_test)\n",
    "\n",
    "    print(f'After tuning, performance had been updated {round((svc_best_result - svc_result) / svc_result * 100, 2)}%')\n",
    "    print('====================================================================================================')\n",
    "    \n",
    "\n",
    "with mlflow.start_run() as run:\n",
    "\n",
    "    mlflow.set_tags(rf_tuning_model_tags)\n",
    "\n",
    "    print(f'======== RandomForest Model Hyperparameters tuning ========\\n')\n",
    "    rf_grid_search = GridSearchCV(rf, param_grid=gen_param_grid(classifier_name='RandomForest'), refit=True, verbose=0, cv=cv).fit(cancer_data_train, cancer_target_train)\n",
    "    rf_best_result = model_evaluation(model=rf_grid_search.best_estimator_, test_data=cancer_data_test, predict_target=cancer_target_test)\n",
    "    rf_grid_search_report = print_classification_report(model=rf_grid_search.best_estimator_, test_data=cancer_data_test, test_target=cancer_target_test)\n",
    "\n",
    "    print(f'After tuning, performance had been updated {round((rf_best_result - rf_result) / rf_result * 100, 2)}%')\n",
    "    print('====================================================================================================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.0 ('py38')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "32414fea7d7f6e9412244063e2542c47caf1761099faa934aa69384da4785e18"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}