Lab 2 Notebook_metric learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import offsetbox\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib.ticker import NullFormatter\n",
    "\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import scipy.io\n",
    "\n",
    "import metric_learn\n",
    "from metric_learn import MMC_Supervised, MMC, NCA, LMNN\n",
    "\n",
    "import sklearn\n",
    "from sklearn import decomposition, manifold, cluster, datasets\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import Isomap, MDS\n",
    "from sklearn.cluster import KMeans, DBSCAN\n",
    "from sklearn.datasets import fetch_openml, fetch_lfw_people, load_iris\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.datasets import make_blobs, load_digits\n",
    "\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Metric Learning\n",
    "----------------\n",
    "## a. Mahalanobis Metric Learning for Clustering (MMC)\n",
    "----------------\n",
    "### MMC (max_iter = 100, convergence_threshold = 0.001)\n",
    "* minimizes the sum of squared distances between similar examples\n",
    "* enforces the sum of distances between dissimilar examples to be greater than a certain margin\n",
    "* #### Arguments: \n",
    " 1.         max_iter: maximum number of iteration\n",
    " 2.         convergence_threshold: maximum difference between 2 consecutive values of iteration to be considered convergence\n",
    "* #### Methods: (on the MMC object)\n",
    " *         fit(X, constraints): learn the MMC model\n",
    " *         X: dataset\n",
    " *         constraints: 4-tuple of arrays, (a,b,c,d): (a,b) similar, (c,d): dissimilar pairs\n",
    " *         fit_transform(X): fit the model for dataset X and transform it\n",
    " *         transform(X): apply the metric transformation\n",
    " *         get_params(): get parameters for this estimator\n",
    " *         metric(): compute the Mahalanobis matrix from the transformation matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = 40\n",
    "random_state = 170\n",
    "X, y = make_blobs(centers = 2)\n",
    "\n",
    "# First, we obtain labels (0 or 1) for this data set\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: variable \"y_pred\" must be the cluster indices\n",
    "kmeans = None\n",
    "y_pred = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualization of Kmeans clustering\n",
    "plt.scatter(X[:,0], X[:,1], c = y_pred)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obtain the labels:\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Find the indices corresponding to each label (0 or 1)\n",
    "A = np.where(kmeans.labels_ == 1)\n",
    "B = np.where(kmeans.labels_ == 0)\n",
    "\n",
    "label_1 = A[0]\n",
    "label_0 = B[0]\n",
    "\n",
    "# Now, data corresponding to indices stored in A are SIMILAR to each other\n",
    "# data corresponding to indices stored in A and B are DISSIMILAR to each other\n",
    "\n",
    "# Perform MMC transformation\n",
    "### YOUR CODE HERE\n",
    "# Hint: define a MMC object of 200 iterations\n",
    "mmc = None\n",
    "# Hint: fit it with pairs of [[X[a], X[b]], [X[c], X[d]]], each of (a,b,c,d) is a sub-array of either label_0 or label_1 \n",
    "# Hint: a,b,c,d must all be of the same size\n",
    "None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Transform the data using the MMC we just fitted\n",
    "### YOUR CODE HERE\n",
    "X_transformed = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the transformed data\n",
    "plt.scatter(X_transformed[:,0], X_transformed[:,1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Metric Learning\n",
    "----------------\n",
    "## b. Local Fisher Discreminant Analysis (LFDA)\n",
    "----------------\n",
    "### NCA(num_dims = None, max_iter = 100, tol = None)\n",
    "* #### Arguments:\n",
    " 1.           num_dims : embedidng dimensionality. If None, set to n_features at fit() time\n",
    " 2.          max_iter: maximum number of iterations done by the algorithm\n",
    " 3.           tol: convergence tolerance for the optimization\n",
    "* #### Methods:\n",
    " *          fit(X, y): fit the NCA model for the data X, scalar labels y\n",
    " *          fit_transform(X, y=None): Fit to data, then transform it\n",
    " *          get_params(): get parameters for this estimator\n",
    " *          metric(): compute the Mahalanobis matrix from the transformation matrix\n",
    " *          transform(X): apply the metric transformation for dataset X\n",
    " *          transformer(): compute the transformation matrix from the Mahalanobis matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXERCISE:\n",
    "\n",
    "# Ex.3.b.1: Visualization of NCA transformation on the data\n",
    "\n",
    "# Generate the data\n",
    "n_samples = 1500\n",
    "random_state = 170\n",
    "X, y = make_blobs(n_samples=n_samples, random_state=random_state)\n",
    "\n",
    "# Visualize the data\n",
    "plt.scatter(X[:,0], X[:,1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex.3.b.1: Visualization of NCA transformation on the data (cont)\n",
    "\n",
    "# NCA require labels\n",
    "# The visualization suggests that labels could be obtained quite accurately using Kmeans algorithm\n",
    "\n",
    "# Compute K-means clustering\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: variable \"y_pred\" must the be cluster indices\n",
    "kmeans = None\n",
    "y_pred = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Perform NCA transformation on the data\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: define a NCA object, 1000 iterations\n",
    "nca = None\n",
    "# Hint: what should we pass as arguments of the fit() function?\n",
    "None\n",
    "# Hint: use the NCA we just fitted to transform data X\n",
    "X_transformed = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the data after transformation\n",
    "plt.scatter(X_transformed[:, 0], X_transformed[:, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of NCA on Kmeans\n",
    "\n",
    "# The aim of this exercise is to see the effect of NCA on data and performance of Kmeans clustering\n",
    "\n",
    "# Generate the data\n",
    "X, y = make_blobs(random_state=170, n_samples=600, centers = 5)\n",
    "rng = np.random.RandomState(72)\n",
    "# transform the data to be stretched\n",
    "transformation = rng.normal(size=(2, 2))\n",
    "X = np.dot(X, transformation)\n",
    "\n",
    "# Visualize the data\n",
    "plt.scatter(X[:,0], X[:,1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.b.2: Effects of NCA on Kmeans (cont)\n",
    "\n",
    "# Perform Kmeans clustering on the data\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: variable \"y_pred\" must the be cluster indices\n",
    "kmeans = None\n",
    "y_pred = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the result of Kmeans clustering\n",
    "plt.scatter(X[:,0], X[:,1], c= y_pred)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.b.2: Effects of NCA on Kmeans (cont)\n",
    "\n",
    "# Kmeans clustering does not seem to perform well on this dataset\n",
    "# Let's see if NCA can improve the performance\n",
    "# We saw previously that with this kind of dataset, DBSCAN could overcome Kmeans in terms of performance\n",
    "# Let's obtain the labels, for NCA, by DBSCAN\n",
    "\n",
    "### Standardize features by removing the mean and scaling to unit variance\n",
    "X = StandardScaler().fit_transform(X)\n",
    "\n",
    "# Perform DBSCAN clustering\n",
    "### YOUR CODE HERE (Fill in the \"None\"). 2 lines of code\n",
    "# Hint: variable \"y_pred\" must be the cluster indices.\n",
    "dbscan = None\n",
    "y_pred = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Perform NCA transformation based on the labels we have just obtained\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: set 1000 iterations\n",
    "nca = None\n",
    "None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Apply the fitted NCA to transform our data\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "X_transformed = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the transformed data\n",
    "plt.scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_pred)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.b.2: Effects of NCA on Kmeans (cont)\n",
    "\n",
    "# The transformed data looks more suitable for KMeans clustering than the original data\n",
    "\n",
    "# Perform KMeans clustering on the transformed data\n",
    "### YOUR CODE HERE (Fill in the \"None\").\n",
    "# Hint: variable \"y_pred\" must be the cluster labels\n",
    "kmeans = None\n",
    "y_pred = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "plt.scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_pred)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QUESTION: Can you see that NCA has improved the performance of Kmeans?\n",
    "# ANSWER: It is very clear in the visualization."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Metric Learning\n",
    "--------------\n",
    "## c. Large Margin Nearest Neighbor (LMNN)\n",
    "--------------\n",
    "### LMNN(k=3, min_iter=50, max_iter=1000, learn_rate=1e-07, regularization=0.5, convergence_tol=0.001, use_pca=True)\n",
    "*  Learns a Mahanalobis distance metric in the kNN classificaiton setting using semidefinite programming\n",
    "* #### Arguments:\n",
    " *          k : number of neighbors to consider\n",
    " *          min_iter: minimum number of iterations\n",
    " *          max_iter: maximum number of iterations\n",
    " *          learn_rate: learning rate\n",
    " *          regularization: weighting of pull and push terms, with 0.5 meaning equal weight.\n",
    "* #### Methods:\n",
    " *           fit(X,y): fit the LMNN model for dataset X and labels y\n",
    " *          fit_transform(X, y): fit the LMNN model for dataset X and labels y, then transform C\n",
    " *          transform(X): after an LMNN object is fitted, call this function to transform dataset X\n",
    " *          get_params(): get parameters for this estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXERCISE\n",
    "# Ex 3.c.1: Basic LMNN\n",
    "import numpy as np\n",
    "from metric_learn import LMNN\n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "### Load iris data, a data with labels\n",
    "iris_data = load_iris()\n",
    "X = iris_data['data']\n",
    "Y = iris_data['target']\n",
    "\n",
    "# Perform LMNN fitting\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: we want to consider 10 neighbors, learning rate 1e-6, maximum 100 iterations\n",
    "lmnn = None\n",
    "# Hint: after this line, we want lmnn to fit on the Iris data\n",
    "None\n",
    "### END OF YOUR CODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of LMNN on Kmeans\n",
    "\n",
    "# Let's consider a dataset in which Kmeans does not work well initially\n",
    "# But after applying LMNN, Kmeans performance is improved\n",
    "\n",
    "# Data (5 clusters)\n",
    "X, y_true = make_blobs(random_state=170, n_samples=600, centers = 5)\n",
    "#X = X[:, ::-1] # flip axes for better plotting\n",
    "rng = np.random.RandomState(74)\n",
    "# transform the data to be stretched\n",
    "transformation = rng.normal(size=(2, 2))\n",
    "X_sketched = np.dot(X, transformation)\n",
    "\n",
    "# Visualize\n",
    "plt.scatter(X_sketched[:,0], X_sketched[:,1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of LMNN on Kmeans (cont)\n",
    "\n",
    "# Based on the visualization, we suspect that data can be formed into 5 clusters\n",
    "# We saw that with this kind of sketched dataset, DBSCAN appeared to perform better than Kmeans\n",
    "# Let's perform Kmeans and DBSCAN clustering on this dataset\n",
    "\n",
    "# Computer Kmeans clustering\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: variable \"y_pred_kmeans\" should be the labels of clustering\n",
    "kmeans = None\n",
    "y_pred_kmeans = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "#Visualization\n",
    "plt.scatter(X_sketched[:, 0], X_sketched[:, 1], c=y_pred_kmeans)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of LMNN on Kmeans (cont)\n",
    "\n",
    "X_sketched_dbscan = StandardScaler().fit_transform(X_sketched)\n",
    "# Computer DBSCAN clustering\n",
    "### YOUR CODE HERE (set 'eps' = 0.123)\n",
    "dbscan = None\n",
    "y_pred_dbscan = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualization\n",
    "plt.scatter(X_sketched_dbscan[:, 0], X_sketched_dbscan[:, 1], c=y_pred_dbscan)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of LMNN on Kmeans (cont)\n",
    "\n",
    "# We suspect that DBSCAN clustering is more accurate\n",
    "# Therefore, let's use y_pred_dbscan (DBSCAN cluster indices) for perform LMNN metric learning\n",
    "\n",
    "# Perform LMNN on X_sketched, using labels from DBSCAN clustering performed above\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "# Hint: set k = 3, learning rate = 1e-6, maximum 100 iterations\n",
    "lmnn = None\n",
    "None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Transform the data using the fitted Metric\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "X_transformed = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the effects of LMNN on the transformed data\n",
    "plt.scatter(X_transformed[:, 0], X_transformed[:, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex 3.c.2: Effects of LMNN on Kmeans (cont)\n",
    "\n",
    "# Perform KMeans on the newly transformed set\n",
    "### YOUR CODE HERE (Fill in the \"None\")\n",
    "kmeans = None\n",
    "y_pred_kmeans = None\n",
    "### END OF YOUR CODE\n",
    "\n",
    "# Visualize the effects of LMNN on KMeans\n",
    "plt.scatter(X_transformed[:, 0], X_transformed[:, 1], c = y_pred_kmeans)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### QUESTION: Compare the performances of Kmeans on the original and transformed data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference:\n",
    " 1. https://metric-learn.github.io/metric-learn/index.html"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}