Practical-06-2. Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Principal Components Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o_dir = os.path.join('outputs','pca')\n",
    "if os.path.isdir(o_dir) is not True:\n",
    "    print(\"Creating '{0}' directory.\".format(o_dir))\n",
    "    os.mkdir(o_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA()                               # Use all Principal Components\n",
    "pca.fit(scdf)                             # Train model on all data\n",
    "pcdf = pd.DataFrame(pca.transform(scdf))  # Transform data using model\n",
    "\n",
    "for i in range(0,21):\n",
    "    print(\"Amount of explained variance for component {0} is: {1:6.2f}%\".format(i, pca.explained_variance_ratio_[i]*100))\n",
    "\n",
    "print(\"The amount of explained variance of the SES score using each component is...\")\n",
    "sns.lineplot(x=list(range(1,len(pca.explained_variance_ratio_)+1)), y=pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=11)\n",
    "pca.fit(scdf)\n",
    "scores = pd.DataFrame(pca.transform(scdf), index=scdf.index)\n",
    "scores.to_csv(os.path.join(o_dir,'Scores.csv.gz'), compression='gzip', index=True)\n",
    "\n",
    "# Adapted from https://stackoverflow.com/questions/22984335/recovering-features-names-of-explained-variance-ratio-in-pca-with-sklearn\n",
    "i = np.identity(scdf.shape[1])  # identity matrix\n",
    "\n",
    "coef = pca.transform(i)\n",
    "\n",
    "loadings = pd.DataFrame(coef, index=scdf.columns)\n",
    "loadings.to_csv(os.path.join(o_dir,'Loadings.csv.gz'), compression='gzip', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(scores.shape)\n",
    "scores.sample(5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(loadings.shape)\n",
    "loadings.sample(5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "odf = pd.DataFrame(columns=['Variable','Component Loading','Score'])\n",
    "for i in range(0,len(loadings.index)):\n",
    "    row = loadings.iloc[i,:]\n",
    "    for c in list(loadings.columns.values):\n",
    "        d = {'Variable':loadings.index[i], 'Component Loading':c, 'Score':row[c]}\n",
    "        odf = odf.append(d, ignore_index=True)\n",
    "\n",
    "g = sns.FacetGrid(odf, col=\"Variable\", col_wrap=4, height=3, aspect=2.0, margin_titles=True, sharey=True)\n",
    "g = g.map(plt.plot, \"Component Loading\", \"Score\", marker=\".\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What Have We Done?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_style('white')\n",
    "sns.jointplot(data=scores, x=0, y=1, kind='hex', height=8, ratio=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create an Output Directory and Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o_dir = os.path.join('outputs','clusters-pca')\n",
    "if os.path.isdir(o_dir) is not True:\n",
    "    print(\"Creating '{0}' directory.\".format(o_dir))\n",
    "    os.mkdir(o_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df = pd.read_csv(os.path.join('outputs','pca','Scores.csv.gz'))\n",
    "score_df.rename(columns={'Unnamed: 0':'lsoacd'}, inplace=True)\n",
    "score_df.set_index('lsoacd', inplace=True)\n",
    "\n",
    "# Ensures that df is initialised but original scores remain accessible\n",
    "df = score_df.copy(deep=True)\n",
    "\n",
    "score_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df.sample(3, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rescale the Loaded Data\n",
    "\n",
    "We need this so that differences in the component scores don't cause the clustering algorithms to focus only on the 1st component."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "df[df.columns] = scaler.fit_transform(df[df.columns])\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sample(3, random_state=42)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "GSA2019",
   "language": "python",
   "name": "gsa2019"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}