Machine_learning_for_trading_problems.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning for Trading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01-02 Working with Multiple Stocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Quiz: Utility functions for reading ...\n",
    "\"\"\"Utility functions\"\"\"\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def symbol_to_path(symbol, base_dir=\"data\"):\n",
    "    \"\"\"Return CSV file path given ticker symbol.\"\"\"\n",
    "    return os.path.join(base_dir, \"{}.csv\".format(str(symbol)))\n",
    "\n",
    "\n",
    "def get_data(symbols, dates):\n",
    "    \"\"\"Read stock data (adjusted close) for given symbols from CSV files.\"\"\"\n",
    "    df = pd.DataFrame(index=dates)\n",
    "    if 'SPY' not in symbols:  # add SPY for reference, if absent\n",
    "        symbols.insert(0, 'SPY')\n",
    "\n",
    "    for symbol in symbols:\n",
    "        df_temp = pd.read_csv(symbol_to_path(symbol), index_col='Date', parse_dates=True,\n",
    "                            usecols=['Date', 'Adj Close'], na_values=['nan'])\n",
    "        df_temp = df_temp.rename(columns={'Adj Close': symbol})\n",
    "        df = df.join(df_temp)\n",
    "    \n",
    "        if symbol == 'SPY':\n",
    "            df = df.dropna(subset=['SPY'])\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def test_run():\n",
    "    # Define a date range\n",
    "    dates = pd.date_range('2010-01-22', '2010-01-26')\n",
    "\n",
    "    # Choose stock symbols to read\n",
    "    symbols = ['GOOG', 'IBM', 'GLD']\n",
    "    \n",
    "    # Get stock data\n",
    "    df = get_data(symbols, dates)\n",
    "    print df\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01-03 The Power of Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Quiz: Locate the index of max value\n",
    "\"\"\"Locate maximum value.\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_max_index(a):\n",
    "    \"\"\"Return the index of the maximum value in given 1D array.\"\"\"\n",
    "    return np.argmax(a)\n",
    "\n",
    "\n",
    "def test_run():\n",
    "    a = np.array([9, 6, 2, 3, 12, 14, 7, 10], dtype=np.int32)  # 32-bit integer array\n",
    "    print \"Array:\", a\n",
    "    \n",
    "    # Find the maximum and its index in array\n",
    "    print \"Maximum value:\", a.max()\n",
    "    print \"Index of max.:\", get_max_index(a)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### 01-03 Statistics Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calculate rolling band\n",
    "\"\"\"Bollinger Bands.\"\"\"\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def symbol_to_path(symbol, base_dir=\"data\"):\n",
    "    \"\"\"Return CSV file path given ticker symbol.\"\"\"\n",
    "    return os.path.join(base_dir, \"{}.csv\".format(str(symbol)))\n",
    "\n",
    "\n",
    "def get_data(symbols, dates):\n",
    "    \"\"\"Read stock data (adjusted close) for given symbols from CSV files.\"\"\"\n",
    "    df = pd.DataFrame(index=dates)\n",
    "    if 'SPY' not in symbols:  # add SPY for reference, if absent\n",
    "        symbols.insert(0, 'SPY')\n",
    "\n",
    "    for symbol in symbols:\n",
    "        df_temp = pd.read_csv(symbol_to_path(symbol), index_col='Date',\n",
    "                parse_dates=True, usecols=['Date', 'Adj Close'], na_values=['nan'])\n",
    "        df_temp = df_temp.rename(columns={'Adj Close': symbol})\n",
    "        df = df.join(df_temp)\n",
    "        if symbol == 'SPY':  # drop dates SPY did not trade\n",
    "            df = df.dropna(subset=[\"SPY\"])\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def plot_data(df, title=\"Stock prices\"):\n",
    "    \"\"\"Plot stock prices with a custom title and meaningful axis labels.\"\"\"\n",
    "    ax = df.plot(title=title, fontsize=12)\n",
    "    ax.set_xlabel(\"Date\")\n",
    "    ax.set_ylabel(\"Price\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def get_rolling_mean(values, window):\n",
    "    \"\"\"Return rolling mean of given values, using specified window size.\"\"\"\n",
    "    #return pd.rolling_mean(values, window=window)\n",
    "    return values.rolling(window=window,center=False).mean()\n",
    "\n",
    "\n",
    "def get_rolling_std(values, window):\n",
    "    \"\"\"Return rolling standard deviation of given values, using specified window size.\"\"\"\n",
    "    #return pd.rolling_std(values, window=window)\n",
    "    return values.rolling(window=window,center=False).std()\n",
    "\n",
    "def get_bollinger_bands(rm, rstd):\n",
    "    \"\"\"Return upper and lower Bollinger Bands.\"\"\"\n",
    "    upper_band = rm + 2 * rstd\n",
    "    lower_band = rm - 2 * rstd\n",
    "    return upper_band, lower_band\n",
    "\n",
    "\n",
    "def test_run():\n",
    "    # Read data\n",
    "    dates = pd.date_range('2012-01-01', '2012-12-31')\n",
    "    symbols = ['SPY']\n",
    "    df = get_data(symbols, dates)\n",
    "\n",
    "    # Compute Bollinger Bands\n",
    "    # 1. Compute rolling mean\n",
    "    rm_SPY = get_rolling_mean(df['SPY'], window=20)\n",
    "\n",
    "    # 2. Compute rolling standard deviation\n",
    "    rstd_SPY = get_rolling_std(df['SPY'], window=20)\n",
    "\n",
    "    # 3. Compute upper and lower bands\n",
    "    upper_band, lower_band = get_bollinger_bands(rm_SPY, rstd_SPY)\n",
    "    \n",
    "    # Plot raw SPY values, rolling mean and Bollinger Bands\n",
    "    ax = df['SPY'].plot(title=\"Bollinger Bands\", label='SPY')\n",
    "    rm_SPY.plot(label='Rolling mean', ax=ax)\n",
    "    upper_band.plot(label='upper band', ax=ax)\n",
    "    lower_band.plot(label='lower band', ax=ax)\n",
    "\n",
    "    # Add axis labels and legend\n",
    "    ax.set_xlabel(\"Date\")\n",
    "    ax.set_ylabel(\"Price\")\n",
    "    ax.legend(loc='upper left')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Quiz: \"\"\"Compute daily returns.\"\"\"\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def symbol_to_path(symbol, base_dir=\"data\"):\n",
    "    \"\"\"Return CSV file path given ticker symbol.\"\"\"\n",
    "    return os.path.join(base_dir, \"{}.csv\".format(str(symbol)))\n",
    "\n",
    "\n",
    "def get_data(symbols, dates):\n",
    "    \"\"\"Read stock data (adjusted close) for given symbols from CSV files.\"\"\"\n",
    "    df = pd.DataFrame(index=dates)\n",
    "    if 'SPY' not in symbols:  # add SPY for reference, if absent\n",
    "        symbols.insert(0, 'SPY')\n",
    "\n",
    "    for symbol in symbols:\n",
    "        df_temp = pd.read_csv(symbol_to_path(symbol), index_col='Date',\n",
    "                parse_dates=True, usecols=['Date', 'Adj Close'], na_values=['nan'])\n",
    "        df_temp = df_temp.rename(columns={'Adj Close': symbol})\n",
    "        df = df.join(df_temp)\n",
    "        if symbol == 'SPY':  # drop dates SPY did not trade\n",
    "            df = df.dropna(subset=[\"SPY\"])\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def plot_data(df, title=\"Stock prices\", xlabel=\"Date\", ylabel=\"Price\"):\n",
    "    \"\"\"Plot stock prices with a custom title and meaningful axis labels.\"\"\"\n",
    "    ax = df.plot(title=title, fontsize=12)\n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def compute_daily_returns(df):\n",
    "    \"\"\"Compute and return the daily return values.\"\"\"\n",
    "    # daily_returns[1:] = (df[1:]/df[:-1].values) - 1\n",
    "    # daily_returns.ix[0, :] = 0\n",
    "    return (df / df.shift(1) - 1).fillna(0)\n",
    "    \n",
    "\n",
    "\n",
    "def test_run():\n",
    "    # Read data\n",
    "    dates = pd.date_range('2012-07-01', '2012-07-31')  # one month only\n",
    "    symbols = ['SPY','XOM']\n",
    "    df = get_data(symbols, dates)\n",
    "    plot_data(df)\n",
    "\n",
    "    # Compute daily returns\n",
    "    daily_returns = compute_daily_returns(df)\n",
    "    plot_data(daily_returns, title=\"Daily returns\", ylabel=\"Daily returns\")\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_run()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Incomplete Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Quiz: \"\"\"Fill missing values\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "def fill_missing_values(df_data):\n",
    "    \"\"\"Fill missing values in data frame, in place.\"\"\"\n",
    "    ##########################################################\n",
    "    pass  # TODO: Your code here (DO NOT modify anything else)\n",
    "    ##########################################################\n",
    "    df_data.fillna(method='ffill', inplace=True)\n",
    "    df_data.fillna(method='bfill', inplace=True)\n",
    "    return df_data\n",
    "\n",
    "def symbol_to_path(symbol, base_dir=\"data\"):\n",
    "    \"\"\"Return CSV file path given ticker symbol.\"\"\"\n",
    "    return os.path.join(base_dir, \"{}.csv\".format(str(symbol)))\n",
    "\n",
    "\n",
    "def get_data(symbols, dates):\n",
    "    \"\"\"Read stock data (adjusted close) for given symbols from CSV files.\"\"\"\n",
    "    df_final = pd.DataFrame(index=dates)\n",
    "    if \"SPY\" not in symbols:  # add SPY for reference, if absent\n",
    "        symbols.insert(0, \"SPY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Histogram and Scatterplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you see in this lesson, the distribution for daily returns for stocks and the market look very similiar like a Gaussian.  This property persists when we look at weekly, monthly or annual return as well.  If they were really Gaussian we would say the returns were normally distributed.  In many cases in financial research we assume the returns are normally distributed.  But this can be dangerous because it ignors the kurtosis or the probability in the tails.  In the early 2000 the investment banks built bonds on mortgages.  They assumed the distribution of returns for these mortgages were normally distributed.  On that basis they were able to show that these bonds had a very low probability of default.  But they made two mistakes.  First they assumed that the return of each of these mortgages was independent and two these returns were normally distributed.  Both of these assumptions were proved to be wrong.  As massive numbers of homeowners defaulted on their mortgages.  It was these defaults that precipited the great recession of 2008. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sharpe Ratio and other Portfolio statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}