WebGraphLysimeterSWD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Read in data, apply indexs and summarise for the first time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-cba9cba58aaa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#Read in data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m AllData=pd.read_csv('K:\\Rainshelter\\StonySoilLysimeters\\DownloadedData\\StonyLysimetersCS650.dat', #specify file path for data to read in\n\u001b[0m\u001b[0;32m      3\u001b[0m                          \u001b[0mparse_dates\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;31m#tell the function to parse date columns to datetime formats\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m                          \u001b[0mdayfirst\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;31m#tell the function that the day is before the year in the data i.e format='%d/%m/%Y %H:%M'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m                          \u001b[0mskiprows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;31m#leave out rows 1, 3 and 4 which have redundant information\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "#Read in data\n",
    "AllData=pd.read_csv('K:\\Rainshelter\\StonySoilLysimeters\\DownloadedData\\StonyLysimetersCS650.dat', #specify file path for data to read in\n",
    "                         parse_dates=True, #tell the function to parse date columns to datetime formats\n",
    "                         dayfirst=True, #tell the function that the day is before the year in the data i.e format='%d/%m/%Y %H:%M'\n",
    "                         skiprows = [0,2,3], #leave out rows 1, 3 and 4 which have redundant information\n",
    "                         index_col = 0, #Use the first column, which is Date, as an index\n",
    "                         na_values = 'NAN')\n",
    "\n",
    "#Read in calibration coefficients calculated in C:\\GitHubRepos\\StonySoilLysimeters\\Calibration\\CS650Calibration.ipynb\n",
    "SensorCoefficients = pd.read_pickle('D:\\GitHubRepos\\StonySoilLysimeters\\Calibration\\CS650Calibration.pkl')\n",
    "\n",
    "#Bring in all index data\n",
    "AllDataIndex=pd.read_csv('D:\\GitHubRepos\\StonySoilLysimeters\\IndexFiles\\StonyLysimetersWater&TempIndex.csv',\n",
    "                         index_col = 0)\n",
    "\n",
    "#Bring in index for horizion types\n",
    "HorizonIndex=pd.read_csv('D:\\GitHubRepos\\StonySoilLysimeters\\IndexFiles\\HorizonIndex.csv', #specify file path for data to read in\n",
    "                         index_col = 0, #Use the first column, which is Date, as an index\n",
    "                         na_values = 'NAN')\n",
    "\n",
    "#Apply indexes to data\n",
    "AllDataTransposed = AllData.transpose()\n",
    "AllDataIndexed = pd.concat([AllDataIndex,AllDataTransposed], axis=1)\n",
    "AllDataIndexed.index.name='ColumnHeader'\n",
    "AllDataIndexed.set_index(['Measurement','Depth','Gravels','Stones','Column','Sensor', 'MUX', 'Port','Units','Summary','Block','Treatment'], \n",
    "                        append=False, inplace=True)\n",
    "AllDataIndexed.sort_index(inplace=True)\n",
    "LysoData=AllDataIndexed.transpose()\n",
    "LysoData.index = LysoData.index.to_datetime()  ## for some reason the concat function changes the data type on the date indes so need to change it back\n",
    "\n",
    "##Apply calibrations\n",
    "VWC_uncalibrated = LysoData.VolumetricWaterContent\n",
    "VWC_uncalibrated.columns = VWC_uncalibrated.columns.swaplevel('Depth','Sensor') #Make sensor the top level so index selection works properly\n",
    "VWC = pd.DataFrame(index = VWC_uncalibrated.index, columns = VWC_uncalibrated.columns)  #Set up empty dataframe with the same index and column structure as TDR data\n",
    "\n",
    "for X in range (1,97): \n",
    "    SensorRef = 'S_' + np.str(X)\n",
    "    Horizon = HorizonIndex.ix[X,'Horizon']\n",
    "    Slope = SensorCoefficients.ix[Horizon,'Slope']\n",
    "    Intercept = SensorCoefficients.ix[Horizon,'Intercept']\n",
    "    VWC[SensorRef] = VWC_uncalibrated.ix[:,SensorRef] * Slope + Intercept \n",
    "\n",
    "#Set up status files\n",
    "LastRow = VWC.index.size\n",
    "np.save('LastRow.npy',LastRow)\n",
    "VWC.to_pickle('.\\DataFrames\\LysoVWC_df')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Bring in libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly\n",
    "import plotly.plotly as py\n",
    "import plotly.tools as tls\n",
    "from plotly.graph_objs import *\n",
    "import time \n",
    "Tokens = tls.get_credentials_file()['stream_ids']\n",
    "\n",
    "#Function to read in fresh data and update status files\n",
    "def UpdataDataFrame():\n",
    "    #Bring in previous status files\n",
    "    LastRead = np.load('.\\LastRow.npy')\n",
    "    VWC = pd.read_pickle('.\\DataFrames\\LysoVWC_df')\n",
    "    \n",
    "    #Bring in fresh data\n",
    "    StartRead = LastRead + 4\n",
    "    Skips = [0,2,3] + range(4,StartRead)\n",
    "    FreshData=pd.read_csv('K:\\Rainshelter\\StonySoilLysimeters\\DownloadedData\\StonyLysimetersCS650.dat', #specify file path for data to read in\n",
    "                             parse_dates=True, #tell the function to parse date columns to datetime formats\n",
    "                             dayfirst=True, #tell the function that the day is before the year in the data i.e format='%d/%m/%Y %H:%M'\n",
    "                             skiprows = Skips, #leave out rows 1, 3 and 4 which have redundant information\n",
    "                             index_col = 0, #Use the first column, which is Date, as an index\n",
    "                             na_values = 'NAN')\n",
    "\n",
    "    #Read in calibration coefficients calculated in C:\\GitHubRepos\\StonySoilLysimeters\\Calibration\\CS650Calibration.ipynb\n",
    "    SensorCoefficients = pd.read_pickle('D:\\GitHubRepos\\StonySoilLysimeters\\Calibration\\CS650Calibration.pkl')\n",
    "\n",
    "    #Bring in all index data\n",
    "    AllDataIndex=pd.read_csv('D:\\GitHubRepos\\StonySoilLysimeters\\IndexFiles\\StonyLysimetersWater&TempIndex.csv',\n",
    "                             index_col = 0)\n",
    "\n",
    "    #Bring in index for horizion types\n",
    "    HorizonIndex=pd.read_csv('D:\\GitHubRepos\\StonySoilLysimeters\\IndexFiles\\HorizonIndex.csv', #specify file path for data to read in\n",
    "                             index_col = 0, #Use the first column, which is Date, as an index\n",
    "                             na_values = 'NAN')\n",
    "\n",
    "    #Apply indexes to data\n",
    "    FreshDataTransposed = FreshData.transpose()\n",
    "    FreshDataIndexed = pd.concat([AllDataIndex,FreshDataTransposed], axis=1)\n",
    "    FreshDataIndexed.index.name='ColumnHeader'\n",
    "    FreshDataIndexed.set_index(['Measurement','Depth','Gravels','Stones','Column','Sensor', 'MUX', 'Port','Units','Summary','Block','Treatment'], \n",
    "                            append=False, inplace=True)\n",
    "    FreshDataIndexed.sort_index(inplace=True)\n",
    "    FreshLysoData=FreshDataIndexed.transpose()\n",
    "    FreshLysoData.index = FreshLysoData.index.to_datetime()  ## for some reason the concat function changes the data type on the date indes so need to change it back\n",
    "\n",
    "    ##Apply calibrations\n",
    "    FreshVWC_uncalibrated = FreshLysoData.VolumetricWaterContent\n",
    "    FreshVWC_uncalibrated.columns = FreshVWC_uncalibrated.columns.swaplevel('Depth','Sensor') #Make sensor the top level so index selection works properly\n",
    "    FreshVWC = pd.DataFrame(index = FreshVWC_uncalibrated.index, columns = FreshVWC_uncalibrated.columns)  #Set up empty dataframe with the same index and column structure as TDR data\n",
    "\n",
    "    for X in range (1,97): \n",
    "        SensorRef = 'S_' + np.str(X)\n",
    "        Horizon = HorizonIndex.ix[X,'Horizon']\n",
    "        Slope = SensorCoefficients.ix[Horizon,'Slope']\n",
    "        Intercept = SensorCoefficients.ix[Horizon,'Intercept']\n",
    "        FreshVWC[SensorRef] = FreshVWC_uncalibrated.ix[:,SensorRef] * Slope + Intercept \n",
    "\n",
    "    #Join fresh data onto existing data\n",
    "    VWC = pd.concat([VWC,FreshVWC])\n",
    "    VWC.sort_index(inplace = True)\n",
    "    \n",
    "    #Update status files\n",
    "    LastRow = VWC.index.size\n",
    "    np.save('LastRow.npy',LastRow)\n",
    "    VWC.to_pickle('.\\DataFrames\\LysoVWC_df')\n",
    "    return VWC\n",
    "\n",
    "#Treatment class constructor\n",
    "class Treatment(object):\n",
    "    def __init__(self, Number, Depth, Stonyness):\n",
    "        self.Number=Number\n",
    "        self.Depth=Depth\n",
    "        self.Stonyness=Stonyness\n",
    "        self.streamID=Tokens[number+6] #First 6 takens used by Rainshelter graph, lysimeter uses the 2nd set\n",
    "        self.Label= Depth + ' ' + Stonyness\n",
    "        self.Trace = Scatter(x=[],y=[], name = self.Label,\n",
    "                stream = Stream(token=self.streamID)\n",
    "                )\n",
    "        self.Stream = py.Stream(self.streamID)\n",
    "\n",
    "#Create class with treatment data\n",
    "Treatments = []\n",
    "Colors = ['blue','green','red']\n",
    "Depths = ['Deep', 'Shallow']\n",
    "Stones = ['StoneFree', 'Stony', 'VeryStony']\n",
    "Lines = ['solid','dash']\n",
    "number = 0\n",
    "LinePos = 0\n",
    "for D in Depths:\n",
    "    ColorPos = 0\n",
    "    for S in Stones:\n",
    "        Treatments.append(Treatment(number, D, S))\n",
    "        Treatments[number].Trace.line = Line(color = Colors[ColorPos], dash = Lines[LinePos])\n",
    "        ColorPos +=1\n",
    "        number += 1\n",
    "    LinePos+=1\n",
    "    \n",
    "def BuildGraph():\n",
    "    GraphData = Data([Treatments[0].Trace,\n",
    "                      Treatments[1].Trace,\n",
    "                      Treatments[2].Trace,\n",
    "                      Treatments[3].Trace,\n",
    "                      Treatments[4].Trace,\n",
    "                      Treatments[5].Trace\n",
    "                     ])\n",
    "\n",
    "    layout = Layout(title='Lysimeter Soil Water Deficits',\n",
    "                    xaxis=XAxis(title='Date'),\n",
    "                    yaxis=YAxis(title='Soil Water Deficit (mm)'))\n",
    "\n",
    "    Fig = Figure(data=GraphData, layout=layout)\n",
    "\n",
    "    unique_url = py.plot(Fig, filename = 'LysimeterSWC', fileopt = 'extend', sharing = 'secret')\n",
    "    \n",
    "    \n",
    "#Function to update data traces on graph\n",
    "def Update():\n",
    "    #Update Data Frame with data that has been logged since last update\n",
    "\n",
    "    FieldData = UpdataDataFrame()\n",
    "\n",
    "    #Calculate treatment means omitting data prior to 2014-11-05 08:00:00 to avoid NaN values\n",
    "    DataMeans =  FieldData.ix['2016-09-01':].groupby(level=['Depth','Gravels','Stones'],axis=1).mean()\n",
    "    DataMeans =  DataMeans.dropna(axis=1) #For some reason it keeps non valid combinations in so need to extract with this function\n",
    "    DataMeans.where(DataMeans<0.7,np.nan,inplace=True)\n",
    "    DataMeans = DataMeans.resample('H12').last()\n",
    "\n",
    "    #Calculate the water content of the soil profile by multiplying the volumetric water content by each layers\n",
    "    #depth and summing.  The 0-15 layers are divided by 2 to average the two readings\n",
    "    ProfileWater = DataMeans.ix[:,'D1'] * 150 + \\\n",
    "                   DataMeans.ix[:,'D2'].values * 150 + \\\n",
    "                   DataMeans.ix[:,'D3'].values * 150 + \\\n",
    "                   DataMeans.ix[:,'D4'].values * 150 \n",
    "\n",
    "    FieldCapacity = ProfileWater.resample('D').max()\n",
    "    FieldCapacity = FieldCapacity.ix['2016-09-09'] # I would have though this would return a data frame with a single row but instead it returns a series with a multiindex in columns\n",
    "    SoilWaterDeficit = -(FieldCapacity - ProfileWater)  # This calculation only works because field capacity is a multi index series\n",
    "\n",
    "    X = SoilWaterDeficit.index\n",
    "    for Treatment in Treatments:\n",
    "        print('Reopening Stream[' + str(Treatment.Number) + '] and updating')\n",
    "        Treatment.Stream.open()\n",
    "        Treatment.Stream.write({'x':X,'y':SoilWaterDeficit.ix[:,(Treatment.Depth,Treatment.Stonyness)]})\n",
    "        Treatment.Stream.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "Level Depth not in index",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-c65a5f3bf5b2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#Calculate treatment means omitting data prior to 2014-11-05 08:00:00 to avoid NaN values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mDataMeans\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mFieldData\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'2016-09-01'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Depth'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'Gravels'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'Stones'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mDataMeans\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mDataMeans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#For some reason it keeps non valid combinations in so need to extract with this function\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mDataMeans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDataMeans\u001b[0m\u001b[1;33m<\u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnan\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\generic.pyc\u001b[0m in \u001b[0;36mgroupby\u001b[1;34m(self, by, axis, level, as_index, sort, group_keys, squeeze)\u001b[0m\n\u001b[0;32m   3746\u001b[0m         \u001b[0maxis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_axis_number\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3747\u001b[0m         return groupby(self, by=by, axis=axis, level=level, as_index=as_index,\n\u001b[1;32m-> 3748\u001b[1;33m                        sort=sort, group_keys=group_keys, squeeze=squeeze)\n\u001b[0m\u001b[0;32m   3749\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3750\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0masfreq\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\groupby.pyc\u001b[0m in \u001b[0;36mgroupby\u001b[1;34m(obj, by, **kwds)\u001b[0m\n\u001b[0;32m   1426\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'invalid type: %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1427\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1428\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mklass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1430\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\groupby.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, squeeze)\u001b[0m\n\u001b[0;32m    343\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mgrouper\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m             grouper, exclusions, obj = _get_grouper(obj, keys, axis=axis,\n\u001b[1;32m--> 345\u001b[1;33m                                                     level=level, sort=sort)\n\u001b[0m\u001b[0;32m    346\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\groupby.pyc\u001b[0m in \u001b[0;36m_get_grouper\u001b[1;34m(obj, key, axis, level, sort)\u001b[0m\n\u001b[0;32m   2415\u001b[0m                         \u001b[0msort\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2416\u001b[0m                         in_axis=in_axis) \\\n\u001b[1;32m-> 2417\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mGrouping\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mgpr\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2418\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2419\u001b[0m         \u001b[0mgroupings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mping\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda\\lib\\site-packages\\pandas\\core\\groupby.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, index, grouper, obj, name, level, sort, in_axis)\u001b[0m\n\u001b[0;32m   2141\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2142\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlevel\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2143\u001b[1;33m                     \u001b[1;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Level %s not in index'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2144\u001b[0m                 \u001b[0mlevel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2145\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAssertionError\u001b[0m: Level Depth not in index"
     ]
    }
   ],
   "source": [
    "FieldData = UpdataDataFrame()\n",
    "\n",
    "#Calculate treatment means omitting data prior to 2014-11-05 08:00:00 to avoid NaN values\n",
    "DataMeans =  FieldData.ix['2016-09-01':].groupby(level=['Depth','Gravels','Stones'],axis=1).mean()\n",
    "DataMeans =  DataMeans.dropna(axis=1) #For some reason it keeps non valid combinations in so need to extract with this function\n",
    "DataMeans.where(DataMeans<0.7,np.nan,inplace=True)\n",
    "DataMeans = DataMeans.resample('H12').last()\n",
    "\n",
    "#Calculate the water content of the soil profile by multiplying the volumetric water content by each layers\n",
    "#depth and summing.  The 0-15 layers are divided by 2 to average the two readings\n",
    "ProfileWater = DataMeans.ix[:,'D1'] * 150 + \\\n",
    "               DataMeans.ix[:,'D2'].values * 150 + \\\n",
    "               DataMeans.ix[:,'D3'].values * 150 + \\\n",
    "               DataMeans.ix[:,'D4'].values * 150 \n",
    "\n",
    "FieldCapacity = ProfileWater.resample('D').max()\n",
    "FieldCapacity = FieldCapacity.ix['2016-09-09'] # I would have though this would return a data frame with a single row but instead it returns a series with a multiindex in columns\n",
    "SoilWaterDeficit = -(FieldCapacity - ProfileWater)  # This calculation only works because field capacity is a multi index series\n",
    "\n",
    "SoilWaterDeficit.graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Build graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "BuildGraph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##One off update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "Update()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Continious update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lub dub[0]\n",
      "lub dub[1]\n",
      "lub dub[2]\n",
      "lub dub[3]\n",
      "lub dub[4]\n",
      "lub dub[5]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-a937a1487cd2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[0mTimeSinceUpdate\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m2.5\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m     \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m180\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m## update again in an hours time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    if time.localtime()[3] == 6:\n",
    "        try:\n",
    "            Update() #get fresh data and send to graph\n",
    "        except:\n",
    "            retrys = 1\n",
    "            print('Attempt to stream data failed')\n",
    "            while retrys < 10: ##retry connecting 10 times\n",
    "                time.sleep(300)  ##wait 5 min \n",
    "                try:\n",
    "                    Update() # have another go\n",
    "                except:\n",
    "                    print('Retry' + str(retrys) + ' failed')\n",
    "                    retrys +=1 #increment try counter and try again\n",
    "                else:\n",
    "                    retrys = 12 #set try counter so loop is exited\n",
    "    time.sleep(3600)  ## update again in an hours time"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}