00-fetch_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img style='float: left' width=\"150px\" src=\"http://bostonlightswim.org/wp/wp-content/uploads/2011/08/BLS-front_4-color.jpg\">\n",
    "<br><br>\n",
    "\n",
    "## [The Boston Light Swim](http://bostonlightswim.org/)\n",
    "\n",
    "### Fetch Sea Surface Temperature time-series data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "start_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except ImportError:\n",
    "    import pickle\n",
    "\n",
    "import iris\n",
    "from datetime import datetime, timedelta\n",
    "from utilities import CF_names, start_log\n",
    "\n",
    "# Today +- 4 days\n",
    "today = datetime.utcnow()\n",
    "today = today.replace(hour=0, minute=0, second=0, microsecond=0)\n",
    "\n",
    "start = today - timedelta(days=4)\n",
    "stop = today + timedelta(days=4)\n",
    "\n",
    "# Boston harbor.\n",
    "spacing = 0.25\n",
    "bbox = [-71.05-spacing, 42.28-spacing,\n",
    "        -70.82+spacing, 42.38+spacing]\n",
    "\n",
    "# CF-names.\n",
    "sos_name = 'sea_water_temperature'\n",
    "name_list = CF_names[sos_name]\n",
    "\n",
    "# Units.\n",
    "units = iris.unit.Unit('celsius')\n",
    "\n",
    "# Logging.\n",
    "run_name = '{:%Y-%m-%d}'.format(stop)\n",
    "log = start_log(start, stop, bbox)\n",
    "\n",
    "# Config.\n",
    "fname = os.path.join(run_name, 'config.pkl')\n",
    "config = dict(start=start,\n",
    "              stop=stop,\n",
    "              bbox=bbox,\n",
    "              name_list=name_list,\n",
    "              units=units,\n",
    "              run_name=run_name)\n",
    "\n",
    "with open(fname, 'wb') as f:\n",
    "    pickle.dump(config, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the data filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from owslib import fes\n",
    "from utilities import fes_date_filter\n",
    "\n",
    "kw = dict(wildCard='*',\n",
    "          escapeChar='\\\\',\n",
    "          singleChar='?',\n",
    "          propertyname='apiso:AnyText')\n",
    "\n",
    "or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw)\n",
    "                  for val in name_list])\n",
    "\n",
    "# Exclude ROMS Averages and History files.\n",
    "not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])\n",
    "\n",
    "begin, end = fes_date_filter(start, stop)\n",
    "filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from owslib.csw import CatalogueServiceWeb\n",
    "\n",
    "endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'\n",
    "csw = CatalogueServiceWeb(endpoint, timeout=60)\n",
    "csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')\n",
    "\n",
    "fmt = '{:*^64}'.format\n",
    "log.info(fmt(' Catalog information '))\n",
    "log.info(\"URL: {}\".format(endpoint))\n",
    "log.info(\"CSW version: {}\".format(csw.version))\n",
    "log.info(\"Number of datasets available: {}\".format(len(csw.records.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from utilities import service_urls\n",
    "\n",
    "dap_urls = service_urls(csw.records, service='odp:url')\n",
    "sos_urls = service_urls(csw.records, service='sos:url')\n",
    "\n",
    "log.info(fmt(' CSW '))\n",
    "for rec, item in csw.records.items():\n",
    "    log.info('{}'.format(item.title))\n",
    "\n",
    "log.info(fmt(' SOS '))\n",
    "for url in sos_urls:\n",
    "    log.info('{}'.format(url))\n",
    "\n",
    "log.info(fmt(' DAP '))\n",
    "for url in dap_urls:\n",
    "    log.info('{}.html'.format(url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from utilities import is_station\n",
    "\n",
    "# Filter out some station endpoints.\n",
    "non_stations = []\n",
    "for url in dap_urls:\n",
    "    try:\n",
    "        if not is_station(url):\n",
    "            non_stations.append(url)\n",
    "    except RuntimeError as e:\n",
    "        log.warn(\"Could not access URL {}. {!r}\".format(url, e))\n",
    "\n",
    "dap_urls = non_stations\n",
    "\n",
    "log.info(fmt(' Filtered DAP '))\n",
    "for url in dap_urls:\n",
    "    log.info('{}.html'.format(url))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NdbcSos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyoos.collectors.ndbc.ndbc_sos import NdbcSos\n",
    "\n",
    "collector_ndbc = NdbcSos()\n",
    "\n",
    "collector_ndbc.set_bbox(bbox)\n",
    "collector_ndbc.end_time = stop\n",
    "collector_ndbc.start_time = start\n",
    "collector_ndbc.variables = [sos_name]\n",
    "\n",
    "ofrs = collector_ndbc.server.offerings\n",
    "title = collector_ndbc.server.identification.title\n",
    "log.info(fmt(' NDBC Collector offerings '))\n",
    "log.info('{}: {} offerings'.format(title, len(ofrs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.info {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".success {\n",
       "    background-color:#d9edf7;\n",
       "    border-color:#bce8f1;\n",
       "    border-left:5px solid #31708f;\n",
       "    padding:.5em;\n",
       "    color:#31708f\n",
       "}\n",
       "\n",
       ".error {\n",
       "    background-color:#f2dede;\n",
       "    border-color:#ebccd1;\n",
       "    border-left:5px solid #a94442;\n",
       "    padding:.5em;\n",
       "    color:#a94442\n",
       "}\n",
       "\n",
       ".warning {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".text-shadow {\n",
       "    text-shadow:0 1px 0 #ccc,0 2px 0 #c9c9c9,0 3px 0 #bbb,0 4px 0 #b9b9b9,0 5px 0 #aaa,0 6px 1px rgba(0,0,0,.1)\n",
       "}\n",
       "\n",
       ".datagrid table {\n",
       "    border-collapse:collapse;\n",
       "    text-align:left;\n",
       "    width:65%\n",
       "}\n",
       "\n",
       ".datagrid td {\n",
       "    border-collapse:collapse;\n",
       "    text-align:right;\n",
       "}\n",
       "\n",
       ".datagrid {\n",
       "    font:normal 12px/150% Arial,Helvetica,sans-serif;\n",
       "    background:#fff;\n",
       "    overflow:hidden;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px\n",
       "}\n",
       "\n",
       ".datagrid table td,.datagrid table th {\n",
       "    padding:3px 10px\n",
       "}\n",
       "\n",
       ".datagrid table thead th {\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069;\n",
       "    color:#FFF;\n",
       "    font-size:15px;\n",
       "    font-weight:700;\n",
       "    border-left:1px solid #0070A8\n",
       "}\n",
       "\n",
       ".datagrid table thead th:first-child {\n",
       "    border:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody td {\n",
       "    color:#00496B;\n",
       "    border-left:1px solid #E1EEF4;\n",
       "    font-size:12px;\n",
       "    font-weight:400\n",
       "}\n",
       "\n",
       ".datagrid table tbody .alt td {\n",
       "    background:#E1EEF4;\n",
       "    color:#00496B\n",
       "}\n",
       "\n",
       ".datagrid table tbody td:first-child {\n",
       "    border-left:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody tr:last-child td {\n",
       "    border-bottom:none\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    border-top:1px solid #069;\n",
       "    background:#E1EEF4\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td {\n",
       "    padding:0;\n",
       "    font-size:12px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    padding:2px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td ul {\n",
       "    margin:0;\n",
       "    padding:0;\n",
       "    list-style:none;\n",
       "    text-align:right\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li {\n",
       "    display:inline\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li a {\n",
       "    text-decoration:none;\n",
       "    display:inline-block;\n",
       "    padding:2px 8px;\n",
       "    margin:1px;\n",
       "    color:#FFF;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px;\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069\n",
       "}\n",
       "\n",
       ".datagrid table tfoot ul.active,.datagrid table tfoot ul a:hover {\n",
       "    text-decoration:none;\n",
       "    border-color:#069;\n",
       "    color:#FFF;\n",
       "    background:none;\n",
       "    background-color:#00557F\n",
       "}\n",
       "\n",
       "div.dhtmlx_window_active,div.dhx_modal_cover_dv {\n",
       "    position:fixed!important\n",
       "}\n",
       "</style><div class=\"datagrid\"><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station</th>\n",
       "      <th>sensor</th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Boston 16 Nm East Of Boston</th>\n",
       "      <td>44013</td>\n",
       "      <td>watertemp1</td>\n",
       "      <td>-70.69</td>\n",
       "      <td>42.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Buoy A01</th>\n",
       "      <td>44029</td>\n",
       "      <td>ct1</td>\n",
       "      <td>-70.57</td>\n",
       "      <td>42.52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML at 0x7f106ae72e90>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utilities import collector2table, to_html, get_ndbc_longname\n",
    "\n",
    "ndbc = collector2table(collector=collector_ndbc)\n",
    "\n",
    "names = []\n",
    "for s in ndbc['station']:\n",
    "    try:\n",
    "        name = get_ndbc_longname(s)\n",
    "    except ValueError:\n",
    "        name = s\n",
    "    names.append(name)\n",
    "\n",
    "ndbc['name'] = names\n",
    "\n",
    "ndbc.set_index('name', inplace=True)\n",
    "to_html(ndbc.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CoopsSoS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyoos.collectors.coops.coops_sos import CoopsSos\n",
    "\n",
    "collector_coops = CoopsSos()\n",
    "\n",
    "collector_coops.set_bbox(bbox)\n",
    "collector_coops.end_time = stop\n",
    "collector_coops.start_time = start\n",
    "collector_coops.variables = [sos_name]\n",
    "\n",
    "ofrs = collector_coops.server.offerings\n",
    "title = collector_coops.server.identification.title\n",
    "log.info(fmt(' Collector offerings '))\n",
    "log.info('{}: {} offerings'.format(title, len(ofrs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.info {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".success {\n",
       "    background-color:#d9edf7;\n",
       "    border-color:#bce8f1;\n",
       "    border-left:5px solid #31708f;\n",
       "    padding:.5em;\n",
       "    color:#31708f\n",
       "}\n",
       "\n",
       ".error {\n",
       "    background-color:#f2dede;\n",
       "    border-color:#ebccd1;\n",
       "    border-left:5px solid #a94442;\n",
       "    padding:.5em;\n",
       "    color:#a94442\n",
       "}\n",
       "\n",
       ".warning {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".text-shadow {\n",
       "    text-shadow:0 1px 0 #ccc,0 2px 0 #c9c9c9,0 3px 0 #bbb,0 4px 0 #b9b9b9,0 5px 0 #aaa,0 6px 1px rgba(0,0,0,.1)\n",
       "}\n",
       "\n",
       ".datagrid table {\n",
       "    border-collapse:collapse;\n",
       "    text-align:left;\n",
       "    width:65%\n",
       "}\n",
       "\n",
       ".datagrid td {\n",
       "    border-collapse:collapse;\n",
       "    text-align:right;\n",
       "}\n",
       "\n",
       ".datagrid {\n",
       "    font:normal 12px/150% Arial,Helvetica,sans-serif;\n",
       "    background:#fff;\n",
       "    overflow:hidden;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px\n",
       "}\n",
       "\n",
       ".datagrid table td,.datagrid table th {\n",
       "    padding:3px 10px\n",
       "}\n",
       "\n",
       ".datagrid table thead th {\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069;\n",
       "    color:#FFF;\n",
       "    font-size:15px;\n",
       "    font-weight:700;\n",
       "    border-left:1px solid #0070A8\n",
       "}\n",
       "\n",
       ".datagrid table thead th:first-child {\n",
       "    border:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody td {\n",
       "    color:#00496B;\n",
       "    border-left:1px solid #E1EEF4;\n",
       "    font-size:12px;\n",
       "    font-weight:400\n",
       "}\n",
       "\n",
       ".datagrid table tbody .alt td {\n",
       "    background:#E1EEF4;\n",
       "    color:#00496B\n",
       "}\n",
       "\n",
       ".datagrid table tbody td:first-child {\n",
       "    border-left:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody tr:last-child td {\n",
       "    border-bottom:none\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    border-top:1px solid #069;\n",
       "    background:#E1EEF4\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td {\n",
       "    padding:0;\n",
       "    font-size:12px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    padding:2px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td ul {\n",
       "    margin:0;\n",
       "    padding:0;\n",
       "    list-style:none;\n",
       "    text-align:right\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li {\n",
       "    display:inline\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li a {\n",
       "    text-decoration:none;\n",
       "    display:inline-block;\n",
       "    padding:2px 8px;\n",
       "    margin:1px;\n",
       "    color:#FFF;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px;\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069\n",
       "}\n",
       "\n",
       ".datagrid table tfoot ul.active,.datagrid table tfoot ul a:hover {\n",
       "    text-decoration:none;\n",
       "    border-color:#069;\n",
       "    color:#FFF;\n",
       "    background:none;\n",
       "    background-color:#00557F\n",
       "}\n",
       "\n",
       "div.dhtmlx_window_active,div.dhx_modal_cover_dv {\n",
       "    position:fixed!important\n",
       "}\n",
       "</style><div class=\"datagrid\"><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station</th>\n",
       "      <th>sensor</th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Boston, MA</th>\n",
       "      <td>8443970</td>\n",
       "      <td>E1</td>\n",
       "      <td>-71.0534</td>\n",
       "      <td>42.3548</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML at 0x7f106830a290>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utilities import get_coops_metadata\n",
    "\n",
    "coops = collector2table(collector=collector_coops)\n",
    "\n",
    "names = []\n",
    "for s in coops['station']:\n",
    "    try:\n",
    "        name = get_coops_metadata(s)[0]\n",
    "    except ValueError:\n",
    "        name = s\n",
    "    names.append(name)\n",
    "\n",
    "coops['name'] = names\n",
    "\n",
    "coops.set_index('name', inplace=True)\n",
    "to_html(coops.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join CoopsSoS and NdbcSos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.info {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".success {\n",
       "    background-color:#d9edf7;\n",
       "    border-color:#bce8f1;\n",
       "    border-left:5px solid #31708f;\n",
       "    padding:.5em;\n",
       "    color:#31708f\n",
       "}\n",
       "\n",
       ".error {\n",
       "    background-color:#f2dede;\n",
       "    border-color:#ebccd1;\n",
       "    border-left:5px solid #a94442;\n",
       "    padding:.5em;\n",
       "    color:#a94442\n",
       "}\n",
       "\n",
       ".warning {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".text-shadow {\n",
       "    text-shadow:0 1px 0 #ccc,0 2px 0 #c9c9c9,0 3px 0 #bbb,0 4px 0 #b9b9b9,0 5px 0 #aaa,0 6px 1px rgba(0,0,0,.1)\n",
       "}\n",
       "\n",
       ".datagrid table {\n",
       "    border-collapse:collapse;\n",
       "    text-align:left;\n",
       "    width:65%\n",
       "}\n",
       "\n",
       ".datagrid td {\n",
       "    border-collapse:collapse;\n",
       "    text-align:right;\n",
       "}\n",
       "\n",
       ".datagrid {\n",
       "    font:normal 12px/150% Arial,Helvetica,sans-serif;\n",
       "    background:#fff;\n",
       "    overflow:hidden;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px\n",
       "}\n",
       "\n",
       ".datagrid table td,.datagrid table th {\n",
       "    padding:3px 10px\n",
       "}\n",
       "\n",
       ".datagrid table thead th {\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069;\n",
       "    color:#FFF;\n",
       "    font-size:15px;\n",
       "    font-weight:700;\n",
       "    border-left:1px solid #0070A8\n",
       "}\n",
       "\n",
       ".datagrid table thead th:first-child {\n",
       "    border:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody td {\n",
       "    color:#00496B;\n",
       "    border-left:1px solid #E1EEF4;\n",
       "    font-size:12px;\n",
       "    font-weight:400\n",
       "}\n",
       "\n",
       ".datagrid table tbody .alt td {\n",
       "    background:#E1EEF4;\n",
       "    color:#00496B\n",
       "}\n",
       "\n",
       ".datagrid table tbody td:first-child {\n",
       "    border-left:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody tr:last-child td {\n",
       "    border-bottom:none\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    border-top:1px solid #069;\n",
       "    background:#E1EEF4\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td {\n",
       "    padding:0;\n",
       "    font-size:12px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    padding:2px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td ul {\n",
       "    margin:0;\n",
       "    padding:0;\n",
       "    list-style:none;\n",
       "    text-align:right\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li {\n",
       "    display:inline\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li a {\n",
       "    text-decoration:none;\n",
       "    display:inline-block;\n",
       "    padding:2px 8px;\n",
       "    margin:1px;\n",
       "    color:#FFF;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px;\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069\n",
       "}\n",
       "\n",
       ".datagrid table tfoot ul.active,.datagrid table tfoot ul a:hover {\n",
       "    text-decoration:none;\n",
       "    border-color:#069;\n",
       "    color:#FFF;\n",
       "    background:none;\n",
       "    background-color:#00557F\n",
       "}\n",
       "\n",
       "div.dhtmlx_window_active,div.dhx_modal_cover_dv {\n",
       "    position:fixed!important\n",
       "}\n",
       "</style><div class=\"datagrid\"><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station</th>\n",
       "      <th>sensor</th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Boston, MA</th>\n",
       "      <td>8443970</td>\n",
       "      <td>E1</td>\n",
       "      <td>-71.0534</td>\n",
       "      <td>42.3548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston 16 Nm East Of Boston</th>\n",
       "      <td>44013</td>\n",
       "      <td>watertemp1</td>\n",
       "      <td>-70.69</td>\n",
       "      <td>42.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Buoy A01</th>\n",
       "      <td>44029</td>\n",
       "      <td>ct1</td>\n",
       "      <td>-70.57</td>\n",
       "      <td>42.52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML at 0x7f106b221ed0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas import concat\n",
    "\n",
    "all_obs = concat([coops, ndbc])\n",
    "\n",
    "to_html(all_obs.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fname = '{}-all_obs.csv'.format(run_name)\n",
    "fname = os.path.join(run_name, fname)\n",
    "all_obs.to_csv(fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download the observed data series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "from owslib.ows import ExceptionReport\n",
    "from utilities import pyoos2df, save_timeseries\n",
    "\n",
    "iris.FUTURE.netcdf_promote = True\n",
    "\n",
    "log.info(fmt(' Observations '))\n",
    "outfile = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop)\n",
    "outfile = os.path.join(run_name, outfile)\n",
    "\n",
    "\n",
    "log.info(fmt(' Downloading to file {} '.format(outfile)))\n",
    "data = dict()\n",
    "col = 'sea_water_temperature (C)'\n",
    "for station in all_obs.index:\n",
    "    try:\n",
    "        idx = all_obs['station'][station]\n",
    "        df = pyoos2df(collector_ndbc, idx, df_name=station)\n",
    "        if df.empty:\n",
    "            df = pyoos2df(collector_coops, idx, df_name=station)\n",
    "        data.update({idx: df[col]})\n",
    "    except ExceptionReport as e:\n",
    "        log.warning(\"[{}] {}:\\n{}\".format(idx, station, e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uniform 1-hour time base for model/data comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import date_range\n",
    "\n",
    "index = date_range(start=start, end=stop, freq='1H')\n",
    "for k, v in data.iteritems():\n",
    "    data[k] = v.reindex(index=index, limit=1, method='nearest')\n",
    "\n",
    "obs_data = DataFrame.from_dict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.info {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".success {\n",
       "    background-color:#d9edf7;\n",
       "    border-color:#bce8f1;\n",
       "    border-left:5px solid #31708f;\n",
       "    padding:.5em;\n",
       "    color:#31708f\n",
       "}\n",
       "\n",
       ".error {\n",
       "    background-color:#f2dede;\n",
       "    border-color:#ebccd1;\n",
       "    border-left:5px solid #a94442;\n",
       "    padding:.5em;\n",
       "    color:#a94442\n",
       "}\n",
       "\n",
       ".warning {\n",
       "    background-color:#fcf8e3;\n",
       "    border-color:#faebcc;\n",
       "    border-left:5px solid #8a6d3b;\n",
       "    padding:.5em;\n",
       "    color:#8a6d3b\n",
       "}\n",
       "\n",
       ".text-shadow {\n",
       "    text-shadow:0 1px 0 #ccc,0 2px 0 #c9c9c9,0 3px 0 #bbb,0 4px 0 #b9b9b9,0 5px 0 #aaa,0 6px 1px rgba(0,0,0,.1)\n",
       "}\n",
       "\n",
       ".datagrid table {\n",
       "    border-collapse:collapse;\n",
       "    text-align:left;\n",
       "    width:65%\n",
       "}\n",
       "\n",
       ".datagrid td {\n",
       "    border-collapse:collapse;\n",
       "    text-align:right;\n",
       "}\n",
       "\n",
       ".datagrid {\n",
       "    font:normal 12px/150% Arial,Helvetica,sans-serif;\n",
       "    background:#fff;\n",
       "    overflow:hidden;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px\n",
       "}\n",
       "\n",
       ".datagrid table td,.datagrid table th {\n",
       "    padding:3px 10px\n",
       "}\n",
       "\n",
       ".datagrid table thead th {\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069;\n",
       "    color:#FFF;\n",
       "    font-size:15px;\n",
       "    font-weight:700;\n",
       "    border-left:1px solid #0070A8\n",
       "}\n",
       "\n",
       ".datagrid table thead th:first-child {\n",
       "    border:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody td {\n",
       "    color:#00496B;\n",
       "    border-left:1px solid #E1EEF4;\n",
       "    font-size:12px;\n",
       "    font-weight:400\n",
       "}\n",
       "\n",
       ".datagrid table tbody .alt td {\n",
       "    background:#E1EEF4;\n",
       "    color:#00496B\n",
       "}\n",
       "\n",
       ".datagrid table tbody td:first-child {\n",
       "    border-left:none\n",
       "}\n",
       "\n",
       ".datagrid table tbody tr:last-child td {\n",
       "    border-bottom:none\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    border-top:1px solid #069;\n",
       "    background:#E1EEF4\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td {\n",
       "    padding:0;\n",
       "    font-size:12px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td div {\n",
       "    padding:2px\n",
       "}\n",
       "\n",
       ".datagrid table tfoot td ul {\n",
       "    margin:0;\n",
       "    padding:0;\n",
       "    list-style:none;\n",
       "    text-align:right\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li {\n",
       "    display:inline\n",
       "}\n",
       "\n",
       ".datagrid table tfoot li a {\n",
       "    text-decoration:none;\n",
       "    display:inline-block;\n",
       "    padding:2px 8px;\n",
       "    margin:1px;\n",
       "    color:#FFF;\n",
       "    border:1px solid #069;\n",
       "    -webkit-border-radius:3px;\n",
       "    -moz-border-radius:3px;\n",
       "    border-radius:3px;\n",
       "    background:-webkit-gradient(linear,left top,left bottom,color-stop(0.05,#069),color-stop(1,#00557F));\n",
       "    background:-moz-linear-gradient(center top,#069 5%,#00557F 100%);\n",
       "    filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699',endColorstr='#00557F');\n",
       "    background-color:#069\n",
       "}\n",
       "\n",
       ".datagrid table tfoot ul.active,.datagrid table tfoot ul a:hover {\n",
       "    text-decoration:none;\n",
       "    border-color:#069;\n",
       "    color:#FFF;\n",
       "    background:none;\n",
       "    background-color:#00557F\n",
       "}\n",
       "\n",
       "div.dhtmlx_window_active,div.dhx_modal_cover_dv {\n",
       "    position:fixed!important\n",
       "}\n",
       "</style><div class=\"datagrid\"><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>44013</th>\n",
       "      <th>44029</th>\n",
       "      <th>8443970</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-08-09 00:00:00</th>\n",
       "      <td>19.5</td>\n",
       "      <td>18.5</td>\n",
       "      <td>17.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-08-09 01:00:00</th>\n",
       "      <td>19.5</td>\n",
       "      <td>18.4</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-08-09 02:00:00</th>\n",
       "      <td>19.5</td>\n",
       "      <td>18.2</td>\n",
       "      <td>18.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-08-09 03:00:00</th>\n",
       "      <td>19.4</td>\n",
       "      <td>17.3</td>\n",
       "      <td>18.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-08-09 04:00:00</th>\n",
       "      <td>19.4</td>\n",
       "      <td>17.6</td>\n",
       "      <td>18.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML at 0x7f1068320090>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comment = \"Several stations from http://opendap.co-ops.nos.noaa.gov\"\n",
    "kw = dict(longitude=all_obs.lon,\n",
    "          latitude=all_obs.lat,\n",
    "          station_attr=dict(cf_role=\"timeseries_id\"),\n",
    "          cube_attr=dict(featureType='timeSeries',\n",
    "                         Conventions='CF-1.6',\n",
    "                         standard_name_vocabulary='CF-1.6',\n",
    "                         cdm_data_type=\"Station\",\n",
    "                         comment=comment,\n",
    "                         url=url))\n",
    "\n",
    "save_timeseries(obs_data, outfile=outfile,\n",
    "                standard_name=sos_name, **kw)\n",
    "\n",
    "to_html(obs_data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loop discovered models and save the nearest time-series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "from iris.exceptions import (CoordinateNotFoundError, ConstraintMismatchError,\n",
    "                             MergeError)\n",
    "from utilities import (quick_load_cubes, proc_cube, is_model,\n",
    "                       get_model_name, get_surface)\n",
    "\n",
    "log.info(fmt(' Models '))\n",
    "cubes = dict()\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")  # Suppress iris warnings.\n",
    "    for k, url in enumerate(dap_urls):\n",
    "        log.info('\\n[Reading url {}/{}]: {}'.format(k+1, len(dap_urls), url))\n",
    "        try:\n",
    "            cube = quick_load_cubes(url, name_list,\n",
    "                                    callback=None, strict=True)\n",
    "            if is_model(cube):\n",
    "                cube = proc_cube(cube, bbox=bbox,\n",
    "                                 time=(start, stop), units=units)\n",
    "            else:\n",
    "                log.warning(\"[Not model data]: {}\".format(url))\n",
    "                continue\n",
    "            cube = get_surface(cube)\n",
    "            mod_name, model_full_name = get_model_name(cube, url)\n",
    "            cubes.update({mod_name: cube})\n",
    "        except (RuntimeError, ValueError,\n",
    "                ConstraintMismatchError, CoordinateNotFoundError,\n",
    "                IndexError) as e:\n",
    "            log.warning('Cannot get cube for: {}\\n{}'.format(url, e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from iris.pandas import as_series\n",
    "from utilities import (make_tree, get_nearest_water,\n",
    "                       add_station, ensure_timeseries, remove_ssh)\n",
    "\n",
    "for mod_name, cube in cubes.items():\n",
    "    fname = '{:%Y-%m-%d}-{}.nc'.format(stop, mod_name)\n",
    "    fname = os.path.join(run_name, fname)\n",
    "    log.info(fmt(' Downloading to file {} '.format(fname)))\n",
    "    try:\n",
    "        tree, lon, lat = make_tree(cube)\n",
    "    except CoordinateNotFoundError as e:\n",
    "        log.warning('Cannot make KDTree for: {}'.format(mod_name))\n",
    "        continue\n",
    "    # Get model series at observed locations.\n",
    "    raw_series = dict()\n",
    "    for station, obs in all_obs.iterrows():\n",
    "        try:\n",
    "            kw = dict(k=10, max_dist=0.08, min_var=0.01)\n",
    "            args = cube, tree, obs.lon, obs.lat\n",
    "            series, dist, idx = get_nearest_water(*args, **kw)\n",
    "        except ValueError as e:\n",
    "            status = \"No Data\"\n",
    "            log.info('[{}] {}'.format(status, obs.name))\n",
    "            continue\n",
    "        if not series:\n",
    "            status = \"Land   \"\n",
    "        else:\n",
    "            raw_series.update({obs['station']: series})\n",
    "            series = as_series(series)\n",
    "            status = \"Water  \"\n",
    "        log.info('[{}] {}'.format(status, obs.name))\n",
    "    if raw_series:  # Save cube.\n",
    "        for station, cube in raw_series.items():\n",
    "            cube = add_station(cube, station)\n",
    "            cube = remove_ssh(cube)\n",
    "        try:\n",
    "            cube = iris.cube.CubeList(raw_series.values()).merge_cube()\n",
    "        except MergeError as e:\n",
    "            log.warning(e)\n",
    "        ensure_timeseries(cube)\n",
    "        iris.save(cube, fname)\n",
    "        del cube\n",
    "    log.info('Finished processing [{}]'.format(mod_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "03:57:44 INFO: ************Saving data inside directory 2015-08-17*************\n",
      "03:57:44 INFO: *********************** Run information ************************\n",
      "03:57:44 INFO: Run date: 2015-08-13 18:57:44\n",
      "03:57:44 INFO: Download start: 2015-08-09 00:00:00\n",
      "03:57:44 INFO: Download stop: 2015-08-17 00:00:00\n",
      "03:57:44 INFO: Bounding box: -71.30, 42.03,-70.57, 42.63\n",
      "03:57:44 INFO: *********************** Software version ***********************\n",
      "03:57:44 INFO: Iris version: 1.8.1\n",
      "03:57:44 INFO: owslib version: 0.9.0\n",
      "03:57:44 INFO: pyoos version: 0.6.2\n",
      "03:57:48 INFO: ********************* Catalog information **********************\n",
      "03:57:48 INFO: URL: http://www.ngdc.noaa.gov/geoportal/csw\n",
      "03:57:48 INFO: CSW version: 2.0.2\n",
      "03:57:48 INFO: Number of datasets available: 10\n",
      "03:57:48 INFO: ***************************** CSW ******************************\n",
      "03:57:48 INFO: NECOFS Massachusetts (FVCOM) - Massachusetts Coastal - Latest Forecast\n",
      "03:57:48 INFO: NYHOPS Forecast Model Results\n",
      "03:57:48 INFO: ROMS ESPRESSO Real-Time Operational IS4DVAR Forecast System Version 2 (NEW) 2013-present FMRC History (Best)\n",
      "03:57:48 INFO: NECOFS GOM3 (FVCOM) - Northeast US - Latest Forecast\n",
      "03:57:48 INFO: COAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)\n",
      "03:57:48 INFO: HYbrid Coordinate Ocean Model (HYCOM): Global\n",
      "03:57:48 INFO: NDBC Standard Meteorological Buoy Data\n",
      "03:57:48 INFO: National Data Buoy Center SOS\n",
      "03:57:48 INFO: G1SST, 1km blended SST\n",
      "03:57:48 INFO: NOAA Coral Reef Watch Operational Twice-Weekly Near-Real-Time Global 50km Satellite Coral Bleaching Monitoring Products\n",
      "03:57:48 INFO: ***************************** SOS ******************************\n",
      "03:57:48 INFO: http://sdf.ndbc.noaa.gov/sos/server.php?service=SOS&request=GetCapabilities&acceptVersions=1.0.0\n",
      "03:57:48 INFO: ***************************** DAP ******************************\n",
      "03:57:48 INFO: http://colossus.dl.stevens-tech.edu/thredds/dodsC/latest/Complete_gcmplt.nc.html\n",
      "03:57:48 INFO: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd.html\n",
      "03:57:48 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/satellite/dhw.html\n",
      "03:57:48 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global.html\n",
      "03:57:48 INFO: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd.html\n",
      "03:57:48 INFO: http://thredds.axiomdatascience.com/thredds/dodsC/G1_SST_GLOBAL.nc.html\n",
      "03:57:48 INFO: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc.html\n",
      "03:57:48 INFO: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc.html\n",
      "03:58:16 INFO: ************************* Filtered DAP *************************\n",
      "03:58:16 INFO: http://colossus.dl.stevens-tech.edu/thredds/dodsC/latest/Complete_gcmplt.nc.html\n",
      "03:58:16 INFO: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd.html\n",
      "03:58:16 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/satellite/dhw.html\n",
      "03:58:16 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global.html\n",
      "03:58:16 INFO: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd.html\n",
      "03:58:16 INFO: http://thredds.axiomdatascience.com/thredds/dodsC/G1_SST_GLOBAL.nc.html\n",
      "03:58:16 INFO: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc.html\n",
      "03:58:16 INFO: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc.html\n",
      "03:58:19 INFO: ******************* NDBC Collector offerings *******************\n",
      "03:58:19 INFO: National Data Buoy Center SOS: 948 offerings\n",
      "03:58:30 INFO: ********************* Collector offerings **********************\n",
      "03:58:30 INFO: NOAA.NOS.CO-OPS SOS: 1048 offerings\n",
      "03:58:32 INFO: ************************* Observations *************************\n",
      "03:58:32 INFO: **** Downloading to file 2015-08-17/2015-08-17-OBS_DATA.nc *****\n",
      "03:58:34 WARNING: /home/filipe/miniconda/envs/SECOORA/lib/python2.7/site-packages/pandas/util/decorators.py:81: FutureWarning: the 'cols' keyword is deprecated, use 'subset' instead\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "\n",
      "03:58:39 WARNING: /home/filipe/miniconda/envs/SECOORA/lib/python2.7/site-packages/iris/fileformats/netcdf.py:1783: UserWarning: NetCDF default saving behaviour currently assigns the outermost dimensions to unlimited. This behaviour is to be deprecated, in favour of no automatic assignment. To switch to the new behaviour, set iris.FUTURE.netcdf_no_unlimited to True.\n",
      "  warnings.warn(msg)\n",
      "\n",
      "03:58:39 INFO: **************************** Models ****************************\n",
      "03:58:39 INFO: \n",
      "[Reading url 1/8]: http://colossus.dl.stevens-tech.edu/thredds/dodsC/latest/Complete_gcmplt.nc\n",
      "04:01:11 WARNING: Cannot get cube for: http://colossus.dl.stevens-tech.edu/thredds/dodsC/latest/Complete_gcmplt.nc\n",
      "zero-size array to reduction operation minimum which has no identity\n",
      "04:01:11 INFO: \n",
      "[Reading url 2/8]: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd\n",
      "04:01:36 INFO: \n",
      "[Reading url 3/8]: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/satellite/dhw\n",
      "04:01:38 WARNING: [Not model data]: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/satellite/dhw\n",
      "04:01:38 INFO: \n",
      "[Reading url 4/8]: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global\n",
      "04:01:42 INFO: \n",
      "[Reading url 5/8]: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd\n",
      "04:01:56 INFO: \n",
      "[Reading url 6/8]: http://thredds.axiomdatascience.com/thredds/dodsC/G1_SST_GLOBAL.nc\n",
      "04:02:35 INFO: \n",
      "[Reading url 7/8]: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc\n",
      "04:02:47 INFO: \n",
      "[Reading url 8/8]: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc\n",
      "04:02:55 INFO: *** Downloading to file 2015-08-17/2015-08-17-G1SST__SST.nc ****\n",
      "04:05:02 INFO: [Water  ] Boston, MA\n",
      "04:05:35 INFO: [Water  ] Boston 16 Nm East Of Boston\n",
      "04:06:05 INFO: [Water  ] Buoy A01\n",
      "04:06:05 INFO: Finished processing [G1SST__SST]\n",
      "04:06:05 INFO:  Downloading to file 2015-08-17/2015-08-17-NECOFS_GOM3_FVCOM.nc \n",
      "04:06:08 INFO: [Water  ] Boston, MA\n",
      "04:06:12 INFO: [Water  ] Boston 16 Nm East Of Boston\n",
      "04:06:15 INFO: [Water  ] Buoy A01\n",
      "04:06:15 INFO: Finished processing [NECOFS_GOM3_FVCOM]\n",
      "04:06:15 INFO: ****** Downloading to file 2015-08-17/2015-08-17-HYCOM.nc ******\n",
      "04:06:28 INFO: [Land   ] Boston, MA\n",
      "04:06:30 INFO: [Water  ] Boston 16 Nm East Of Boston\n",
      "04:06:33 INFO: [Water  ] Buoy A01\n",
      "04:06:33 INFO: Finished processing [HYCOM]\n",
      "04:06:33 INFO: **** Downloading to file 2015-08-17/2015-08-17-COAWST_4.nc *****\n",
      "04:07:42 INFO: [Water  ] Boston, MA\n",
      "04:07:50 INFO: [Water  ] Boston 16 Nm East Of Boston\n",
      "04:07:59 INFO: [Water  ] Buoy A01\n",
      "04:08:10 INFO: Finished processing [COAWST_4]\n",
      "04:08:10 INFO: **** Downloading to file 2015-08-17/2015-08-17-ESPRESSO.nc *****\n",
      "04:08:54 INFO: [Land   ] Boston, MA\n",
      "04:09:10 INFO: [Land   ] Boston 16 Nm East Of Boston\n",
      "04:09:10 INFO: [No Data] Buoy A01\n",
      "04:09:10 INFO: Finished processing [ESPRESSO]\n",
      "04:09:10 INFO: ** Downloading to file 2015-08-17/2015-08-17-NECOFS_FVCOM.nc ***\n",
      "04:09:14 INFO: [Water  ] Boston, MA\n",
      "04:09:17 INFO: [Water  ] Boston 16 Nm East Of Boston\n",
      "04:09:21 INFO: [Water  ] Buoy A01\n",
      "04:09:21 INFO: Finished processing [NECOFS_FVCOM]\n",
      "04:09:21 INFO: 11.67 minutes\n",
      "04:09:21 INFO: EOF\n",
      "\n"
     ]
    }
   ],
   "source": [
    "elapsed = time.time() - start_time\n",
    "log.info('{:.2f} minutes'.format(elapsed/60.))\n",
    "log.info('EOF')\n",
    "\n",
    "with open('{}/log.txt'.format(run_name)) as f:\n",
    "    print(f.read())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "name": "python",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}