99-get_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load '99-get_data.py'\n",
    "from urllib.request import urlopen\n",
    "import gzip\n",
    "\n",
    "import os\n",
    "import os.path\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import cartopy\n",
    "import cartopy.crs\n",
    "import cartopy.feature as cfeature\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "def get_filtered_isd(data_dir, s_date=None, f_date=None,\n",
    "                     allow_download=True):\n",
    "    fname = 'isd-history.csv'\n",
    "    target_file = os.path.join(data_dir, fname)\n",
    "\n",
    "    os.makedirs(data_dir, exist_ok=True)\n",
    "    if not os.path.exists(target_file) and allow_download:\n",
    "        url_target = 'ftp://ftp.ncdc.noaa.gov/pub/data/noaa/isd-history.csv'\n",
    "        with open(target_file, 'wb') as fout:\n",
    "            print(url_target)\n",
    "            fout.write(urlopen(url_target).read())\n",
    "\n",
    "    isd_history = pd.read_csv(target_file)\n",
    "    if s_date is not None:\n",
    "        isd_history = isd_history[isd_history['BEGIN'] < s_date]\n",
    "    if f_date is not None:\n",
    "        isd_history = isd_history[isd_history['END'] > f_date]\n",
    "\n",
    "    return isd_history\n",
    "\n",
    "\n",
    "def extract_date_time(row):\n",
    "    '''\n",
    "    '''\n",
    "    fmt_str = '%Y%m%d%H%M'\n",
    "    dt = datetime.datetime.strptime(row[15:27], fmt_str)\n",
    "    return dt, dt.year, dt.month, dt.day, dt.hour\n",
    "\n",
    "\n",
    "def extract_temperature(row):\n",
    "    t = int(row[87:92])\n",
    "    if t == 9999:\n",
    "        return (np.nan, )\n",
    "    return (t / 10,)\n",
    "\n",
    "\n",
    "def injest_file(fname):\n",
    "    with gzip.open(fname, 'rt', encoding='ascii') as f:\n",
    "        data = [extract_date_time(ln) + extract_temperature(ln)\n",
    "                for ln in f]\n",
    "\n",
    "    return pd.DataFrame(data,\n",
    "                        columns=('datetime', 'year', 'month',\n",
    "                                 'day', 'hour', 'T')).dropna()\n",
    "\n",
    "\n",
    "def get_hourly_data(data_dir, template, years, allow_download=True,\n",
    "                    urlbase='ftp://ftp.ncdc.noaa.gov/pub/data/noaa/{year}'):\n",
    "\n",
    "    data_dir_template = os.path.join(data_dir, '{year}')\n",
    "    target_template = os.path.join(data_dir_template, template)\n",
    "    url_template = '/'.join((urlbase, template))\n",
    "    data = []\n",
    "\n",
    "    for year in years:\n",
    "        os.makedirs(data_dir_template.format(year=year), exist_ok=True)\n",
    "\n",
    "        target_file = target_template.format(year=year)\n",
    "\n",
    "        if not os.path.exists(target_file) and allow_download:\n",
    "            url_target = url_template.format(year=year)\n",
    "            with open(target_file, 'wb') as fout:\n",
    "                print(url_target)\n",
    "                fout.write(urlopen(url_target).read())\n",
    "\n",
    "        data.append(injest_file(target_file))\n",
    "    data = pd.concat(data)\n",
    "    data.set_index('datetime', inplace=True)\n",
    "    return data\n",
    "\n",
    "\n",
    "class StationPicker:\n",
    "    def __init__(self, station_artist, data, data_path=None):\n",
    "        if data_path is None:\n",
    "            data_path = os.path.expanduser('~/data_cache')\n",
    "        self.data_path = data_path\n",
    "        self.event = None\n",
    "        self.data = data\n",
    "        self.station_artist = station_artist\n",
    "        self.cid = station_artist.figure.canvas.mpl_connect('pick_event',\n",
    "                                                            self._id_station)\n",
    "        self.station_templates = {}\n",
    "        self.station_rows = {}\n",
    "\n",
    "    def _id_station(self, event):\n",
    "        print('HIT')\n",
    "        if event.artist is not self.station_artist:\n",
    "            return True\n",
    "\n",
    "        N = len(event.ind)\n",
    "        if not N:\n",
    "            return True\n",
    "        for i in event.ind:\n",
    "            row = self.data.iloc[i]\n",
    "            label = row['STATION NAME']\n",
    "            tmplate = '{USAF:05d}-{WBAN:05d}-{{year}}.gz'.format(**row)\n",
    "            self.station_rows[label] = row\n",
    "            self.station_templates[label] = tmplate\n",
    "            print('{!r}: {!r}'.format(label, tmplate))\n",
    "\n",
    "    def remove(self):\n",
    "        self.station_artist.figure.canvas.mpl_disconnect(self.cid)\n",
    "        self.cid = None\n",
    "\n",
    "    def get_station_data(self, station_name, years):\n",
    "        '''Get data from NOAA\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        station_name : str\n",
    "            This has to be the name of a station you have clicked on\n",
    "            (see sp.station_templates.keys())\n",
    "        years : list\n",
    "           List of years to get data for\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        DataFrame\n",
    "            Only extracts the temperature, year, month, day, and hour\n",
    "        '''\n",
    "        return get_hourly_data(self.data_path,\n",
    "                               self.station_templates[station_name],\n",
    "                               years)\n",
    "\n",
    "\n",
    "def plot_station_locations(fig, fih, pick_radius=10):\n",
    "    fig.clf()\n",
    "    fig.add_subplot(1, 1, 1, projection=cartopy.crs.PlateCarree())\n",
    "\n",
    "    countries = cfeature.NaturalEarthFeature(\n",
    "            category='cultural',\n",
    "            name='admin_0_countries',\n",
    "            scale='50m',\n",
    "            facecolor='none',\n",
    "            edgecolor='gray')\n",
    "\n",
    "    states_provinces = cfeature.NaturalEarthFeature(\n",
    "            category='cultural',\n",
    "            name='admin_1_states_provinces_lines',\n",
    "            scale='50m',\n",
    "            facecolor='none',\n",
    "            edgecolor='gray')\n",
    "\n",
    "    land = cfeature.NaturalEarthFeature(\n",
    "            category='physical',\n",
    "            name='land',\n",
    "            scale='50m',\n",
    "            facecolor=cfeature.COLORS['land'])\n",
    "\n",
    "    lakes = cfeature.NaturalEarthFeature(\n",
    "        category='physical',\n",
    "        name='lakes',\n",
    "        scale='50m',\n",
    "        facecolor=cfeature.COLORS['water'])\n",
    "\n",
    "    ax = plt.axes(projection=cartopy.crs.PlateCarree())\n",
    "    ax.set_xlim(-80.5, -71)\n",
    "    ax.set_ylim(36, 45)\n",
    "\n",
    "    ax.add_feature(land)\n",
    "    ax.add_feature(lakes)\n",
    "    ax.add_feature(states_provinces, edgecolor='gray')\n",
    "    ax.add_feature(countries, edgecolor='gray')\n",
    "    art, = ax.plot('LON', 'LAT', 'o', data=fih, ms=5, picker=10)\n",
    "\n",
    "    sp = StationPicker(art, fih)\n",
    "\n",
    "    return ax, art, sp\n",
    "\n",
    "\n",
    "data_path = os.path.expanduser('~/data_cache')\n",
    "fih = get_filtered_isd(data_path)\n",
    "fig = plt.figure()\n",
    "ax, art, sp = plot_station_locations(fig, fih)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}