Merge pull request #6 from ssj-delta-cu/wy16

Water Year 2016 update

Makefile

@@ -31,8 +31,14 @@ comma:=,
 target_ids := $(foreach t,${targets},${$t.id})
 target_ids := $(subst ${sp},|,${target_ids})
 
-water-years:=2015
+water-years:=2015 2016
 #start.2015:=2014-10-01
+
+# 2016 was a leap year
+start.2016:=$(shell date --date='2016-10-01 - 1 year' +%Y-%m-%d)
+dates.2016:=$(shell for i in `seq 0 365`; do date --date="${start.2016} + $$i days" +%Y/%m/%d; done)
+months.2016:=$(shell declare -A mo; for i in ${dates.2016}; do m=$${i%/??}; mo[$$m]=1; done; echo $${!mo[@]})
+
 start.2015:=$(shell date --date='2015-10-01 - 1 year' +%Y-%m-%d)
 dates.2015:=$(shell for i in `seq 0 364`; do date --date="${start.2015} + $$i days" +%Y/%m/%d; done)
 months.2015:=$(shell declare -A mo; for i in ${dates.2015}; do m=$${i%/??}; mo[$$m]=1; done; echo $${!mo[@]})
@@ -69,6 +75,7 @@ INFO:
 	@echo "dates: ${dates.2015}"
 	@echo "months: ${months.2015}"
 	@echo "hourly: ${items.hourly.json}"
+	@echo "start of wy16: ${start.2016}"
 
 .PHONY:csv
 
@@ -117,8 +124,17 @@ endef
 $(foreach w,${water-years},$(foreach t,${targets},$(eval $(call get_target,$w,$t))))
 
 clean-csv.cimis::
-	rm -f cimis/2015.wy/station.csv cimis/2014.wy_partial/station.csv
+	rm -f cimis/2015.wy/station.csv cimis/2014.wy_partial/station.csv cimis/2016.wy/station.csv
 
+cimis/2016.wy/station.csv:
+	  [[ -d cimis/2016.wy ]]  || mkdir -p cimis/2016.wy
+	  echo 'x,y,z,station_id,date,day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc' > $@;
+		for i in `seq 0 365`; do \
+			ymd=`date --date="2015-10-01 + $$i days" +%Y/%m/%d`; \
+			echo $$ymd; \
+			http http://cimis.casil.ucdavis.edu/cimis/$$ymd/station.csv | tail -n +1 >> $@; \
+		done	
+
 cimis/2015.wy/station.csv:
 	  [[ -d cimis/2015.wy ]]  || mkdir -p cimis/2015.wy
 	  echo 'x,y,z,station_id,date,day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc' > $@;
@@ -140,15 +156,15 @@ cimis/2014.wy/station.csv:
 rast.cimis:=ETo K Rnl Rs Rso Tdew Tn Tx U2
 
 define cimis_y
-cimis.yearly:: cimis/$1/$2.tif
+cimis.yearly:: cimis/$1.wy/$2.tif
 
 cimis/$1.wy/$2.tif:
 		gdal_merge.py -separate -o cimis/$1.wy/$2.tif  cimis/$(shell let y=$1-1; echo $$y)/1?/$2.tif cimis/$1/0?/$2.tif
 
 endef
 
 define cimis_ym
-cimis.monthly:: cimis/$1.wy/$2.tif
+cimis.monthly:: cimis/$1/$2.tif
 
 cimis/$1/$2.tif:
 	gdal_merge.py -separate -o cimis/$1/$2.tif  cimis/$1/??/$2.tif
@@ -161,7 +177,7 @@ $3:: cimis/$1/$2/$3.tif
 cimis/$1/$3.tif:: cimis/$1/$2/$3.tif
 
 cimis/$1/$2/$3.tif:
-		[[ -d $1/$2 ]] || mkdir -p $1/$2
+		[[ -d cimis/$1/$2 ]] || mkdir -p cimis/$1/$2
 		http http://cimis.casil.ucdavis.edu/cimis/$1/$2/$3.asc.gz > cimis/$1/$2/$3.asc
 		gdal_translate -a_srs EPSG:3310 -projwin -164000 68000 -108000 -44000 cimis/$1/$2/$3.asc cimis/$1/$2/$3.tif
 		rm cimis/$1/$2/$3.asc
@@ -170,4 +186,8 @@ endef
 
 $(foreach d,${dates.2015},$(foreach r,${rast.cimis},$(eval $(call cimis,$(dir $d),$(notdir $d),$r))))
 $(foreach m,${months.2015},$(foreach r,${rast.cimis},$(eval $(call cimis_ym,$m,$r))))
-$(foreach y,2015,$(foreach r,${rast.cimis},$(eval $(call cimis_y,$y,$r))))
+$(foreach y,${water-years},$(foreach r,${rast.cimis},$(eval $(call cimis_y,$y,$r))))
+
+$(foreach d,${dates.2016},$(foreach r,${rast.cimis},$(eval $(call cimis,$(dir $d),$(notdir $d),$r))))
+$(foreach m,${months.2016},$(foreach r,${rast.cimis},$(eval $(call cimis_ym,$m,$r))))
+$(foreach y,2016,$(foreach r,${rast.cimis},$(eval $(call cimis_y,$y,$r))))

README.md

@@ -1,7 +1,7 @@
 # ssj-weather
 CIMIS and Spatial CIMIS derived weather components.
 
-This project summarizes weather data for water year 2015 for the Bay Delta.  The California 2015 water year goes from 2014-10-01 to 2015-09-30.   There are a number of datasets available: daily and hourly weather for a subset of CIMIS stations; the CIMIS station data used for Spatial CIMIS; and Spatial CIMIS data clipped to the Delta.
+This project summarizes weather data for water year 2015 and 2016 for the Bay Delta.  The California 2015 water year goes from 2014-10-01 to 2015-09-30. The California 2016 water year goes from 2015-10-01 to 2016-09-30 (Note: 2016 was a leap year).  There are a number of datasets available: daily and hourly weather for a subset of CIMIS stations; the CIMIS station data used for Spatial CIMIS; and Spatial CIMIS data clipped to the Delta.
 
 Included in the project is a Makefile for recreating all these data.  If you need a slightly different dataset, that should be helpful in processing steps.  The Makefile uses [```httpie```](http://httpie.org) and [```jq```](https://stedolan.github.io/jq/) for processing.
 
@@ -17,22 +17,21 @@ Bryte | 155 | | Brentwood | 47
 Davis | 6 | | Manteca | 70
 Winters | 139 | | Tracy | 167
 Dixon | 121 | | Pleasanton | 191
-Lodi West | 166 || Modesto | 71
+Lodi West* | 166 || Modesto | 71
 
-You can use this handy [Overview Map](https://www.google.com/maps/d/edit?mid=zDpfBkZvP6Yk.kvF3ZvPShMdE&usp=sharing) to locate where these stations are located.
-
-<iframe src="https://www.google.com/maps/d/u/0/embed?mid=zDpfBkZvP6Yk.kvF3ZvPShMdE" width="640" height="480"></iframe>
+*Lodi West (station ID = 166) went offline on 3/20/2015
 
+You can use this handy [Overview Map](https://www.google.com/maps/d/edit?mid=zDpfBkZvP6Yk.kvF3ZvPShMdE&usp=sharing) to locate where these stations are located.
 
-### Daily Station ```daily_station/2015.wy/[station](.json|.csv)```
+### Daily Station ```daily_station/[year].wy/[station](.json|.csv)```
 
 The easiest way to download CIMIS station data is via the [CIMIS API](http://et.water.ca.gov).  In order to use that, you need to [register](http://et.water.ca.gov/Home/Register/) to obtain an API key.  Once registered, creating new station data is fairly straight forward.  We simply used this API to download the data.  
 
  We then used ```jq``` to do a very simple process of the json data to a CSV file.  If you need another format for this infomation, consider working directly with the JSON data, as it has more complete information then the csv files. See the Makefile for complete information, ```make csv.daily```.
 
 There is one file per station per water year.
 
-### Hourly Station ```hourly_station/2015.wy/[station](.json|.csv)```
+### Hourly Station ```hourly_station/[year].wy/[station](.json|.csv)```
 
 Similar to the ```daily_station``` data, only using calls for hourly data from the [CIMIS API](http://et.water.ca.gov).  Made with ```make csv.hourly```.  There is one file per station per water year.
 
@@ -55,9 +54,9 @@ Tx |C| Maximum Daily Temperature
 U2 |m/s| Average Daily Wind Speed at 2m
 
 
-### Station Data ```cimis/2015.wy/station.csv``` ```cimis/2015.wy/[station].csv```
+### Station Data ```cimis/[water year]/station.csv``` ```cimis/[water year]/[station].csv```
 
-It is possible that the station data used at the time of the Spatial CIMIS calculations do not match the data currently provided.  The station data used in the calculating  the Spatial CIMIS rasters is included in the ```cimis/2015.wy/``` directory.  We include the station data for each of the above Delta stations.  We also include all the data used for every day.  This data comes from the ```station.csv``` located in each directory from the [Spatial CIMIS download site](https://cimis.casil.ucdavis.edu/cimis). Made with ```make csv.cimis```.
+It is possible that the station data used at the time of the Spatial CIMIS calculations do not match the data currently provided.  The station data used in the calculating  the Spatial CIMIS rasters is included in the ```cimis/[water year]/``` directory.  We include the station data for each of the above Delta stations.  We also include all the data used for every day.  This data comes from the ```station.csv``` located in each directory from the [Spatial CIMIS download site](https://cimis.casil.ucdavis.edu/cimis). Made with ```make csv.cimis```.
 
 Note the the headers for the cimis data and the station data are slightly different, but the correspondence between them is clear.
 
@@ -70,8 +69,8 @@ For each day, we have downloaded, clipped, and reformatted the Spatial CIMIS ras
 
 For each month we combine each parameter into a multi-band tif image, one band for each day.  This is to simplify input into image processing packages, potentially.  Made with ```make cimis.monthly```.
 
-### Yearly Raster data ```cimis/2015.wy/[parm].tif```
+### Yearly Raster data ```cimis/[wateryr/[parm].tif```
 
-For each water year, we combine the months, starting from November.  To create a 365 band image of each parameter for each day.  Again this is potentially useful for image processing.
+For each water year, we combine the months, starting from October to create a 365 band image of each parameter for each day.  Again this is potentially useful for image processing.
 
 Additionally, these data are uploaded into Google Earth Engine, in the shared area for the ssj-delta-cu project. Please contact us for information on access to that data.