Merge pull request #39 from ceenell/time-chart

plug timeline into pipeline

2_process.yml

@@ -20,6 +20,7 @@ targets:
       - 2_process/out/gw_daily_quantiles.csv
       - 2_process/out/gw_data_anomalies.csv
       - gw_sites_sf
+      - public/gw-conditions-time-labels.csv
 
   # Sites that use "depth below" as their gw level need to be inversed. In the 
   # current implementation, this means any site that used pcode == '72019'
@@ -48,8 +49,12 @@ targets:
 
   # Prepare data for peaks SVG
   gw_anomaly_data:
-    command: read_csv("2_process/out/gw_data_anomalies.csv")
+    command: read_csv("2_process/out/gw_data_anomalies.csv", col_types = I('cDnnc'))
+  gw_time:
+    command: generate_time(gw_anomaly_data)
+  public/gw-conditions-time-labels.csv:
+    command: generate_months(target_name, gw_time)
   gw_anomaly_data_w_paths:
     command: add_paths_to_data(gw_anomaly_data)
   gw_anomaly_data_w_colors:
-    command: add_colors_to_data(gw_anomaly_data_w_paths, scico_palette)
+    command: add_colors_to_data(gw_anomaly_data_w_paths, scico_palette, gw_time)

2_process/src/prep_data_for_visualizing.R

@@ -3,16 +3,20 @@ add_paths_to_data <- function(data_in) {
   data_in %>% mutate(path = build_path_peak(daily_quant))
 }
 
-add_colors_to_data <- function(data_in, scico_palette_nm = "roma") {
+add_colors_to_data <- function(data_in, scico_palette_nm = "roma", gw_time) {
   # Create 5 color palette
   col_palette <- rev(scico::scico(5, palette = scico_palette_nm))
 
-  data_in %>% 
-    # Add number 1:365 for day of water year for each site
-    group_by(site_no) %>% 
-    arrange(Date) %>% 
-    mutate(wyday = row_number()) %>% 
-    ungroup() %>% 
+  date_full <- data_in %>%
+    # create a row for every date x site
+    expand(Date, site_no) %>%
+    distinct()
+
+  # bind data to full date sequence so data are complete for each site even with missing data
+  # this prevents misalignment of dates and animation sequence in later steps
+  gw_time %>%
+    left_join(date_full) %>%
+    left_join(data_in) %>% 
     # Add color based on quantile category
     mutate(color = ifelse(
       quant_category == "Very high",
@@ -26,3 +30,31 @@ add_colors_to_data <- function(data_in, scico_palette_nm = "roma") {
               quant_category == "Very low",
               yes = col_palette[5], no = "black"))))))
 }
+generate_time <- function(data_in) {
+  date_start <- min(data_in$Date)
+  date_end <- max(data_in$Date)
+
+  time_df <- tibble(Date = seq.Date(from = date_start,
+           to = date_end,
+           by = "1 day")) %>%
+    mutate(day_seq = dplyr::row_number(.))
+
+  return(time_df)
+
+}
+generate_months <- function(file_out, data_in){
+  # create data to drive annotations on the timeline
+  # this includes month and year labels
+  data_in %>%
+    mutate(month = lubridate::month(Date),
+           month_label = lubridate::month(Date, label = TRUE),
+           year = lubridate::year(Date)) %>%
+    group_by(month, month_label, year) %>% 
+    # draw month labels to the first day of the month
+    filter(day_seq == min(day_seq)) %>% 
+    ungroup() %>%
+    group_by(year) %>%
+    # label years on first month they appear
+    mutate(year_label = ifelse(day_seq == min(day_seq), year, NA)) %>% 
+    write_csv(file_out)
+}

3_visualize.yml

@@ -19,37 +19,32 @@ targets:
 
   3_visualize:
     depends:
-      - 3_visualize/out/anomaly_peaks.svg
-      - src/assets/anomaly_peaks.svg
-      - public/gw_sites.csv
-      - public/date_peaks.csv
+      - src/assets/anomaly_peaks.svg 
+      - public/gw-conditions-site-coords.csv ## this is in aws as gw-conditions-sites.csv
+      - public/gw-conditions-wy20.csv ## in aws as gw-conditions-wy20.csv
+      - public/gw-conditions-daily-proportions.csv ## in aws as gw-conditions-daily-count.csv
 
-  3_visualize/out/anomaly_peaks.svg:
+  # exports that are leveraged by vue
+  # TODO: send to s3
+  src/assets/anomaly_peaks.svg:
     command: build_peaks_svg(
       target_name, 
-      data_in = gw_anomaly_data_w_colors, 
-      sites_sf = gw_sites_sf,
       svg_width = svg_width, 
       svg_height = svg_height)
 
-  src/assets/anomaly_peaks.svg:
-    command: file.copy(
-      to = target_name,
-      from = '3_visualize/out/anomaly_peaks.svg', 
-      overwrite = TRUE)
-
-  public/date_peaks.csv:
+  public/gw-conditions-wy20.csv:
     command: gwl_to_peak(
       target_name, 
       gw_anomaly_data_w_colors)
 
-  public/gw_sites.csv:
+  public/gw-conditions-site-coords.csv:
     command: get_site_coords(
       target_name,
       sites_sf = gw_sites_sf)
-      
-  public/gwl_daily_count.csv:
+
+  public/gw-conditions-daily-proportions.csv:
     command: site_prop_timeseries(target_name, 
       gw_anomaly_data_w_colors)
 
+
 

3_visualize/src/build_peaks_svg.R

@@ -1,37 +1,9 @@
 
-build_peaks_svg <- function(out_file, data_in, sites_sf, svg_width, svg_height) {
+build_peaks_svg <- function(out_file, svg_width, svg_height) {
 
   svg_root <- init_svg(viewbox_dims = c(0, 0, svg_width=svg_width, svg_height=svg_height))
 
   add_background_map(svg_root, svg_width = svg_width, outline_states = FALSE)
 
-  # Add spark line within group per site
-
-  # TODO: THIS ONLY WORKS FOR A SINGLE DAY RIGHT NOW. I CHOSE OCT 31, 2019
-  # TODO: Blanket removing any path with an NA in it. There is a smarter way 
-  # to do this but will come back to that.
-  data_in <- data_in %>% 
-    filter(Date == as.Date("2019-10-31")) %>% 
-    filter(!grepl("NA", path))
-
-  sites <- sites_sf %>% pull(site_no) %>% unique
-  for(s in sites) {
-    site_coords_svg <- sites_sf %>%
-      filter(site_no == s) %>%
-      convert_coords_to_svg(svg_width = svg_width, view_bbox = st_bbox(generate_usa_map_data()))
-
-    gw_data_s <- data_in %>% filter(site_no == s)
-
-    # Spark lines centered at GW location
-    svg_root %>%
-      add_grp(grp_nm = s, trans_x = site_coords_svg$x,
-              trans_y = site_coords_svg$y) %>%
-      xml_add_child("path",
-                    class = sprintf('gwl_%s', s),
-                    style = sprintf("stroke: none; fill: %s; fill-opacity: 50%%", gw_data_s$color),
-                    d = gw_data_s$path)
-
-  }
-
   xml2::write_xml(svg_root, file = out_file)
 }

3_visualize/src/svg_utils_general.R

@@ -13,3 +13,4 @@ add_grp <- function(svg_root, grp_nm, trans_x, trans_y) {
   xml_add_child(svg_root, 'g', id = grp_nm, 
                 transform = sprintf("translate(%s %s) scale(0.35, 0.35)", trans_x, trans_y))
 }
+

3_visualize/src/svg_utils_mapping.R

@@ -5,14 +5,19 @@
 add_background_map <- function(svg, svg_width, outline_states) {
   map_data <- generate_usa_map_data(outline_states = outline_states)
 
-  bkgrd_grp <- xml_add_child(svg, 'g', id = "bkgrd-map-grp")
+  bkgrd_grp <- xml_add_child(svg, 'g', 
+                             id = "bkgrd-map-grp", 
+                             class='map-bkgrd', 
+                             style="stroke:white;stroke-width:0.2;fill:white")
   purrr::map(map_data$ID, function(polygon_id, map_data, svg_width) {
     d <- map_data %>% 
       filter(ID == polygon_id) %>% 
       convert_coords_to_svg(view_bbox = st_bbox(map_data), svg_width) %>% 
       build_path(connect = TRUE)
-    # "#e8d9c5" tan (what about slate grey #9fabb7)
-    xml_add_child(bkgrd_grp, 'path', d = d, class='map-bkgrd', style="stroke:#9fabb7;stroke-width:0.5;fill:none")
+    xml_add_child(bkgrd_grp, 'path', 
+                  d = d, 
+                  class='map-bkgrd', 
+                  style="stroke:white;stroke-width:0.2;fill:white")
   }, map_data, svg_width)
 
 }

3_visualize/src/svg_utils_vue.R

@@ -1,13 +1,13 @@
 gwl_to_peak <- function(file_out, gw_anomaly_data_w_colors){
   # Create timeseries data to draw peak animation with D3
-  gw_anomaly_data_w_colors %>%
+  gw_anomaly_data_w_colors %>% 
     mutate(quant = str_replace(quant_category, " ", ""),
            path_y = round(50-daily_quant, digits = 0)) %>%
-    filter(!is.na(quant)) %>%
+    #filter(!is.na(quant)) %>%
     mutate(site_no = paste0('gwl_', site_no)) %>%
-    select(site_no, wyday, daily_quant, path_y) %>%
-    dcast(wyday~site_no, value.var = 'path_y') %>%
-    arrange(wyday) %>%
+    select(site_no, day_seq, daily_quant, path_y) %>%
+    dcast(day_seq~site_no, value.var = 'path_y') %>%
+    arrange(day_seq) %>%
     write_csv(file_out)
 }
 
@@ -20,25 +20,26 @@ get_site_coords <- function(file_out, sites_sf){
 }
 
 site_prop_timeseries <- function(file_out, gw_anomaly_data_w_colors){
-  gw <- gw_anomaly_data_w_colors %>% 
-    filter(!is.na(quant_category)) %>% 
-    group_by(Date) %>%
-    filter(wyday == max(wyday))
-
-  ## write json with timeseries of sites in each category 
-  gw %>% 
-    group_by(Date, wyday, quant_category) %>%
+
+  ## write timeseries % of sites in each category 
+  gw_anomaly_data_w_colors %>% 
+    filter(!is.na(quant_category)) %>% # filtering out dates with no category 
+    group_by(Date, day_seq, quant_category) %>%
     summarize(n_sites = length(unique(site_no))) %>% 
-    left_join(gw  %>% 
-                group_by(Date, wyday) %>%
+    # join with the total number of sites WITH DATA for each day
+    left_join(gw_anomaly_data_w_colors  %>%  
+                filter(!is.na(quant_category))%>% 
+                group_by(Date, day_seq) %>%
                 summarize(n_sites_total = length(unique(site_no)))) %>%
     mutate(perc = n_sites/n_sites_total) %>%
     ungroup() %>%
     mutate(cat = gsub(" ", "", quant_category),
            perc = round(perc, 3)) %>%
     select(-quant_category) %>%
-    reshape2::dcast(Date+wyday+n_sites_total~cat, value.var = "perc") %>%
-    arrange(wyday) %>%
+    reshape2::dcast(Date+day_seq+n_sites_total~cat, value.var = "perc") %>%
+    arrange(day_seq) %>%
     write_csv(file_out)
 
-}
+}
+
+

public/index.html

@@ -1,6 +1,15 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
+    <!-- Global site tag (gtag.js) - Google Analytics -->
+    <script async src="https://www.googletagmanager.com/gtag/js?id=G-MK065DF5XP"></script>
+    <script>
+      window.dataLayer = window.dataLayer || [];
+      function gtag(){dataLayer.push(arguments);}
+      gtag('js', new Date());
+
+      gtag('config', 'G-MK065DF5XP');
+    </script>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
@@ -11,18 +20,18 @@
           <%= VUE_APP_TITLE %>
       </title>
       <meta name="title" content="U.S. Groundwater Conditions">
-      <meta name="description" content="">
+      <meta name="description" content="The low down on flow down low">
       <!-- Open Graph / Facebook -->
       <meta property="og:type" content="website">
       <meta property="og:url" content="https://labs.waterdata.usgs.gov/visualizations/gw-conditions/index.html#/">
       <meta property="og:title" content="U.S. Groundwater Conditions">
-      <meta property="og:description" content="">
+      <meta property="og:description" content="The low down on flow down low">
       <meta property="og:image" content="https://labs.waterdata.usgs.gov/visualizations/gw-conditions/SnowToFlowMetacard.jpg">
       <!-- Twitter -->
       <meta property="twitter:card" content="summary_large_image">
       <meta property="twitter:url" content="https://labs.waterdata.usgs.gov/visualizations/gw-conditions/index.html#/">
       <meta property="twitter:title" content="U.S. Groundwater Conditions">
-      <meta property="twitter:description" content="">
+      <meta property="twitter:description" content="The low down on flow down low">
       <meta property="twitter:image" content="https://labs.waterdata.usgs.gov/visualizations/gw-conditions/social-tag-cards-1-1.jpg">
     <link rel="icon" href="<%= BASE_URL %>favicon.ico">
 <script type='application/ld+json'>
@@ -33,7 +42,7 @@
         "datePublished": "FILL THIS OUT",
         "contributor": [
           { "@type": "Person",
-            "name": "Colleen Nell",
+            "name": "Cee Nell",
             "email": "[email protected]",
             "affiliation": { "@type": "Organization", "legalName": "U.S. Geological Survey" }
           },
@@ -46,7 +55,7 @@
       },
       "audience": { "@context": "http://schema.org", "@type": "Audience", "audienceType": "General public"},
       "publisher": { "@context": "http://schema.org", "@type": "Organization", "legalName": "U.S. Geological Survey" },
-      "keywords": "water, data viz, data visualization, groundwater" }
+      "keywords": "water, data viz, data visualization, groundwater, animation, map" }
     </script>
   </head>
   <body>

public/quant_peaks.csv

@@ -1,7 +1,6 @@
-color,peak_mid,quant,path_quant
-#7E1900,5,Verylow,M-10 0 C -10 0 0 45 10 0 Z
-#C1A53A,17.5,Low,M-10 0 C -10 0 0 32 10 0 Z
-#8fce83,50,Normal,M-10 0 C -10 0 0 15 10 0 Z
-#8fce83,50,Normal,M-10 0 C -10 0 0 -15 10 0 Z
-#479BC5,82.5,High,M-10 0 C -10 0 0 -32 10 0 Z
-#1A3399,95,Veryhigh,M-10 0 C -10 0 0 -45 10 0 Z
+peak_mid,quant,path_quant
+5,Verylow,M-10 0 C -10 0 0 45 10 0 Z
+17.5,Low,M-10 0 C -10 0 0 32 10 0 Z
+50,Normal,M-10 0 C -10 0 0 15 10 0 C 10 0 0 -15 -10 0 Z
+82.5,High,M-10 0 C -10 0 0 -32 10 0 Z
+95,Veryhigh,M-10 0 C -10 0 0 -45 10 0 Z