From cb419716ee3ae90534d92e868c9a63683237759b Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Wed, 3 Jul 2024 20:08:36 +0530 Subject: [PATCH 01/37] Accept site.data as function input In do_conversion.R Signed-off-by: Abhinav Pandey --- base/workflow/R/do_conversions.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/workflow/R/do_conversions.R b/base/workflow/R/do_conversions.R index 2d886ba4cbf..54c988d208e 100644 --- a/base/workflow/R/do_conversions.R +++ b/base/workflow/R/do_conversions.R @@ -5,10 +5,11 @@ ##' @description Input conversion workflow ##' @param settings PEcAn settings list ##' @param overwrite.met,overwrite.fia,overwrite.ic logical +##' @param site.data ##' ##' @author Ryan Kelly, Rob Kooper, Betsy Cowdery, Istem Fer -do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE) { +do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE, site.data=NULL) { if (PEcAn.settings::is.MultiSettings(settings)) { return(PEcAn.settings::papply(settings, do_conversions)) } From 8b82f2bb1a15bc40301d48feb8d0bc1634177901 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Wed, 3 Jul 2024 21:19:48 +0530 Subject: [PATCH 02/37] Pass site.data via args Signed-off-by: Abhinav Pandey --- web/workflow.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/workflow.R b/web/workflow.R index ea5ae7d9510..0e4ea5320c0 100755 --- a/web/workflow.R +++ b/web/workflow.R @@ -75,7 +75,7 @@ if (args$continue && file.exists(status_file)) { } # Do conversions -settings <- PEcAn.workflow::do_conversions(settings) +settings <- PEcAn.workflow::do_conversions(settings, args$site.data) # Query the trait database for data and priors if (PEcAn.utils::status.check("TRAIT") == 0) { From 8c998fdba504751a44e5af2a8eaf907b658a6b95 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Wed, 3 Jul 2024 21:27:58 +0530 Subject: [PATCH 03/37] Read Command Line arguments Signed-off-by: Abhinav Pandey --- base/settings/R/get_args.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/settings/R/get_args.R b/base/settings/R/get_args.R index 277ad5e0e53..ea203d6b59b 100644 --- a/base/settings/R/get_args.R +++ b/base/settings/R/get_args.R @@ -24,6 +24,9 @@ get_args <- function() { type = "logical", help = "Continue processing", ) + make_option(c("-l", "--lat"), type = "numeric", default = NULL, help = "Latitude"), + make_option(c("-o", "--lon"), type = "numeric", default = NULL, help = "Longitude"), + make_option(c("-s", "--siteID"), type = "character", default = NULL, help = "Site ID (optional)", action = "store") ) parser <- optparse::OptionParser(option_list = option_list) From 7a5a84847e79bbc3f7a9a1da787903c025b1e6fb Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 4 Jul 2024 02:40:56 +0530 Subject: [PATCH 04/37] Update str_ns declaration Signed-off-by: Abhinav Pandey --- base/settings/R/get_args.R | 3 --- base/workflow/R/do_conversions.R | 3 +-- modules/data.atmosphere/R/met.process.R | 10 +++++++--- web/workflow.R | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/base/settings/R/get_args.R b/base/settings/R/get_args.R index ea203d6b59b..277ad5e0e53 100644 --- a/base/settings/R/get_args.R +++ b/base/settings/R/get_args.R @@ -24,9 +24,6 @@ get_args <- function() { type = "logical", help = "Continue processing", ) - make_option(c("-l", "--lat"), type = "numeric", default = NULL, help = "Latitude"), - make_option(c("-o", "--lon"), type = "numeric", default = NULL, help = "Longitude"), - make_option(c("-s", "--siteID"), type = "character", default = NULL, help = "Site ID (optional)", action = "store") ) parser <- optparse::OptionParser(option_list = option_list) diff --git a/base/workflow/R/do_conversions.R b/base/workflow/R/do_conversions.R index 54c988d208e..2d886ba4cbf 100644 --- a/base/workflow/R/do_conversions.R +++ b/base/workflow/R/do_conversions.R @@ -5,11 +5,10 @@ ##' @description Input conversion workflow ##' @param settings PEcAn settings list ##' @param overwrite.met,overwrite.fia,overwrite.ic logical -##' @param site.data ##' ##' @author Ryan Kelly, Rob Kooper, Betsy Cowdery, Istem Fer -do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE, site.data=NULL) { +do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE) { if (PEcAn.settings::is.MultiSettings(settings)) { return(PEcAn.settings::papply(settings, do_conversions)) } diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index 8e1408b3887..417ea9f6b50 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,11 +159,15 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] - new.site <- data.frame(id = as.numeric(site$id), + if(is.null(site$lat) | is.null(site.lon)) { + latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] + new.site <- data.frame(id = as.numeric(site$id), lat = latlon$lat, lon = latlon$lon) - str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + str_ns <- paste0(new.site$lat,'-',new.site$lon) + } else { + str_ns <- paste0(site$lat,'-',site.lon) + } if (is.null(format.vars$lat)) { format.vars$lat <- new.site$lat diff --git a/web/workflow.R b/web/workflow.R index 0e4ea5320c0..ea5ae7d9510 100755 --- a/web/workflow.R +++ b/web/workflow.R @@ -75,7 +75,7 @@ if (args$continue && file.exists(status_file)) { } # Do conversions -settings <- PEcAn.workflow::do_conversions(settings, args$site.data) +settings <- PEcAn.workflow::do_conversions(settings) # Query the trait database for data and priors if (PEcAn.utils::status.check("TRAIT") == 0) { From 74365fdc73edf26b5c6db9f1af16804d1c3ad4c3 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 4 Jul 2024 02:42:03 +0530 Subject: [PATCH 05/37] Update site$lon Signed-off-by: Abhinav Pandey --- modules/data.atmosphere/R/met.process.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index 417ea9f6b50..c7a17d546e8 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,14 +159,14 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - if(is.null(site$lat) | is.null(site.lon)) { + if(is.null(site$lat) | is.null(site$lon)) { latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] new.site <- data.frame(id = as.numeric(site$id), lat = latlon$lat, lon = latlon$lon) str_ns <- paste0(new.site$lat,'-',new.site$lon) } else { - str_ns <- paste0(site$lat,'-',site.lon) + str_ns <- paste0(site$lat,'-',site$lon) } if (is.null(format.vars$lat)) { From a0ae68948d0ae3e651ae91e595c8548bec35d9a7 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 4 Jul 2024 13:33:58 +0530 Subject: [PATCH 06/37] Update if-else block for site.id Signed-off-by: Abhinav Pandey --- modules/data.atmosphere/R/met.process.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index c7a17d546e8..f99d6ea00d8 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -166,6 +166,9 @@ met.process <- function(site, input_met, start_date, end_date, model, lon = latlon$lon) str_ns <- paste0(new.site$lat,'-',new.site$lon) } else { + new.site <- data.frame(id = as.numeric(site$id), + lat = site$lat, + lon = site$lon) str_ns <- paste0(site$lat,'-',site$lon) } From 18dd086b64239925792fe03a59832ae2740e4b0a Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Fri, 5 Jul 2024 23:43:28 +0530 Subject: [PATCH 07/37] Add helper function to get new.site Signed-off-by: Abhinav Pandey --- base/utils/R/get.new.site.R | 45 +++++++++++++++++++++++++ base/workflow/R/do_conversions.R | 8 +++-- modules/data.atmosphere/R/met.process.R | 17 +++------- modules/data.land/R/ic_process.R | 14 ++++---- modules/data.land/R/soil_process.R | 12 +++---- 5 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 base/utils/R/get.new.site.R diff --git a/base/utils/R/get.new.site.R b/base/utils/R/get.new.site.R new file mode 100644 index 00000000000..08b121f8dfe --- /dev/null +++ b/base/utils/R/get.new.site.R @@ -0,0 +1,45 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2012 University of Illinois, NCSA. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the +# University of Illinois/NCSA Open Source License +# which accompanies this distribution, and is available at +# http://opensource.ncsa.illinois.edu/license.html +#------------------------------------------------------------------------------- + +##' Get new site info using provided site information +##' +##' @title Get New Site Info +##' @params site a dataframe with site information on lat, lon and time_zone +##' @param con Database connection object +##' @return a dataframe with new site information on lat, lon and time_zone +##' @export +##' @author Abhinav Pandey +##' +##' @examples +##' get.site.info(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) + +get.site.info <- function(site, con) { + + # setup site database number, lat, lon and name and copy for format.vars if new input + if (is.null(site$lat) | is.null(site$lon)) { + latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] + new.site <- data.frame( + id = as.numeric(site$id), + lat = latlon$lat, + lon = latlon$lon + ) + str_ns <- paste0(new.site$lat, "-", new.site$lon) + } else { + new.site <- data.frame( + id = as.numeric(site$id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(site$lat, "-", site$lon) + } + + site.info <- list(new.site = new.site, str_ns = str_ns) + + return(site.info) +} diff --git a/base/workflow/R/do_conversions.R b/base/workflow/R/do_conversions.R index 2d886ba4cbf..af615c4aeaf 100644 --- a/base/workflow/R/do_conversions.R +++ b/base/workflow/R/do_conversions.R @@ -9,6 +9,10 @@ ##' @author Ryan Kelly, Rob Kooper, Betsy Cowdery, Istem Fer do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE) { + + # extract site info from settings to be utilizes later in processing. + site.info <- settings$run$site + if (PEcAn.settings::is.MultiSettings(settings)) { return(PEcAn.settings::papply(settings, do_conversions)) } @@ -52,7 +56,7 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALS # IC conversion : for now for ED only, hence the css/pss/site check # TRUE if (ic.flag) { - settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic) + settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic, site = site.info) needsave <- TRUE } @@ -88,7 +92,7 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALS PEcAn.logger::logger.info("calling met.process: ",settings$run$inputs[[i]][['path']]) settings$run$inputs[[i]] <- PEcAn.data.atmosphere::met.process( - site = settings$run$site, + site = site.info, input_met = settings$run$inputs$met, start_date = settings$run$start.date, end_date = settings$run$end.date, diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index f99d6ea00d8..b41dd08e7c0 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,18 +159,11 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - if(is.null(site$lat) | is.null(site$lon)) { - latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] - new.site <- data.frame(id = as.numeric(site$id), - lat = latlon$lat, - lon = latlon$lon) - str_ns <- paste0(new.site$lat,'-',new.site$lon) - } else { - new.site <- data.frame(id = as.numeric(site$id), - lat = site$lat, - lon = site$lon) - str_ns <- paste0(site$lat,'-',site$lon) - } + site.info <- PEcAn.remote::get.site.info(site, con) + + # extract new.site and str_ns from site.info + new.site <- site.info$new.site + str_ns <- site.info$str_ns if (is.null(format.vars$lat)) { format.vars$lat <- new.site$lat diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 2c402692835..5a4a48184db 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -6,14 +6,14 @@ ##' @param input Taken from settings$run$inputs. This should include id, path, and source ##' @param dir settings$database$dbfiles ##' @param overwrite Default = FALSE. whether to force ic_process to proceed +##' @param site Current site information ##' ##' @author Istem Fer, Hamze Dokoohaki -ic_process <- function(settings, input, dir, overwrite = FALSE){ +ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$run$site){ #--------------------------------------------------------------------------------------------------# # Extract info from settings and setup - site <- settings$run$site model <- list() model$type <- settings$model$type model$id <- settings$model$id @@ -49,12 +49,12 @@ ic_process <- function(settings, input, dir, overwrite = FALSE){ con <- PEcAn.DB::db.open(dbparms$bety) on.exit(PEcAn.DB::db.close(con), add = TRUE) - #grab site lat and lon info - latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] # setup site database number, lat, lon and name and copy for format.vars if new input - new.site <- data.frame(id = as.numeric(site$id), - lat = latlon$lat, - lon = latlon$lon) + site.info <- PEcAn.remote::get.site.info(site, con) + + # extract new.site and str_ns from site.info + new.site <- site.info$new.site + str_ns <- site.info$str_ns new.site$name <- settings$run$site$name diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index 585f08a4311..c6801ca90cb 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -28,13 +28,13 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T # set up bety connection con <- PEcAn.DB::db.open(dbparms$bety) on.exit(PEcAn.DB::db.close(con), add = TRUE) - # get site info - latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] - new.site <- data.frame(id = as.numeric(site$id), - lat = latlon$lat, - lon = latlon$lon) + + # setup site database number, lat, lon and name and copy for format.vars if new input + site.info <- PEcAn.remote::get.site.info(site, con) - str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + # extract new.site and str_ns from site.info + new.site <- site.info$new.site + str_ns <- site.info$str_ns outfolder <- file.path(dbfiles, paste0(input$source, "_site_", str_ns)) From 7a691135b1e299c8f6a5975db61a8dbcc7e0542b Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 00:04:18 +0530 Subject: [PATCH 08/37] Roxygenise the documentations Signed-off-by: Abhinav Pandey --- base/utils/NAMESPACE | 1 + base/utils/R/get.new.site.R | 2 +- base/utils/man/get.site.info.Rd | 25 +++++++++++++++++++++++++ modules/data.land/man/ic_process.Rd | 4 +++- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 base/utils/man/get.site.info.Rd diff --git a/base/utils/NAMESPACE b/base/utils/NAMESPACE index 300bd155fbb..c3af52c8c35 100644 --- a/base/utils/NAMESPACE +++ b/base/utils/NAMESPACE @@ -22,6 +22,7 @@ export(get.quantiles) export(get.run.id) export(get.sa.sample.list) export(get.sa.samples) +export(get.site.info) export(left.pad.zeros) export(listToArgString) export(load.modelpkg) diff --git a/base/utils/R/get.new.site.R b/base/utils/R/get.new.site.R index 08b121f8dfe..08e5d3b0221 100644 --- a/base/utils/R/get.new.site.R +++ b/base/utils/R/get.new.site.R @@ -10,7 +10,7 @@ ##' Get new site info using provided site information ##' ##' @title Get New Site Info -##' @params site a dataframe with site information on lat, lon and time_zone +##' @param site a dataframe with site information on lat, lon and time_zone ##' @param con Database connection object ##' @return a dataframe with new site information on lat, lon and time_zone ##' @export diff --git a/base/utils/man/get.site.info.Rd b/base/utils/man/get.site.info.Rd new file mode 100644 index 00000000000..93e1f6f6d04 --- /dev/null +++ b/base/utils/man/get.site.info.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get.new.site.R +\name{get.site.info} +\alias{get.site.info} +\title{Get New Site Info} +\usage{ +get.site.info(site, con) +} +\arguments{ +\item{site}{a dataframe with site information on lat, lon and time_zone} + +\item{con}{Database connection object} +} +\value{ +a dataframe with new site information on lat, lon and time_zone +} +\description{ +Get new site info using provided site information +} +\examples{ +get.site.info(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) +} +\author{ +Abhinav Pandey +} diff --git a/modules/data.land/man/ic_process.Rd b/modules/data.land/man/ic_process.Rd index dcf245e41f8..35b69a60ce6 100644 --- a/modules/data.land/man/ic_process.Rd +++ b/modules/data.land/man/ic_process.Rd @@ -4,7 +4,7 @@ \alias{ic_process} \title{ic_process} \usage{ -ic_process(settings, input, dir, overwrite = FALSE) +ic_process(settings, input, dir, overwrite = FALSE, site = settings$run$site) } \arguments{ \item{settings}{pecan settings list} @@ -14,6 +14,8 @@ ic_process(settings, input, dir, overwrite = FALSE) \item{dir}{settings$database$dbfiles} \item{overwrite}{Default = FALSE. whether to force ic_process to proceed} + +\item{site}{Current site information} } \description{ ic_process From 604634b9796c220730541f7913a29d0bf173e00f Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 00:15:09 +0530 Subject: [PATCH 09/37] Update Function Name Signed-off-by: Abhinav Pandey --- base/utils/NAMESPACE | 2 +- base/utils/R/get.new.site.R | 4 ++-- base/utils/man/{get.site.info.Rd => get.new.site.Rd} | 8 ++++---- modules/data.atmosphere/R/met.process.R | 2 +- modules/data.land/R/ic_process.R | 2 +- modules/data.land/R/soil_process.R | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) rename base/utils/man/{get.site.info.Rd => get.new.site.Rd} (72%) diff --git a/base/utils/NAMESPACE b/base/utils/NAMESPACE index c3af52c8c35..8f77c8fc0d1 100644 --- a/base/utils/NAMESPACE +++ b/base/utils/NAMESPACE @@ -17,12 +17,12 @@ export(download.url) export(download_file) export(full.path) export(get.ensemble.inputs) +export(get.new.site) export(get.parameter.stat) export(get.quantiles) export(get.run.id) export(get.sa.sample.list) export(get.sa.samples) -export(get.site.info) export(left.pad.zeros) export(listToArgString) export(load.modelpkg) diff --git a/base/utils/R/get.new.site.R b/base/utils/R/get.new.site.R index 08e5d3b0221..e86c6f43249 100644 --- a/base/utils/R/get.new.site.R +++ b/base/utils/R/get.new.site.R @@ -17,9 +17,9 @@ ##' @author Abhinav Pandey ##' ##' @examples -##' get.site.info(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) +##' get.new.site(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) -get.site.info <- function(site, con) { +get.new.site <- function(site, con) { # setup site database number, lat, lon and name and copy for format.vars if new input if (is.null(site$lat) | is.null(site$lon)) { diff --git a/base/utils/man/get.site.info.Rd b/base/utils/man/get.new.site.Rd similarity index 72% rename from base/utils/man/get.site.info.Rd rename to base/utils/man/get.new.site.Rd index 93e1f6f6d04..2787a3a5ab7 100644 --- a/base/utils/man/get.site.info.Rd +++ b/base/utils/man/get.new.site.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/get.new.site.R -\name{get.site.info} -\alias{get.site.info} +\name{get.new.site} +\alias{get.new.site} \title{Get New Site Info} \usage{ -get.site.info(site, con) +get.new.site(site, con) } \arguments{ \item{site}{a dataframe with site information on lat, lon and time_zone} @@ -18,7 +18,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.site.info(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) +get.new.site(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) } \author{ Abhinav Pandey diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index b41dd08e7c0..62b9252a09b 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,7 +159,7 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.site.info(site, con) + site.info <- PEcAn.remote::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 5a4a48184db..b0f8d24151f 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -50,7 +50,7 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.site.info(site, con) + site.info <- PEcAn.remote::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index c6801ca90cb..277c77936fa 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -30,7 +30,7 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.site.info(site, con) + site.info <- PEcAn.remote::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site From 0f154e59b8917285087a9ce20fe815e482d1bd0d Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 00:38:06 +0530 Subject: [PATCH 10/37] Export dependency to DB rather than utils Signed-off-by: Abhinav Pandey --- base/db/NAMESPACE | 1 + base/{utils => db}/R/get.new.site.R | 2 +- base/{utils => db}/man/get.new.site.Rd | 0 base/utils/NAMESPACE | 1 - modules/data.atmosphere/R/met.process.R | 2 +- modules/data.land/R/ic_process.R | 2 +- modules/data.land/R/soil_process.R | 2 +- 7 files changed, 5 insertions(+), 5 deletions(-) rename base/{utils => db}/R/get.new.site.R (95%) rename base/{utils => db}/man/get.new.site.Rd (100%) diff --git a/base/db/NAMESPACE b/base/db/NAMESPACE index 938b1cfe2d4..7379be2d1de 100644 --- a/base/db/NAMESPACE +++ b/base/db/NAMESPACE @@ -30,6 +30,7 @@ export(derive.traits) export(dplyr.count) export(fancy_scientific) export(get.id) +export(get.new.site) export(get.trait.data) export(get.trait.data.pft) export(get_postgres_envvars) diff --git a/base/utils/R/get.new.site.R b/base/db/R/get.new.site.R similarity index 95% rename from base/utils/R/get.new.site.R rename to base/db/R/get.new.site.R index e86c6f43249..85290e441f7 100644 --- a/base/utils/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -23,7 +23,7 @@ get.new.site <- function(site, con) { # setup site database number, lat, lon and name and copy for format.vars if new input if (is.null(site$lat) | is.null(site$lon)) { - latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] + latlon <- query.site(site$id, con = con)[c("lat", "lon")] new.site <- data.frame( id = as.numeric(site$id), lat = latlon$lat, diff --git a/base/utils/man/get.new.site.Rd b/base/db/man/get.new.site.Rd similarity index 100% rename from base/utils/man/get.new.site.Rd rename to base/db/man/get.new.site.Rd diff --git a/base/utils/NAMESPACE b/base/utils/NAMESPACE index 8f77c8fc0d1..300bd155fbb 100644 --- a/base/utils/NAMESPACE +++ b/base/utils/NAMESPACE @@ -17,7 +17,6 @@ export(download.url) export(download_file) export(full.path) export(get.ensemble.inputs) -export(get.new.site) export(get.parameter.stat) export(get.quantiles) export(get.run.id) diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index 62b9252a09b..bfe273a2b12 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,7 +159,7 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.new.site(site, con) + site.info <- PEcAn.DB::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index b0f8d24151f..8dfde49327f 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -50,7 +50,7 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.new.site(site, con) + site.info <- PEcAn.DB::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index 277c77936fa..845d4077583 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -30,7 +30,7 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.remote::get.new.site(site, con) + site.info <- PEcAn.DB::get.new.site(site, con) # extract new.site and str_ns from site.info new.site <- site.info$new.site From fae37a8e70ea3174dda75758f690dbb1554df610 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 01:45:38 +0530 Subject: [PATCH 11/37] Update test case Signed-off-by: Abhinav Pandey --- modules/data.atmosphere/tests/testthat/test.met.process.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data.atmosphere/tests/testthat/test.met.process.R b/modules/data.atmosphere/tests/testthat/test.met.process.R index 01e819c0744..1b25d14da9f 100644 --- a/modules/data.atmosphere/tests/testthat/test.met.process.R +++ b/modules/data.atmosphere/tests/testthat/test.met.process.R @@ -8,7 +8,7 @@ test_that("`met.process` able to call .download.raw.met.module based on met proc mockery::stub(met.process, 'PEcAn.DB::query.format.vars', list()) mockery::stub(met.process, 'PEcAn.DB::dbfile.check', list(id = 1)) mockery::stub(met.process, 'assign', 1) - mockery::stub(met.process, 'PEcAn.DB::query.site', list(lat = 0, lon = 0)) + mockery::stub(met.process, 'get.new.site', list(new.site = data.frame(id = 1, lat = 0, lon = 0), str_ns = "0-0")) mockery::stub(met.process, 'met.process.stage', list(download.raw = TRUE, met2cf = FALSE, standardize = FALSE, met2model = FALSE)) mocked_res <- mockery::mock(1) From cb420339e12639ea209f0985e746b576eaea1073 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 02:02:48 +0530 Subject: [PATCH 12/37] Update 'exampes' Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 12 +++++++++++- base/db/man/get.new.site.Rd | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 85290e441f7..69b75860976 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -17,9 +17,19 @@ ##' @author Abhinav Pandey ##' ##' @examples -##' get.new.site(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) +##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) get.new.site <- function(site, con) { + + # Check if site dataframe has an id column + if (is.null(site$id)) { + # Trying to stop current running process if no ID present. + # Currently, i want to replace this with a helper function which will check for the ID + ## If ID is not present, the helper function will generate a new one. + PEcAn.logger::logger.error("Site dataframe does not have an id column") + } + + latlon <- NULL # setup site database number, lat, lon and name and copy for format.vars if new input if (is.null(site$lat) | is.null(site$lon)) { diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 2787a3a5ab7..11b2e2d2366 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -18,7 +18,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.new.site(site = data.frame(lat = 40.1, lon = -88.2, time_zone = "America/Chicago"), con = con) +get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) } \author{ Abhinav Pandey From 5ee450a3de9a664936817df1ae4d0f63040e74b2 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 16:27:04 +0530 Subject: [PATCH 13/37] Return null when site$id is not present Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 69b75860976..09e274c3882 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -21,15 +21,16 @@ get.new.site <- function(site, con) { + latlon <- NULL + # Check if site dataframe has an id column if (is.null(site$id)) { # Trying to stop current running process if no ID present. # Currently, i want to replace this with a helper function which will check for the ID ## If ID is not present, the helper function will generate a new one. PEcAn.logger::logger.error("Site dataframe does not have an id column") + return NULL } - - latlon <- NULL # setup site database number, lat, lon and name and copy for format.vars if new input if (is.null(site$lat) | is.null(site$lon)) { From e37b93b3d0f4fc73b6fbfee846b333914a56b01a Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 6 Jul 2024 16:29:25 +0530 Subject: [PATCH 14/37] Remove NULL calling Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 1 - 1 file changed, 1 deletion(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 09e274c3882..82c7a11c420 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -29,7 +29,6 @@ get.new.site <- function(site, con) { # Currently, i want to replace this with a helper function which will check for the ID ## If ID is not present, the helper function will generate a new one. PEcAn.logger::logger.error("Site dataframe does not have an id column") - return NULL } # setup site database number, lat, lon and name and copy for format.vars if new input From 144ce9c79c24124a828d9d335f968f2b324b3533 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sun, 7 Jul 2024 09:04:38 +0530 Subject: [PATCH 15/37] Fixing 'no global binding err' Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 82c7a11c420..57fc90f7664 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -1,7 +1,7 @@ #------------------------------------------------------------------------------- # Copyright (c) 2012 University of Illinois, NCSA. # All rights reserved. This program and the accompanying materials -# are made available under the terms of the +# are made available under the terms of the # University of Illinois/NCSA Open Source License # which accompanies this distribution, and is available at # http://opensource.ncsa.illinois.edu/license.html @@ -20,17 +20,16 @@ ##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) get.new.site <- function(site, con) { - latlon <- NULL # Check if site dataframe has an id column if (is.null(site$id)) { # Trying to stop current running process if no ID present. - # Currently, i want to replace this with a helper function which will check for the ID + # Currently, i want to replace this with a helper function which will check for the ID ## If ID is not present, the helper function will generate a new one. PEcAn.logger::logger.error("Site dataframe does not have an id column") } - + # setup site database number, lat, lon and name and copy for format.vars if new input if (is.null(site$lat) | is.null(site$lon)) { latlon <- query.site(site$id, con = con)[c("lat", "lon")] @@ -48,7 +47,7 @@ get.new.site <- function(site, con) { ) str_ns <- paste0(site$lat, "-", site$lon) } - + site.info <- list(new.site = new.site, str_ns = str_ns) return(site.info) From 54ce64461cad7be15b35123a3cba5d8a1b2d59b2 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 09:31:27 +0530 Subject: [PATCH 16/37] update global variable handling Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 57fc90f7664..0ce49459bd6 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -19,6 +19,9 @@ ##' @examples ##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) +# Trying to Inform R CMD check about global variables +utils::globalVariables(c("latlon")) + get.new.site <- function(site, con) { latlon <- NULL From 6d76455ab3f71477a6f05a861762548a05ef859f Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 09:42:32 +0530 Subject: [PATCH 17/37] Remove global binding changes Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 3 --- 1 file changed, 3 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 0ce49459bd6..57fc90f7664 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -19,9 +19,6 @@ ##' @examples ##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) -# Trying to Inform R CMD check about global variables -utils::globalVariables(c("latlon")) - get.new.site <- function(site, con) { latlon <- NULL From ca0b50e37576d11271b10aef7866551785cfb961 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 15:25:46 +0530 Subject: [PATCH 18/37] Update global variables Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 5 ++--- modules/data.atmosphere/R/met.process.R | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 57fc90f7664..7f510a09705 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -19,8 +19,7 @@ ##' @examples ##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) -get.new.site <- function(site, con) { - latlon <- NULL +get.new.site <- function(site, con, latlon = NULL) { # Check if site dataframe has an id column if (is.null(site$id)) { @@ -50,5 +49,5 @@ get.new.site <- function(site, con) { site.info <- list(new.site = new.site, str_ns = str_ns) - return(site.info) + return (site.info) } diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index bfe273a2b12..f32c06dd6a3 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -159,7 +159,8 @@ met.process <- function(site, input_met, start_date, end_date, model, } # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.DB::get.new.site(site, con) + latlon <- NULL + site.info <- PEcAn.DB::get.new.site(site, con, latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site From 80821948cc6bec5e260d2618da5300a73bab9b5d Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 15:27:31 +0530 Subject: [PATCH 19/37] Ipdate all instances of latlon Signed-off-by: Abhinav Pandey --- modules/data.land/R/ic_process.R | 3 ++- modules/data.land/R/soil_process.R | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 8dfde49327f..52fa252a773 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -50,7 +50,8 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.DB::get.new.site(site, con) + latlon <- NULL + site.info <- PEcAn.DB::get.new.site(site, con, latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index 845d4077583..e9ba0e87672 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -30,7 +30,8 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T on.exit(PEcAn.DB::db.close(con), add = TRUE) # setup site database number, lat, lon and name and copy for format.vars if new input - site.info <- PEcAn.DB::get.new.site(site, con) + latlon <- NULL + site.info <- PEcAn.DB::get.new.site(site, con, latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site From 65cdea76de0ecc20dd561a9513fda93b9cd668b9 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 15:40:35 +0530 Subject: [PATCH 20/37] Roxygenise documents Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 1 + base/db/man/get.new.site.Rd | 5 +++-- modules/data.atmosphere/R/met.process.R | 2 +- modules/data.land/R/ic_process.R | 2 +- modules/data.land/R/soil_process.R | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 7f510a09705..35528688d1b 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -12,6 +12,7 @@ ##' @title Get New Site Info ##' @param site a dataframe with site information on lat, lon and time_zone ##' @param con Database connection object +##' latlon Global latlon abjoect which will store updated lat/lon ##' @return a dataframe with new site information on lat, lon and time_zone ##' @export ##' @author Abhinav Pandey diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 11b2e2d2366..9030fcf237f 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -4,12 +4,13 @@ \alias{get.new.site} \title{Get New Site Info} \usage{ -get.new.site(site, con) +get.new.site(site, con, latlon = NULL) } \arguments{ \item{site}{a dataframe with site information on lat, lon and time_zone} -\item{con}{Database connection object} +\item{con}{Database connection object +latlon Global latlon abjoect which will store updated lat/lon} } \value{ a dataframe with new site information on lat, lon and time_zone diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index f32c06dd6a3..5225820d23f 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -160,7 +160,7 @@ met.process <- function(site, input_met, start_date, end_date, model, # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon) + site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 52fa252a773..63cd9a55a93 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -51,7 +51,7 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon) + site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index e9ba0e87672..b31894baedf 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -31,7 +31,7 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon) + site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site From acb850798ba0cce13b270d74ca824ca98609596e Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 15:46:32 +0530 Subject: [PATCH 21/37] Usage Updates added Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 2 +- base/db/man/get.new.site.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 35528688d1b..38dcd4eaee2 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -18,7 +18,7 @@ ##' @author Abhinav Pandey ##' ##' @examples -##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) +##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con, latlon = NULL) get.new.site <- function(site, con, latlon = NULL) { diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 9030fcf237f..849e7b5e14f 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -19,7 +19,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con) +get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con, latlon = NULL) } \author{ Abhinav Pandey From 59f451aefdda9209860c6d49edfb0be4a2e64e21 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Mon, 8 Jul 2024 15:58:50 +0530 Subject: [PATCH 22/37] Doc Updates added Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 11 +++++------ base/db/man/get.new.site.Rd | 9 +++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 38dcd4eaee2..835a71641c5 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -10,18 +10,17 @@ ##' Get new site info using provided site information ##' ##' @title Get New Site Info -##' @param site a dataframe with site information on lat, lon and time_zone -##' @param con Database connection object -##' latlon Global latlon abjoect which will store updated lat/lon +##' @param site a dataframe with site information including id, lat, lon, and time_zone. +##' @param con Database connection object. +##' @param latlon Optional global latlon object which will store updated lat/lon. ##' @return a dataframe with new site information on lat, lon and time_zone ##' @export ##' @author Abhinav Pandey ##' ##' @examples -##' get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con, latlon = NULL) +##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=con,latlon=NULL) get.new.site <- function(site, con, latlon = NULL) { - # Check if site dataframe has an id column if (is.null(site$id)) { # Trying to stop current running process if no ID present. @@ -50,5 +49,5 @@ get.new.site <- function(site, con, latlon = NULL) { site.info <- list(new.site = new.site, str_ns = str_ns) - return (site.info) + return(site.info) } diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 849e7b5e14f..023de7bdb35 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -7,10 +7,11 @@ get.new.site(site, con, latlon = NULL) } \arguments{ -\item{site}{a dataframe with site information on lat, lon and time_zone} +\item{site}{a dataframe with site information including id, lat, lon, and time_zone.} -\item{con}{Database connection object -latlon Global latlon abjoect which will store updated lat/lon} +\item{con}{Database connection object.} + +\item{latlon}{Optional global latlon object which will store updated lat/lon.} } \value{ a dataframe with new site information on lat, lon and time_zone @@ -19,7 +20,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.new.site(site = data.frame(id = 1, lat = 40.1, lon = -88.2, time_zone = "UTC"), con = con, latlon = NULL) +get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=con,latlon=NULL) } \author{ Abhinav Pandey From f0ca425b5c01d6dfb03b05c5afffdb7ae16041ff Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 09:02:20 +0530 Subject: [PATCH 23/37] Update test files Signed-off-by: Abhinav Pandey --- modules/data.atmosphere/tests/testthat/test.met.process.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data.atmosphere/tests/testthat/test.met.process.R b/modules/data.atmosphere/tests/testthat/test.met.process.R index 1b25d14da9f..5534d6ca7e1 100644 --- a/modules/data.atmosphere/tests/testthat/test.met.process.R +++ b/modules/data.atmosphere/tests/testthat/test.met.process.R @@ -1,4 +1,4 @@ -test_that("`met.process` able to call .download.raw.met.module based on met process stage params", { +test_that("'met.proces' able to call .download.raw.met.module based on met process stage params", { input_met <- list(source = 'CRUNCEP', id = '1') mockery::stub(met.process, 'PEcAn.DB::db.open', 1) @@ -8,7 +8,7 @@ test_that("`met.process` able to call .download.raw.met.module based on met proc mockery::stub(met.process, 'PEcAn.DB::query.format.vars', list()) mockery::stub(met.process, 'PEcAn.DB::dbfile.check', list(id = 1)) mockery::stub(met.process, 'assign', 1) - mockery::stub(met.process, 'get.new.site', list(new.site = data.frame(id = 1, lat = 0, lon = 0), str_ns = "0-0")) + mockery::stub(met.process, 'PEcAn.DB::get.new.site', list(new.site = data.frame(id = 1, lat = 0, lon = 0), str_ns = "0-0")) mockery::stub(met.process, 'met.process.stage', list(download.raw = TRUE, met2cf = FALSE, standardize = FALSE, met2model = FALSE)) mocked_res <- mockery::mock(1) From d66503f34b008448283776493c2083368310c506 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 11:35:40 +0530 Subject: [PATCH 24/37] Refactor get.new.site function to generate site IDs and handle missing values when con is not provided Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 143 +++++++++++++++++++++++++++++++++------ 1 file changed, 121 insertions(+), 22 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 835a71641c5..73a30965b87 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -21,33 +21,132 @@ ##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=con,latlon=NULL) get.new.site <- function(site, con, latlon = NULL) { - # Check if site dataframe has an id column - if (is.null(site$id)) { - # Trying to stop current running process if no ID present. - # Currently, i want to replace this with a helper function which will check for the ID - ## If ID is not present, the helper function will generate a new one. - PEcAn.logger::logger.error("Site dataframe does not have an id column") - } + if (is.null(con)) { + # No DB connection present. Generate a new ID using one of below steps: - # setup site database number, lat, lon and name and copy for format.vars if new input - if (is.null(site$lat) | is.null(site$lon)) { - latlon <- query.site(site$id, con = con)[c("lat", "lon")] - new.site <- data.frame( - id = as.numeric(site$id), - lat = latlon$lat, - lon = latlon$lon - ) - str_ns <- paste0(new.site$lat, "-", new.site$lon) + if (is.null(site$id) | is.na(site$id)) { + if ((!is.null(site$lat) && !is.null(site$lon)) | + (!is.na(site$lat) && !is.na(site$lon)) + ) { + site.id <- generate_siteID(site$lat, site$lon) + new.site <- data.frame( + id = as.numeric(site.id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(new.site$lat, "-", new.site$lon) + } # lat/lon present but ID is absent + # Use Pre-existing normalised lat/lon + else { + PEcAn.logger::logger.warn("Site dataframe does not have an id column") + site.id <- generate_new_siteID() + PEcAn.logger::logger.info(paste0("Generated siteID:", site.id)) + # Optionally, create a new site data frame with the random ID + new.site <- data.frame( + id = site.id, + lat = NULL, + lon = NULL + ) + str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + # We used a different str_ns as identifier here due to absence of lat/lon + } + # ID as well as lat/lon absent. + # Return a WARN as we will be unable to identify such an instance due to lack of information. + # We'll try to Generate a new ID similar to previous ones. + } else { + if ((!is.null(site$lat) && !is.null(site$lon)) | + (!is.na(site$lat) && !is.na(site$lon)) + ) { + new.site <- data.frame( + id = as.numeric(site$id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(new.site$lat, "-", new.site$lon) + } # siteId as well as lat/lon present + else { + new.site <- data.frame( + id = site$id, + lat = NULL, + lon = NULL + ) + str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + } # siteId present but lat/lon absent + } } else { - new.site <- data.frame( - id = as.numeric(site$id), - lat = site$lat, - lon = site$lon - ) - str_ns <- paste0(site$lat, "-", site$lon) + # Check if site dataframe has an id column + if (is.null(site$id) | is.na(site$id)) { + PEcAn.logger::warn("Site dataframe does not have an id column. Generating a unique ID") + site.id <- generate_new_siteID() + PEcAn.logger::logger.info(paste0("Generated siteID:", site.id)) + if ((!is.null(site$lat) && !is.null(site$lon)) | + (!is.na(site$lat) && !is.na(site$lon)) + ) { + new.site <- data.frame( + id = as.numeric(site.id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(site$lat, "-", site$lon) + } else { + new.site <- data.frame( + id = site.id, + lat = NULL, + lon = NULL + ) + str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + # We used a different str_ns as identifier here due to absence of lat/lon + } + } else { + # setup site database number, lat, lon and name and copy for format.vars if new input + if ((!is.null(site$lat) && !is.null(site$lon)) | + (!is.na(site$lat) && !is.na(site$lon)) + ) { + new.site <- data.frame( + id = as.numeric(site$id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(site$lat, "-", site$lon) + } else { + latlon <- query.site(site$id, con = con)[c("lat", "lon")] + new.site <- data.frame( + id = as.numeric(site$id), + lat = latlon$lat, + lon = latlon$lon + ) + str_ns <- paste0(new.site$lat, "-", new.site$lon) + } + } } site.info <- list(new.site = new.site, str_ns = str_ns) return(site.info) } + +# Function to generate a normalized site ID using lat / lon +# Example usage +# siteID <- generate_siteID(-34.9284989, 138.6007456) +generate_siteID <- function(lat, lon) { + # Normalize latitude and longitude + norm_lat <- lat + 90 # Shift range to [0, 180] + norm_lon <- lon + 180 # Shift range to [0, 360] + + # Scale normalized values (Assuming 4 digits each for lat and lon) + scaled_lat <- as.integer((norm_lat / 180) * 9999) # Scaled latitude + scaled_lon <- as.integer((norm_lon / 360) * 9999) # Scaled longitude + + # Create a unique ID by concatenating scaled values + siteID <- as.integer(paste0(scaled_lat, scaled_lon)) + + # Return the siteID + return(siteID) +} + +# Function to generate a new siteID (db-less runs ONLY) +generate_new_siteID <- function() { + # Generate a random number. Assuming higher order integers to increase randomness in IDs + random_id <- sample(10000:99999999, 1) + return(as.numeric(random_id)) +} From 6d0b73039ec050cb660edb8ccce80400d550c7ea Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 11:39:11 +0530 Subject: [PATCH 25/37] Update site utilizations Signed-off-by: Abhinav Pandey --- base/workflow/R/do_conversions.R | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/base/workflow/R/do_conversions.R b/base/workflow/R/do_conversions.R index af615c4aeaf..adffc015e24 100644 --- a/base/workflow/R/do_conversions.R +++ b/base/workflow/R/do_conversions.R @@ -10,9 +10,6 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALSE, overwrite.ic = FALSE) { - # extract site info from settings to be utilizes later in processing. - site.info <- settings$run$site - if (PEcAn.settings::is.MultiSettings(settings)) { return(PEcAn.settings::papply(settings, do_conversions)) } @@ -56,7 +53,7 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALS # IC conversion : for now for ED only, hence the css/pss/site check # TRUE if (ic.flag) { - settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic, site = site.info) + settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic, site = settings$run$site) needsave <- TRUE } @@ -92,7 +89,7 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALS PEcAn.logger::logger.info("calling met.process: ",settings$run$inputs[[i]][['path']]) settings$run$inputs[[i]] <- PEcAn.data.atmosphere::met.process( - site = site.info, + site = settings$run$site, input_met = settings$run$inputs$met, start_date = settings$run$start.date, end_date = settings$run$end.date, From f5fac4463069a985bd8e7fbcee0de52a7a1a868e Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 11:54:07 +0530 Subject: [PATCH 26/37] Update documentations Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 2 +- base/db/man/get.new.site.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 73a30965b87..21cb9fcacd9 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -18,7 +18,7 @@ ##' @author Abhinav Pandey ##' ##' @examples -##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=con,latlon=NULL) +##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con,latlon=NULL) get.new.site <- function(site, con, latlon = NULL) { if (is.null(con)) { diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 023de7bdb35..4fdf79e2f80 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -20,7 +20,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=con,latlon=NULL) +get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con,latlon=NULL) } \author{ Abhinav Pandey From 9d328eb460ca853b9c970b28be5982418754a999 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 12:04:49 +0530 Subject: [PATCH 27/37] Refactor Errors in check Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 21cb9fcacd9..a21d52639d6 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -76,7 +76,7 @@ get.new.site <- function(site, con, latlon = NULL) { } else { # Check if site dataframe has an id column if (is.null(site$id) | is.na(site$id)) { - PEcAn.logger::warn("Site dataframe does not have an id column. Generating a unique ID") + PEcAn.logger::logger.warn("Site dataframe does not have an id column. Generating a unique ID") site.id <- generate_new_siteID() PEcAn.logger::logger.info(paste0("Generated siteID:", site.id)) if ((!is.null(site$lat) && !is.null(site$lon)) | From 45fc781a3f7d1523bbf2846e191d9a5730ab9551 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Tue, 9 Jul 2024 12:26:22 +0530 Subject: [PATCH 28/37] Update conn pass Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 4 ++-- base/db/man/get.new.site.Rd | 4 ++-- modules/data.atmosphere/R/met.process.R | 2 +- modules/data.land/R/ic_process.R | 2 +- modules/data.land/R/soil_process.R | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index a21d52639d6..e8d2c8124ae 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -18,9 +18,9 @@ ##' @author Abhinav Pandey ##' ##' @examples -##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con,latlon=NULL) +##' get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=NULL,latlon=NULL) -get.new.site <- function(site, con, latlon = NULL) { +get.new.site <- function(site, con = NULL, latlon = NULL) { if (is.null(con)) { # No DB connection present. Generate a new ID using one of below steps: diff --git a/base/db/man/get.new.site.Rd b/base/db/man/get.new.site.Rd index 4fdf79e2f80..f00c6103513 100644 --- a/base/db/man/get.new.site.Rd +++ b/base/db/man/get.new.site.Rd @@ -4,7 +4,7 @@ \alias{get.new.site} \title{Get New Site Info} \usage{ -get.new.site(site, con, latlon = NULL) +get.new.site(site, con = NULL, latlon = NULL) } \arguments{ \item{site}{a dataframe with site information including id, lat, lon, and time_zone.} @@ -20,7 +20,7 @@ a dataframe with new site information on lat, lon and time_zone Get new site info using provided site information } \examples{ -get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con,latlon=NULL) +get.new.site(site=data.frame(id=1,lat=40.1,lon=-88.2,time_zone="UTC"),con=NULL,latlon=NULL) } \author{ Abhinav Pandey diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index 5225820d23f..fbad54da104 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -160,7 +160,7 @@ met.process <- function(site, input_met, start_date, end_date, model, # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) + site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 63cd9a55a93..444d77835cb 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -51,7 +51,7 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) + site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index b31894baedf..88fbf09e763 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -31,7 +31,7 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con, latlon = latlon) + site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) # extract new.site and str_ns from site.info new.site <- site.info$new.site From e2cf4f12fe9938592b939368031f5591955df16b Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 18 Jul 2024 23:15:54 +0530 Subject: [PATCH 29/37] Update ic_process Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 1 + modules/data.land/R/ic_process.R | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index e8d2c8124ae..dda12b7fdc6 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -22,6 +22,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { if (is.null(con)) { + PEcAn.logger::loger.info("DB connection is closed. Trying to generate a new site ID or use pre-existing one.") # No DB connection present. Generate a new ID using one of below steps: if (is.null(site$id) | is.na(site$id)) { diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 444d77835cb..0c513748f91 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -51,15 +51,24 @@ ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$ # setup site database number, lat, lon and name and copy for format.vars if new input latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) + if(is.null(site$lat) | is.null(site$lon)) { + site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) - # extract new.site and str_ns from site.info - new.site <- site.info$new.site - str_ns <- site.info$str_ns + # extract new.site and str_ns from site.info + new.site <- site.info$new.site + str_ns <- site.info$str_ns + } else { + latlon <- list(lon = site$lat, lon=site$lon) + new.site <- data.frame( + id = as.numeric(site$id), + lat = site$lat, + lon = site$lon + ) + str_ns <- paste0(site$lat, "-", site$lon) + } new.site$name <- settings$run$site$name - str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) From 18cc1ae1fb990d3af9cad360daee9598270a2661 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Fri, 19 Jul 2024 16:03:58 +0530 Subject: [PATCH 30/37] Update Logger statements Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index dda12b7fdc6..0a9f4ee7059 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -22,7 +22,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { if (is.null(con)) { - PEcAn.logger::loger.info("DB connection is closed. Trying to generate a new site ID or use pre-existing one.") + PEcAn.logger::logger.debug("DB connection is closed. Trying to generate a new site ID or use pre-existing one.") # No DB connection present. Generate a new ID using one of below steps: if (is.null(site$id) | is.na(site$id)) { From d3928472ad724dea1550ae67222a07969efceb70 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Wed, 21 Aug 2024 21:51:04 +0530 Subject: [PATCH 31/37] Update base/db/R/get.new.site.R Co-authored-by: Chris Black --- base/db/R/get.new.site.R | 8 -------- 1 file changed, 8 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 0a9f4ee7059..d165a6e69b7 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -1,11 +1,3 @@ -#------------------------------------------------------------------------------- -# Copyright (c) 2012 University of Illinois, NCSA. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the -# University of Illinois/NCSA Open Source License -# which accompanies this distribution, and is available at -# http://opensource.ncsa.illinois.edu/license.html -#------------------------------------------------------------------------------- ##' Get new site info using provided site information ##' From 0ac5b39d5e0d611b628112d530ea4ef86d4c44ad Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Wed, 28 Aug 2024 22:32:14 +0530 Subject: [PATCH 32/37] remove browndog support from benchmark Signed-off-by: Abhinav Pandey --- docker/depends/pecan_package_dependencies.csv | 1 - modules/benchmark/DESCRIPTION | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/depends/pecan_package_dependencies.csv b/docker/depends/pecan_package_dependencies.csv index fe66050ff8c..da7c272f06a 100644 --- a/docker/depends/pecan_package_dependencies.csv +++ b/docker/depends/pecan_package_dependencies.csv @@ -11,7 +11,6 @@ "BayesianTools","*","modules/rtm","Imports",FALSE "BioCro","*","models/biocro","Suggests",FALSE "bit64","*","base/db","Suggests",FALSE -"BrownDog","*","modules/benchmark","Suggests",FALSE "coda","*","models/maespa","Suggests",FALSE "coda","*","models/sipnet","Suggests",FALSE "coda","*","modules/assim.sequential","Imports",FALSE diff --git a/modules/benchmark/DESCRIPTION b/modules/benchmark/DESCRIPTION index 2cfa4e0593a..71e9330b30e 100644 --- a/modules/benchmark/DESCRIPTION +++ b/modules/benchmark/DESCRIPTION @@ -40,8 +40,7 @@ Imports: zoo Suggests: PEcAn.data.land, - testthat (>= 2.0.0), - BrownDog + testthat (>= 2.0.0) License: BSD_3_clause + file LICENSE Copyright: Authors LazyLoad: yes From 9edc268186fab50483460ec8a98121b458b75b66 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sun, 1 Sep 2024 01:51:12 +0530 Subject: [PATCH 33/37] Refactor changes from review Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 25 +++---------------- base/workflow/R/do_conversions.R | 2 +- .../tests/testthat/test.met.process.R | 2 +- modules/data.land/R/ic_process.R | 3 ++- modules/data.land/R/soil_process.R | 12 ++++----- modules/data.land/man/ic_process.Rd | 2 +- 6 files changed, 14 insertions(+), 32 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index d165a6e69b7..3c645c09ba7 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -1,4 +1,3 @@ - ##' Get new site info using provided site information ##' ##' @title Get New Site Info @@ -18,10 +17,10 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { # No DB connection present. Generate a new ID using one of below steps: if (is.null(site$id) | is.na(site$id)) { - if ((!is.null(site$lat) && !is.null(site$lon)) | + if ((!is.null(site$lat) && !is.null(site$lon)) && (!is.na(site$lat) && !is.na(site$lon)) ) { - site.id <- generate_siteID(site$lat, site$lon) + site.id <- paste0(lat, "_", lon) new.site <- data.frame( id = as.numeric(site.id), lat = site$lat, @@ -47,7 +46,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { # Return a WARN as we will be unable to identify such an instance due to lack of information. # We'll try to Generate a new ID similar to previous ones. } else { - if ((!is.null(site$lat) && !is.null(site$lon)) | + if ((!is.null(site$lat) && !is.null(site$lon)) && (!is.na(site$lat) && !is.na(site$lon)) ) { new.site <- data.frame( @@ -118,24 +117,6 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { return(site.info) } -# Function to generate a normalized site ID using lat / lon -# Example usage -# siteID <- generate_siteID(-34.9284989, 138.6007456) -generate_siteID <- function(lat, lon) { - # Normalize latitude and longitude - norm_lat <- lat + 90 # Shift range to [0, 180] - norm_lon <- lon + 180 # Shift range to [0, 360] - - # Scale normalized values (Assuming 4 digits each for lat and lon) - scaled_lat <- as.integer((norm_lat / 180) * 9999) # Scaled latitude - scaled_lon <- as.integer((norm_lon / 360) * 9999) # Scaled longitude - - # Create a unique ID by concatenating scaled values - siteID <- as.integer(paste0(scaled_lat, scaled_lon)) - - # Return the siteID - return(siteID) -} # Function to generate a new siteID (db-less runs ONLY) generate_new_siteID <- function() { diff --git a/base/workflow/R/do_conversions.R b/base/workflow/R/do_conversions.R index adffc015e24..dea5a5b0793 100644 --- a/base/workflow/R/do_conversions.R +++ b/base/workflow/R/do_conversions.R @@ -53,7 +53,7 @@ do_conversions <- function(settings, overwrite.met = FALSE, overwrite.fia = FALS # IC conversion : for now for ED only, hence the css/pss/site check # TRUE if (ic.flag) { - settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic, site = settings$run$site) + settings <- PEcAn.data.land::ic_process(settings, input, dir = dbfiles, overwrite = overwrite.ic) needsave <- TRUE } diff --git a/modules/data.atmosphere/tests/testthat/test.met.process.R b/modules/data.atmosphere/tests/testthat/test.met.process.R index 5534d6ca7e1..b39e6198737 100644 --- a/modules/data.atmosphere/tests/testthat/test.met.process.R +++ b/modules/data.atmosphere/tests/testthat/test.met.process.R @@ -1,4 +1,4 @@ -test_that("'met.proces' able to call .download.raw.met.module based on met process stage params", { +test_that("'met.process' able to call .download.raw.met.module based on met process stage params", { input_met <- list(source = 'CRUNCEP', id = '1') mockery::stub(met.process, 'PEcAn.DB::db.open', 1) diff --git a/modules/data.land/R/ic_process.R b/modules/data.land/R/ic_process.R index 0c513748f91..5e799dde61b 100644 --- a/modules/data.land/R/ic_process.R +++ b/modules/data.land/R/ic_process.R @@ -9,11 +9,12 @@ ##' @param site Current site information ##' ##' @author Istem Fer, Hamze Dokoohaki -ic_process <- function(settings, input, dir, overwrite = FALSE, site = settings$run$site){ +ic_process <- function(settings, input, dir, overwrite = FALSE){ #--------------------------------------------------------------------------------------------------# # Extract info from settings and setup + site <- settings$run$site model <- list() model$type <- settings$model$type model$id <- settings$model$id diff --git a/modules/data.land/R/soil_process.R b/modules/data.land/R/soil_process.R index 88fbf09e763..7967353b979 100644 --- a/modules/data.land/R/soil_process.R +++ b/modules/data.land/R/soil_process.R @@ -29,13 +29,13 @@ soil_process <- function(settings, input, dbfiles, overwrite = FALSE,run.local=T con <- PEcAn.DB::db.open(dbparms$bety) on.exit(PEcAn.DB::db.close(con), add = TRUE) - # setup site database number, lat, lon and name and copy for format.vars if new input - latlon <- NULL - site.info <- PEcAn.DB::get.new.site(site, con=con, latlon = latlon) + # get site info + latlon <- PEcAn.DB::query.site(site$id, con = con)[c("lat", "lon")] + new.site <- data.frame(id = as.numeric(site$id), + lat = latlon$lat, + lon = latlon$lon) - # extract new.site and str_ns from site.info - new.site <- site.info$new.site - str_ns <- site.info$str_ns + str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) outfolder <- file.path(dbfiles, paste0(input$source, "_site_", str_ns)) diff --git a/modules/data.land/man/ic_process.Rd b/modules/data.land/man/ic_process.Rd index 35b69a60ce6..524ffc3f95f 100644 --- a/modules/data.land/man/ic_process.Rd +++ b/modules/data.land/man/ic_process.Rd @@ -4,7 +4,7 @@ \alias{ic_process} \title{ic_process} \usage{ -ic_process(settings, input, dir, overwrite = FALSE, site = settings$run$site) +ic_process(settings, input, dir, overwrite = FALSE) } \arguments{ \item{settings}{pecan settings list} From 59b293ea79ee1a2a4fb4da9973d37c8b5db223a5 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 5 Sep 2024 12:06:06 +0530 Subject: [PATCH 34/37] Implement temporary hashing to generate UID Signed-off-by: Abhinav Pandey --- base/db/R/get.new.site.R | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 3c645c09ba7..74494f67127 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -26,7 +26,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { lat = site$lat, lon = site$lon ) - str_ns <- paste0(new.site$lat, "-", new.site$lon) + str_ns <- paste0(new.site$lat, "_", new.site$lon) } # lat/lon present but ID is absent # Use Pre-existing normalised lat/lon else { @@ -39,7 +39,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { lat = NULL, lon = NULL ) - str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) + str_ns <- paste0(new.site$id %/% 1e+09, "_", new.site$id %% 1e+09) # We used a different str_ns as identifier here due to absence of lat/lon } # ID as well as lat/lon absent. @@ -69,11 +69,11 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { # Check if site dataframe has an id column if (is.null(site$id) | is.na(site$id)) { PEcAn.logger::logger.warn("Site dataframe does not have an id column. Generating a unique ID") - site.id <- generate_new_siteID() - PEcAn.logger::logger.info(paste0("Generated siteID:", site.id)) - if ((!is.null(site$lat) && !is.null(site$lon)) | + if ((!is.null(site$lat) && !is.null(site$lon)) && (!is.na(site$lat) && !is.na(site$lon)) ) { + PEcAn.logger::logger.info(paste0("Generated siteID using lat and lon:", site.id)) + site.id <- generate_new_siteID(site$lat, site$lon) new.site <- data.frame( id = as.numeric(site.id), lat = site$lat, @@ -81,13 +81,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { ) str_ns <- paste0(site$lat, "-", site$lon) } else { - new.site <- data.frame( - id = site.id, - lat = NULL, - lon = NULL - ) - str_ns <- paste0(new.site$id %/% 1e+09, "-", new.site$id %% 1e+09) - # We used a different str_ns as identifier here due to absence of lat/lon + PEcAn.logger::logger.severe("Missing site-id, site lat & site-lon. Stopping the process due to lack of information") } } else { # setup site database number, lat, lon and name and copy for format.vars if new input @@ -99,7 +93,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { lat = site$lat, lon = site$lon ) - str_ns <- paste0(site$lat, "-", site$lon) + str_ns <- paste0(site$lat, "_", site$lon) } else { latlon <- query.site(site$id, con = con)[c("lat", "lon")] new.site <- data.frame( @@ -107,7 +101,7 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { lat = latlon$lat, lon = latlon$lon ) - str_ns <- paste0(new.site$lat, "-", new.site$lon) + str_ns <- paste0(new.site$lat, "_", new.site$lon) } } } @@ -119,8 +113,9 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { # Function to generate a new siteID (db-less runs ONLY) -generate_new_siteID <- function() { - # Generate a random number. Assuming higher order integers to increase randomness in IDs - random_id <- sample(10000:99999999, 1) - return(as.numeric(random_id)) +generate_new_siteID <- function(lat, lon) { + latlon_str <- paste0(round(lat, 8), "_", round(lon, 8)) + hash <- digest::digest(latlon_str, algo = "sha256") + uid <- substr(hash, 1, 10) + return(uid) } From 65b976deaa68cacf468cf2bc97798df5002c6174 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 5 Sep 2024 12:11:08 +0530 Subject: [PATCH 35/37] Remove unrequired changes which were accidently pushed to this branch Signed-off-by: Abhinav Pandey --- modules/benchmark/DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/benchmark/DESCRIPTION b/modules/benchmark/DESCRIPTION index 71e9330b30e..2cfa4e0593a 100644 --- a/modules/benchmark/DESCRIPTION +++ b/modules/benchmark/DESCRIPTION @@ -40,7 +40,8 @@ Imports: zoo Suggests: PEcAn.data.land, - testthat (>= 2.0.0) + testthat (>= 2.0.0), + BrownDog License: BSD_3_clause + file LICENSE Copyright: Authors LazyLoad: yes From 99e4345db9ca02e1776b3765f4203767789c1337 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 5 Sep 2024 13:50:05 +0530 Subject: [PATCH 36/37] Refactor Dependency chart Signed-off-by: Abhinav Pandey --- docker/depends/pecan_package_dependencies.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/depends/pecan_package_dependencies.csv b/docker/depends/pecan_package_dependencies.csv index da7c272f06a..fe66050ff8c 100644 --- a/docker/depends/pecan_package_dependencies.csv +++ b/docker/depends/pecan_package_dependencies.csv @@ -11,6 +11,7 @@ "BayesianTools","*","modules/rtm","Imports",FALSE "BioCro","*","models/biocro","Suggests",FALSE "bit64","*","base/db","Suggests",FALSE +"BrownDog","*","modules/benchmark","Suggests",FALSE "coda","*","models/maespa","Suggests",FALSE "coda","*","models/sipnet","Suggests",FALSE "coda","*","modules/assim.sequential","Imports",FALSE From a4025a812e331336ff06871f2ce4c96e0fee8e81 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Thu, 5 Sep 2024 22:04:48 +0530 Subject: [PATCH 37/37] Using openssl for hashing lat/lon Signed-off-by: Abhinav Pandey --- base/db/DESCRIPTION | 3 ++- base/db/R/get.new.site.R | 10 ++++++---- docker/depends/pecan_package_dependencies.csv | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/base/db/DESCRIPTION b/base/db/DESCRIPTION index 73a564a8a7f..52c8047dd9c 100644 --- a/base/db/DESCRIPTION +++ b/base/db/DESCRIPTION @@ -73,7 +73,8 @@ Suggests: rmarkdown (>= 2.19), testthat (>= 2.0.0), tidyverse, - withr + withr, + openssl License: BSD_3_clause + file LICENSE VignetteBuilder: knitr Copyright: Authors diff --git a/base/db/R/get.new.site.R b/base/db/R/get.new.site.R index 74494f67127..c44354aae1c 100644 --- a/base/db/R/get.new.site.R +++ b/base/db/R/get.new.site.R @@ -112,10 +112,12 @@ get.new.site <- function(site, con = NULL, latlon = NULL) { } -# Function to generate a new siteID (db-less runs ONLY) +# Function to generate a new siteID using hashing (db-less runs ONLY) generate_new_siteID <- function(lat, lon) { - latlon_str <- paste0(round(lat, 8), "_", round(lon, 8)) - hash <- digest::digest(latlon_str, algo = "sha256") - uid <- substr(hash, 1, 10) + latlon_str <- paste0(round(lat, 8), round(lon, 8)) + hash <- openssl::sha256(latlon_str) + + # Extracting first 10 characters of hash as a UID + uid <- substr(as.character(hash), 1, 10) return(uid) } diff --git a/docker/depends/pecan_package_dependencies.csv b/docker/depends/pecan_package_dependencies.csv index fe66050ff8c..efbe256d11a 100644 --- a/docker/depends/pecan_package_dependencies.csv +++ b/docker/depends/pecan_package_dependencies.csv @@ -241,6 +241,7 @@ "neonUtilities","*","modules/data.land","Imports",FALSE "nimble","*","modules/assim.sequential","Imports",FALSE "nneo","*","modules/data.atmosphere","Imports",FALSE +"openssl","*","base/db","Suggests",FALSE "optparse","*","base/settings","Imports",FALSE "parallel","*","modules/assim.batch","Imports",FALSE "parallel","*","modules/data.atmosphere","Suggests",FALSE