Merge pull request #435 from orichters/conv2005-17

allow renaming old piam_variable automatically, fix remaining transport+CCS issues

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '80637090'
+ValidationKey: '8245510'
 AutocreateReadme: yes
 allowLinterWarnings: no
 AddInReadme: tutorial.md

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'piamInterfaces: Project specific interfaces to REMIND / MAgPIE'
-version: 0.40.10
-date-released: '2025-01-21'
+version: 0.41.0
+date-released: '2025-01-23'
 abstract: Project specific interfaces to REMIND / MAgPIE.
 authors:
 - family-names: Benke

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: piamInterfaces
 Title: Project specific interfaces to REMIND / MAgPIE
-Version: 0.40.10
-Date: 2025-01-21
+Version: 0.41.0
+Date: 2025-01-23
 Authors@R: c(
     person("Falk", "Benke", , "[email protected]", role = c("aut", "cre")),
     person("Oliver", "Richters", role = "aut")

NAMESPACE

@@ -29,6 +29,7 @@ export(priceIndicesFix)
 export(priceIndicesIIASA)
 export(readMifs)
 export(removePlus)
+export(renameOldInMappings)
 export(renameOldVariables)
 export(sumNamesWithFactors)
 export(summationsNames)

R/areUnitsIdentical.R

@@ -8,33 +8,30 @@
 #' @export
 areUnitsIdentical <- function(vec1, vec2 = NULL) {
   if (is.null(vec2)) vec2 <- head(vec1, n = 1)
-  # add abbreviations here that are used in the units
+  # add abbreviations here that are used in the units.
   abbreviations <- list(
-    "bn" = "billion",
+    "G" = "billion ?|bn ?",
+    "M" = "million ?|Million ?|mio ?",
     "US" = "US\\$|USD_|USD|US_",
     "US05" = "US2005",
     "US10" = "US2010",
     "EUR" = "EUR_",
     "yr" = "year",
-    "mio" = "million|Million",
     "CO2" = "CO2e|CO2eq|CO2-equiv",
+    "CF4" = "CF4-equiv",
     "%" = "Percentage|Percent|percent",
-    "cap" = "capita"
+    "cap" = "capita",
+    "/" = " per "
   )
   # only add units that actually have the same meaning, just different spelling
   identicalUnits <- list(
     c("\u00B0C", "\u00C2\u00B0C", "K"),
-    c("Gtkm/yr", "bn tkm/yr"),
-    c("Gpkm/yr", "bn pkm/yr"),
     c("kcal/cap/day", "kcal/cap/d"),
-    c("kt CF4/yr", "kt CF4-equiv/yr"),
     c("ktU/yr", "kt U/yr"),
     c("km\u00b3", "km3"),
     c("mio", "mio people"),
-    c("million t DM/yr", "Mt DM/yr"),
     c("Mt Nr/yr", "Tg N/yr"),
     c("Mt NO2/yr", "Mt NOX/yr"),
-    c("Nr/Nr", "Nr per Nr"),
     c("unitless", "", "-", "1", "index", NA),
     c("W/m2", "W/m^2"),
     c("million vehicles", "million veh"),
@@ -45,8 +42,6 @@ areUnitsIdentical <- function(vec1, vec2 = NULL) {
     c("t DM/ha", "t DM/ha/yr", "dm t/ha"),
     # for 'Water|Environmental flow violation volume'
     c("km3/yr", "km3"),
-    c("Mm3/yr", "million m3/yr"),
-    c("Mha/yr", "million ha/yr"),
     c("mha per yr", "million ha/yr"),
   NULL)
 
@@ -58,7 +53,7 @@ areUnitsIdentical <- function(vec1, vec2 = NULL) {
     return(unit)
   }
 
-  # apply abbreviations
+  # apply abbreviations to inputs and identicalUnits
   vec1 <- .abbreviateUnit(vec1)
   vec2 <- .abbreviateUnit(vec2)
   identicalUnits <- lapply(identicalUnits, .abbreviateUnit)

R/checkUnitFactor.R

@@ -44,6 +44,7 @@ checkUnitFactor <- function(template, logFile = NULL, failOnUnitMismatch = TRUE)
                           c("1000", "G", "T"),
                           c("1000", "M", "G"),
                           c("1000", "k", "M"),
+                          c("0.001", "/k", "/M"),
                           # conversion factors taken from ECEMF Model Comparison Protocol, DOI:10.5281/zenodo.6811317
                           c("1.12", "US$2010", "US$2005"),
                           c("1.12", "US$2010", "US$05"),
@@ -86,14 +87,15 @@ checkUnitFactor <- function(template, logFile = NULL, failOnUnitMismatch = TRUE)
   template$piam_factor[is.na(template$piam_factor)] <- 1
   success <- areUnitsIdentical(template$piam_unit, template$unit) & template$piam_factor %in% c(1, -1)
   success <- success | is.na(template$piam_variable)
-  ignore <-  c(paste0("Emi|CO2|Energy|Waste", c("", "|Feedstocks unknown fate", "|Plastics Incineration")),
-               "Emi|CO2|Gross|Energy|Waste")
+  ignore <- NULL # for temporary deletions, for example if a project template is wrong
   success <- success | removePlus(as.character(template$piam_variable)) %in% ignore
 
   firsterror <- TRUE
   for (sc in scaleConversion) {
     fails <- template %>%
+               # check whether substitution implies identical units
                mutate(matches = .data$piam_unit == gsub(sc[[2]], sc[[3]], .data$unit, fixed = TRUE)) %>%
+               # check whether any substitution has actually taken place (excludes all other where piam_unit = unit)
                mutate(matches = .data$matches & grepl(sc[[2]], .data$unit, fixed = TRUE)) %>%
                mutate(matches = .data$matches & ! grepl(paste0("/", sc[[2]]), .data$unit, fixed = TRUE)) %>%
                mutate(failed  = ! .data$piam_factor %in% c(sc[[1]], paste0("-", sc[[1]])))

R/checkVarNames.R

@@ -1,21 +1,23 @@
 #' checkVariablesNames checks reporting and mappings on inconsistency in variable names
 #'
-#' Pass a vector of variable names (including the units if withunits=TRUE).
+#' Pass a vector of variable names (including the units if withunits=TRUE) or
+#' a quitte object.
 #' Get warnings if inconsistencies are found for the reporting
 #' @param vars vector with variable names (and units such as "PE (EJ)")
 #' @param withunits should the var vector contain units in paranthesis?
+#' @return vector with all variables that show issues
 #' @author Oliver Richters
 #' @importFrom quitte is.quitte
 #' @export
 checkVarNames <- function(vars, withunits = TRUE) {
-  if (is.quitte(vars)) vars <- paste0(vars$variable, if (withunits) paste0(" (", vars$unit, ")"))
+  if (is.quitte(vars)) vars <- unique(paste0(vars$variable, if (withunits) paste0(" (", vars$unit, ")")))
+
   barspace <- unique(grep("[\\| ]{2}|^[\\| ]|[\\| ]$", vars, value = TRUE))
   if (length(barspace) > 0) {
     warning("These variable names have wrong bars and spaces: ", paste(barspace, collapse = ", "))
   }
 
   naVar <- unique(grep("[\\|\\( ]NA[\\|\\) ]|^NA", vars, value = TRUE))
-
   if (length(naVar) > 0) {
     warning("These variables and units contain NA: ", paste(naVar, collapse = ", "))
   }
@@ -28,5 +30,5 @@ checkVarNames <- function(vars, withunits = TRUE) {
     }
   }
 
-  return(c(barspace, naVar, noUnit))
+  return(sort(unique(c(barspace, naVar, noUnit))))
 }

R/getMappingVariables.R

@@ -7,20 +7,21 @@
 #'        such as c('AR6', 'AR6_NGFS') or 'mapping.csv'. Use TRUE for all mappings.
 #' @param sources model abbreviation(s) used in 'source' column.
 #'        R = REMIND, M = MAgPIE, T = EDGE-T, B = Brick, C = Climate/MAGICC, TRUE = all
+#' @param addunit boolean whether units should be appended as 'Variable (Unit)'
 #' @importFrom dplyr %>% select filter mutate
 #' @importFrom rlang .data
 #' @importFrom tidyselect any_of
 #' @examples
 #' getMappingVariables("AR6", "RT")
 #' @export
-getMappingVariables <- function(project = TRUE, sources = TRUE) {
+getMappingVariables <- function(project = TRUE, sources = TRUE, addunit = TRUE) {
   if (isTRUE(project)) project <- names(mappingNames())
   mapping <- NULL
   for (p in project) {
     mapping <- getMapping(p) %>%
       select(any_of(c("piam_variable", "piam_unit", "source"))) %>%
       filter(! is.na(.data$piam_variable)) %>%
-      mutate("modelvars" = paste0(.data$piam_variable, " (", .data$piam_unit, ")")) %>%
+      mutate("modelvars" = paste0(.data$piam_variable, if (isTRUE(addunit)) paste0(" (", .data$piam_unit, ")"))) %>%
       rbind(mapping)
   }
   if (! isTRUE(sources)) {

R/renameOldInMappings.R

@@ -0,0 +1,53 @@
+#' renameOldInMappings
+#'
+#' Loads "renamed_piam_variables.csv" and replaces all instances of old_name
+#' in the piam_variable columns of all mapping files
+#' Handle with care, better commit your changes before using it
+#' Best run this in your piamInterfaces folder:
+#' Rscript -e 'devtools::load_all(); renameOldInMappings()'
+#'
+#' @md
+#' @author Oliver Richters
+#' @param folder piamInterfaces folder
+#' @importFrom piamutils deletePlus
+#' @importFrom stringr str_split
+#' @export
+renameOldInMappings <- function(folder = ".") {
+
+  # find out where the csvfile is exactly and use that as a starting point
+  csvfile <- "renamed_piam_variables.csv"
+  csvfiles <- normalizePath(file.path(folder, c(".", "..", "inst", "../inst"), csvfile), mustWork = FALSE)
+  folder <- head(dirname(csvfiles[file.exists(csvfiles)]), 1)
+  if (is.null(folder)) stop(csvfile, " not found, specify 'folder' with such file.")
+
+  # search files locally to avoid overwriting anything in renv or so
+  mappingfiles <- list.files(file.path(folder, "mappings"), pattern = "mapping_.*\\.csv", full.names = TRUE)
+
+  # collect all variables and expand renamed_piam_variables based on it
+  vars <- getMappingVariables(addunit = FALSE)
+  csvdata <- getExpandRenamedVariables(deletePlus(vars))
+
+  # loop over mapping files
+  for (m in mappingfiles) {
+    mapping <- readLines(m)
+    splitted <- str_split(mapping, ";")
+    # col is number of piam_variable column
+    col <- which(splitted[[1]] == "piam_variable")
+    for (i in seq_along(splitted)) {
+      if (length(splitted[[i]] >= col)) {
+        replaceid <- which(csvdata$old_name == deletePlus(splitted[[i]][col]))
+        if (length(replaceid) == 0) next
+        replaceby <- unique(csvdata$piam_variable[replaceid])
+        if (length(replaceby) == 1) {
+          message("In ", basename(m), ", replace: ", splitted[[i]][col], " -> ", replaceby)
+          splitted[[i]][col] <- replaceby
+        } else if (length(replaceby) > 1) {
+          stop("For ", splitted[[i]][col], ", more than one replacement variable found. ",
+               "Please correct renamed_piam_variables.csv: ",
+               paste(replaceby, collapse = ", "))
+        }
+      }
+    }
+    writeLines(unlist(lapply(splitted, paste0, collapse = ";")), con = m)
+  }
+}

R/renameOldVariables.R

@@ -5,7 +5,7 @@
 #' @param mifdata quitte object or filename of mif file
 #' @param variables the list of requested variables
 #' @param logFile filename of file for logging
-#' @importFrom dplyr filter
+#' @importFrom dplyr filter distinct
 #' @importFrom piamutils deletePlus
 #' @importFrom rlang .data
 #' @importFrom tibble as_tibble
@@ -42,6 +42,11 @@ getExpandRenamedVariables <- function(variables) {
            old_name = deletePlus(.data$old_name))
   variables <- deletePlus(variables)
 
+  duplicates <- csvdata$old_name[duplicated(csvdata$old_name)]
+  if (length(duplicates) > 0) {
+    stop("Duplicates in 'old_name' found: ", paste(duplicates, collapse = ", "))
+  }
+
   csvdataNew <- NULL
   for (i in seq_len(nrow(csvdata))) {
     # if both end with *, replace by options taken from 'variables'
@@ -57,5 +62,14 @@ getExpandRenamedVariables <- function(variables) {
       csvdataNew <- rbind(csvdataNew, csvdata[i, ])
     }
   }
-  return(filter(csvdataNew, .data$old_name %in% variables))
+  # remove duplicated lines with distinct
+  csvdataNew <- distinct(filter(csvdataNew, .data$old_name %in% variables))
+
+  # check for remaining duplicates in old_name
+  duplicates <- csvdata$old_name[duplicated(csvdata$old_name)]
+  if (length(duplicates) > 0) {
+    stop("Duplicates in 'old_name' found after expansion: ", paste(duplicates, collapse = ", "))
+  }
+
+  return(csvdataNew)
 }

README.md

@@ -1,6 +1,6 @@
 # Project specific interfaces to REMIND / MAgPIE
 
-R package **piamInterfaces**, version **0.40.10**
+R package **piamInterfaces**, version **0.41.0**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/piamInterfaces)](https://cran.r-project.org/package=piamInterfaces) [![R build status](https://github.com/pik-piam/piamInterfaces/workflows/check/badge.svg)](https://github.com/pik-piam/piamInterfaces/actions) [![codecov](https://codecov.io/gh/pik-piam/piamInterfaces/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/piamInterfaces) [![r-universe](https://pik-piam.r-universe.dev/badges/piamInterfaces)](https://pik-piam.r-universe.dev/builds)
 
@@ -44,6 +44,8 @@ Additionally, some mappings use those columns:
 - `interpolation`: sets the interpolation method for the `variable` (i.e. not `piam_variable`) (currently only supports `linear`). When set to `linear`, adds yearly values between 2005 and 2100 through linear interpolation for the selected output variables.
 - `weight`: calculates a weighted average of multiple entries of `piam_variable`. Provide a different `piam_variable` in this column, and `generateIIASASubmission()` will split the data on the rows which contain weight pointers, resolve these weights into numerical values (via a join operation between the submission and the input data) and then modify the value based on the weighting. This takes place in the private .resolveWeights method.
 
+### Editing a mapping
+
 To edit a mapping in `R`, use:
 ```
 mappingdata <- getMapping("AR6")
@@ -68,6 +70,23 @@ remind2::compareScenConf(fileList = c("oldfile.csv", "mappingfile.csv"), row.nam
 ```
 On the PIK cluster, you can run `comparescenconf mapping_AR6.csv` in the `inst/mappings` folder and it will compare to a recent `master` version.
 
+### Renaming a piam_variable
+
+If a variable used as `piam_variable` has to be renamed, please add it with its `old_name` to [`inst/renamed_piam_variables.csv`](./inst/renamed_piam_variables.csv).
+Like this, if someone arrives with a dataset that contains the old name but not the new, [`renameOldVariables()`](./R/renameOldVariables.R) makes sure the data is automatically adjusted.
+[`test-renameOldVariables.R`](./tests/testthat/test-renameOldVariables.R) enforces that the old variable name is not used anywhere in the mappings.
+To adjust the mappings automatically, make sure you commit the current state to be able to reset its results, and then run `Rscript -e "devtools::load_all(); renameOldInMappings()"`.
+Check the `diff` carefully, for example using `comparescenconf`, see above.
+
+### piam_factor and unit checks
+
+While running the tests, an extensive check of the compatibility of `piam_unit`, `unit` and `piam_factor` is performed.
+It helps to find mismatches, for example mapping `Mt` to `Gt` with a factor of `0.001` or mapping `US$2005` to `US$2017` without accounting for inflation.
+These checks are performed using [`checkUnitFactor()`](./R/checkUnitFactor.R).
+It first calls [`areUnitsIdentical()`](./R/areUnitsIdentical.R) where a number of identical units are specified (such as `Mt CO2` = `Mt CO2eq`, where `piam_factor` is 1).
+Then, it compares a list of accepted factors against the templates.
+In case your tests fails, carefully check whether the `piam_factor` is correct, and if so, add it to one of the functions.
+
 ### Creating a new mapping
 
 Since templates contain between several hundreds and a few thousand variables, relying on existing mappings can save substantial amounts of work compared to setting up a new mapping from scratch. Since the template itself is most likely built based on earlier templates from other projects, chances are good that existing mappings already provide parts of the required new mapping. Using `R`, we describe a simple way to create a new mapping `mapping_NEW.csv` based on existing mappings. 
@@ -124,15 +143,17 @@ In case of questions / problems please contact Falk Benke <[email protected]>
 
 To cite package **piamInterfaces** in publications use:
 
-Benke F, Richters O (2025). "piamInterfaces: Project specific interfaces to REMIND / MAgPIE - Version 0.40.10."
+Benke F, Richters O (2025). "piamInterfaces: Project specific interfaces to REMIND / MAgPIE." Version: 0.41.0, <https://github.com/pik-piam/piamInterfaces>.
 
 A BibTeX entry for LaTeX users is
 
  ```latex
 @Misc{,
-  title = {piamInterfaces: Project specific interfaces to REMIND / MAgPIE - Version 0.40.10},
+  title = {piamInterfaces: Project specific interfaces to REMIND / MAgPIE},
   author = {Falk Benke and Oliver Richters},
-  date = {2025-01-21},
+  date = {2025-01-23},
   year = {2025},
+  url = {https://github.com/pik-piam/piamInterfaces},
+  note = {Version: 0.41.0},
 }
 ```