split out pagination function

R/client.R

@@ -57,29 +57,14 @@ get_measurements_for_location <- function(
     )
 
     query_params <- modifyList(query_params, parsed_args)
-    api_response <- run_query(
-        endpoint_name = "measurements",
-        query_params = query_params
-    )
-
-    measures_list <- list()
-
-    measures_list[[1]] <- get_measures(api_response)
-    i <- 1
 
-    while (length(api_response[[2]]) == max_observations) {
-        query_params[["page"]] <- as.character(i + 1)
-
-        api_response <- run_query(
-            endpoint_name = "measurements",
-            query_params = query_params
-        )
-
-        measures_list[[i + 1]] <- get_measures(api_response)
+    collate_paginated_output(
+        endpoint = "measurements",
+        query_params = query_params,
+        max_pbservations = max_observations,
+        response_parser = get_measures
+    )
 
-        i <- i+1
-    }
+}
 
-    do.call(rbind, measures_list)
 
-}

R/data_processing.R

@@ -50,3 +50,63 @@ get_measures <- function(req_cont) {
 
 }
 
+#' Collate paginated output for a given query
+#' 
+#' Collates output for a given query, where the query must be called multiple times 
+#' due to pagination in the API. Assumes that there is a parser for the JSON response
+#' from the API.
+#' 
+#' @param endpoint Endpoint of the API
+#' @param query_params List of parameters to pass to the API, values must be strings
+#' @param pagination_size The size of the response returned (the page size). Should be equal to the value in the query_params
+#' @param response_parser A callable that parses the response into something that can be concatenated
+#' 
+#' @examples 
+#' 
+#' # Construct a query
+#' endpoint <- "measurements"
+#' query_params <- list(
+#'      location = "Melbourne", 
+#'      date_from="2024-05-24", 
+#'      date_to="2024-05-25",
+#'      limit="100"
+#' )
+#' 
+#' # Paginate over results
+#' collate_paginated_output(
+#'  endpoint = endpoint,
+#'  query_params = query_params,
+#'  pagination_size = 100,
+#'  response_parser = get_measures
+#' )
+#' 
+
+collate_paginated_output <- function(
+    endpoint,
+    query_params,
+    pagination_size,
+    response_parser
+) {
+
+    output_list <- list()
+
+    api_response <- run_query(endpoint, query_params)
+
+    output_list[[1]] <- response_parser(api_response)
+    i <- 1
+
+    while (length(api_response[[2]]) == pagination_size){
+        query_params[["page"]] <- as.character(i + 1)
+
+        api_response <- run_query(
+            endpoint_name = endpoint,
+            query_params = query_params,
+        )
+
+        output_list[[i + 1]] <- response_parser(api_response)
+
+        i <- i + 1
+    }
+
+    do.call(rbind, output_list)
+}