Merge pull request #8 from pranavanba/add-yml-config

RMHDR-200 Use yml config file instead of R script to store pipeline parameters

README.md

@@ -37,7 +37,7 @@ Note: If you are having issues during installation of the Synapse CLI client, co
 #### Clone the repo and install required R libraries
 
 7. Clone this repository and switch to the new project
-8. Modify the parameters in [params.R](params.R)
+8. Modify the parameters in [config.yml](config.yml)
 9. Run [install_requirements.R](install_requirements.R)
 10. Start a new R session (type `q()` in the R console)
 

config.yml

@@ -0,0 +1,18 @@
+default:
+  default_key: "default_value"
+
+prod:
+  PARQUET_BUCKET: recover-processed-data # S3 bucket containing the post-ETL parquet datasets
+  PARQUET_BUCKET_BASE_KEY: main/parquet/
+  PARQUET_BUCKET_EXTERNAL: recover-main-project
+  PARQUET_BUCKET_BASE_KEY_ARCHIVE: main/archive/
+  PARQUET_BUCKET_BASE_KEY_EXTERNAL: main/parquet/
+  PARQUET_FOLDER_INTERNAL: syn51406699 # Local location where parquet bucket files are synced to
+  AWS_PARQUET_DOWNLOAD_LOCATION: ./temp_aws_parquet
+  AWS_ARCHIVE_DOWNLOAD_LOCATION: ./temp_aws_archive
+  PARQUET_FILTERED_LOCATION: ./parquet_filtered
+  PARQUET_FINAL_LOCATION: ./parquet_final # Synapse locations where the S3 bucket objects are to be indexed
+  PARQUET_FOLDER_CURRENT: syn52506068
+  PARQUET_FOLDER_ARCHIVE: syn52506069 # synID of the file view containing a list of all currently indexed S3 objects from the parquet bucket folder in Synapse
+  SYNAPSE_FILEVIEW_ID: syn52504776
+

install_requirements.R

@@ -11,4 +11,5 @@ install.packages("reticulate")
 install.packages("synapser", repos=c("http://ran.synapse.org", "http://cran.fhcrc.org"))
 install.packages("synapserutils", repos=c("http://ran.synapse.org", "http://cran.fhcrc.org"))
 install.packages("arrow")
+install.packages("config")
 

sts_synindex_external.R

@@ -91,7 +91,7 @@ replace_equal_with_underscore <- function(directory_path) {
 
 # Setup -------------------------------------------------------------------
 synapser::synLogin(authToken = Sys.getenv('SYNAPSE_AUTH_TOKEN'))
-source('~/recover-parquet-external/params.R')
+config::get(config = "prod") %>% list2env(envir = .GlobalEnv)
 
 
 # Get STS credentials for input data bucket -------------------------------