diff --git a/README.md b/README.md index e57cf02..972603d 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Note: If you are having issues during installation of the Synapse CLI client, co #### Clone the repo and install required R libraries 7. Clone this repository and switch to the new project -8. Modify the parameters in [params.R](params.R) +8. Modify the parameters in [config.yml](config.yml) 9. Run [install_requirements.R](install_requirements.R) 10. Start a new R session (type `q()` in the R console) diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..c9fb3e7 --- /dev/null +++ b/config.yml @@ -0,0 +1,18 @@ +default: + default_key: "default_value" + +prod: + PARQUET_BUCKET: recover-processed-data # S3 bucket containing the post-ETL parquet datasets + PARQUET_BUCKET_BASE_KEY: main/parquet/ + PARQUET_BUCKET_EXTERNAL: recover-main-project + PARQUET_BUCKET_BASE_KEY_ARCHIVE: main/archive/ + PARQUET_BUCKET_BASE_KEY_EXTERNAL: main/parquet/ + PARQUET_FOLDER_INTERNAL: syn51406699 # Local location where parquet bucket files are synced to + AWS_PARQUET_DOWNLOAD_LOCATION: ./temp_aws_parquet + AWS_ARCHIVE_DOWNLOAD_LOCATION: ./temp_aws_archive + PARQUET_FILTERED_LOCATION: ./parquet_filtered + PARQUET_FINAL_LOCATION: ./parquet_final # Synapse locations where the S3 bucket objects are to be indexed + PARQUET_FOLDER_CURRENT: syn52506068 + PARQUET_FOLDER_ARCHIVE: syn52506069 # synID of the file view containing a list of all currently indexed S3 objects from the parquet bucket folder in Synapse + SYNAPSE_FILEVIEW_ID: syn52504776 + diff --git a/install_requirements.R b/install_requirements.R index f7227e9..b44d9a5 100644 --- a/install_requirements.R +++ b/install_requirements.R @@ -11,4 +11,5 @@ install.packages("reticulate") install.packages("synapser", repos=c("http://ran.synapse.org", "http://cran.fhcrc.org")) install.packages("synapserutils", repos=c("http://ran.synapse.org", "http://cran.fhcrc.org")) install.packages("arrow") +install.packages("config") diff --git a/sts_synindex_external.R b/sts_synindex_external.R index a43b618..da084ae 100644 --- a/sts_synindex_external.R +++ b/sts_synindex_external.R @@ -91,7 +91,7 @@ replace_equal_with_underscore <- function(directory_path) { # Setup ------------------------------------------------------------------- synapser::synLogin(authToken = Sys.getenv('SYNAPSE_AUTH_TOKEN')) -source('~/recover-parquet-external/params.R') +config::get(config = "prod") %>% list2env(envir = .GlobalEnv) # Get STS credentials for input data bucket -------------------------------