diff --git a/yaetos/libs/generic_jobs/copy_raw_job.py b/yaetos/libs/generic_jobs/copy_raw_job.py index 0e08a53a..3b79db5e 100644 --- a/yaetos/libs/generic_jobs/copy_raw_job.py +++ b/yaetos/libs/generic_jobs/copy_raw_job.py @@ -1,3 +1,6 @@ +""" +Job meant to run locally to get data from AWS S3 to local. Updates require to run in cluster. +""" from yaetos.etl_utils import ETL_Base, Commandliner, get_aws_setup import os from cloudpathlib import CloudPath as CPt diff --git a/yaetos/scripts/copy/aws_config.cfg.example b/yaetos/scripts/copy/aws_config.cfg.example index 2da7b8fe..9257cf68 100644 --- a/yaetos/scripts/copy/aws_config.cfg.example +++ b/yaetos/scripts/copy/aws_config.cfg.example @@ -4,7 +4,9 @@ ec2_key_name : to_be_filled user : to_be_filled s3_region : to_be_filled ec2_subnet_id : to_be_filled -extra_security_gp : None +extra_security_gp : to_be_filled # optional +emr_ec2_role : to_be_filled # optional +emr_role : to_be_filled # optional [prod] profile_name : to_be_filled @@ -12,4 +14,6 @@ ec2_key_name : to_be_filled user : to_be_filled s3_region : to_be_filled ec2_subnet_id : to_be_filled -extra_security_gp : None +extra_security_gp : to_be_filled # optional +emr_ec2_role : to_be_filled # optional +emr_role : to_be_filled # optional