diff --git a/jobs/generic/list_files_job.py b/jobs/generic/list_files_job.py index 554aa27b..656574c1 100755 --- a/jobs/generic/list_files_job.py +++ b/jobs/generic/list_files_job.py @@ -27,7 +27,7 @@ def transform(self, files): pattern_type = 'glob' session = get_aws_setup(self.jargs.merged_args) - + s3 = session.client('s3') if not CPt(f"s3://{path_raw_in.bucket}").exists() and self.jargs.merged_args.get('ignore_empty_bucket'): @@ -42,7 +42,7 @@ def transform(self, files): def get_filenames(self, s3, bucket_name, prefix, pattern, pattern_type): files = [] for (obj, file_name) in self.s3_iterator(s3, bucket_name, prefix, pattern, pattern_type): - files.append('s3://'+bucket_name + '/' + obj['Key']) + files.append('s3://' + bucket_name + '/' + obj['Key']) return files def s3_iterator(self, s3, bucket_name, prefix, pattern, pattern_type): diff --git a/yaetos/scripts/yaetos_cmdline.py b/yaetos/scripts/yaetos_cmdline.py index cac7b26b..a8e725a6 100644 --- a/yaetos/scripts/yaetos_cmdline.py +++ b/yaetos/scripts/yaetos_cmdline.py @@ -153,6 +153,7 @@ def setup_env(args): copyfile(Pt(f'{package_path}/libs/generic_jobs/sql_spark_job.py'), Pt(f'{cwd}/jobs/generic/sql_spark_job.py')) copyfile(Pt(f'{package_path}/libs/generic_jobs/sql_pandas_job.py'), Pt(f'{cwd}/jobs/generic/sql_pandas_job.py')) copyfile(Pt(f'{package_path}/libs/generic_jobs/compare_pandas_job.py'), Pt(f'{cwd}/jobs/generic/compare_pandas_job.py')) + copyfile(Pt(f'{package_path}/libs/generic_jobs/list_files_job.py'), Pt(f'{cwd}/jobs/generic/list_files_job.py')) # Sample jobs os.makedirs(Pt('jobs/examples/'), exist_ok=True)