From 6ebcdbddd2a1afec72732013c8177118af66bacf Mon Sep 17 00:00:00 2001 From: Arthur Prevot Date: Sun, 13 Oct 2024 22:27:28 +0200 Subject: [PATCH] cleaning --- yaetos/etl_utils.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/yaetos/etl_utils.py b/yaetos/etl_utils.py index aa75c6f..7ac678f 100755 --- a/yaetos/etl_utils.py +++ b/yaetos/etl_utils.py @@ -358,15 +358,7 @@ def load_input(self, input_name): path = self.jargs.inputs[input_name]['path'] path = path.replace('s3://', 's3a://') if 'dev_local' in self.jargs.mode.split(',') else path logger.info("Input '{}' to be loaded from files '{}'.".format(input_name, path)) - - # # Get base_path. TODO: remove section (and all name_base_in_param and name_base_out_param) now that it is done with replace_placeholders - # if self.jargs.merged_args.get('name_base_in_param'): - # base_path = self.jargs.merged_args[self.jargs.merged_args.get('name_base_in_param')] - # path = path.replace('{' + self.jargs.merged_args.get('name_base_in_param') + '}', '{{base_path}}') - # else: - # base_path = self.jargs.merged_args['base_path'] base_path = self.jargs.merged_args['base_path'] - path = Path_Handler(path, base_path, self.jargs.merged_args.get('root_path')).expand_latest() self.jargs.inputs[input_name]['path_expanded'] = path @@ -500,14 +492,7 @@ def expand_params(jargs, **kwargs): def expand_input_path(self, path, **kwargs): # Function call isolated to be overridable. - # # Get base_path. TODO: centralize - # if self.jargs.merged_args.get('name_base_in_param'): - # base_path = self.jargs.merged_args[self.jargs.merged_args.get('name_base_in_param')] - # path = path.replace('{' + self.jargs.merged_args.get('name_base_in_param') + '}', '{{base_path}}') - # else: - # base_path = self.jargs.merged_args['base_path'] base_path = self.jargs.merged_args['base_path'] - return Path_Handler(path, base_path, self.jargs.merged_args.get('root_path')).expand_latest() def expand_output_path(self, path, now_dt, **kwargs): @@ -1014,13 +999,6 @@ def update_args(self, args, loaded_inputs): if args.get('output'): args['output']['type'] = args.pop('output.type', None) or args['output'].get('type', 'none') if args.get('spark_app_args'): # hack to have scala sample job working. TODO: remove hardcoded case when made more generic - - # Get base_path. TODO: remove section (and all name_base_in_param and name_base_out_param) now that it is done with replace_placeholders - # if args.get('name_base_in_param'): # TODO: check if requires name_base_in_param or name_base_out_param - # base_path = args[args.get('name_base_in_param')] - # args['spark_app_args'] = args['spark_app_args'].replace('{' + self.jargs.merged_args.get('name_base_in_param') + '}', '{{base_path}}') - # else: - # base_path = args.get('base_path') base_path = args.get('base_path') args['spark_app_args'] = Path_Handler(args['spark_app_args'], base_path, args.get('root_path')).path # TODO: remove root_path since it is now done with replace_placeholders