From 2660c029e6a54736d5216ff68bcad1fc3bb8b387 Mon Sep 17 00:00:00 2001 From: Alex Moreno Date: Mon, 16 Dec 2024 10:49:50 -0600 Subject: [PATCH] [Enabler] [module_utils/copy.py] Replace calls to cp for dcp's Python API (#1831) * Replace calls to cp for dcp * Replace copy functions in copy.py * Replace copy calls in modules * Fix typo * Fix return statement * Add changelog fragment * Fix sanity issues --- .../1831-replace-copy-commands-copy-util.yml | 4 + plugins/module_utils/copy.py | 225 +++--------------- plugins/module_utils/encode.py | 12 +- plugins/modules/zos_copy.py | 7 +- plugins/modules/zos_mount.py | 7 +- 5 files changed, 43 insertions(+), 212 deletions(-) create mode 100644 changelogs/fragments/1831-replace-copy-commands-copy-util.yml diff --git a/changelogs/fragments/1831-replace-copy-commands-copy-util.yml b/changelogs/fragments/1831-replace-copy-commands-copy-util.yml new file mode 100644 index 000000000..f6e3c75f2 --- /dev/null +++ b/changelogs/fragments/1831-replace-copy-commands-copy-util.yml @@ -0,0 +1,4 @@ +trivial: + - module_utils/copy.py - Replaced calls to cp with calls to ZOAU's + datasets' Python API. + (https://github.com/ansible-collections/ibm_zos_core/pull/1831) diff --git a/plugins/module_utils/copy.py b/plugins/module_utils/copy.py index 5336a90d7..ffb21631c 100644 --- a/plugins/module_utils/copy.py +++ b/plugins/module_utils/copy.py @@ -29,13 +29,12 @@ from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import \ ZOAUImportError -from shlex import quote - try: - from zoautil_py import datasets, gdgs + from zoautil_py import datasets, gdgs, exceptions as zoau_exceptions except Exception: datasets = ZOAUImportError(traceback.format_exc()) gdgs = ZOAUImportError(traceback.format_exc()) + zoau_exceptions = ZOAUImportError(traceback.format_exc()) REPRO = """ REPRO INDATASET({}) - OUTDATASET({}) REPLACE """ @@ -79,146 +78,51 @@ def _validate_path(path): return parsed_args.get("path") -def copy_uss2mvs(src, dest, ds_type, is_binary=False): - """Copy uss a file or path to an MVS data set. +def copy_uss_mvs(src, dest, is_binary=False): + """Wrapper function for datasets.copy that handles possible + exceptions that may occur. Parameters ---------- src : str - The uss file or path to be copied. + Source dataset or file. dest : str - The destination MVS data set, it must be a PS or PDS(E). - ds_type : str - The dsorg of the dest. + Destination dataset name or file. Keyword Parameters ------------------ is_binary : bool - Whether the file to be copied contains binary data. + Whether to perform a binary copy. Returns ------- - bool - The return code after the copy command executed successfully. - str - The stdout after the copy command executed successfully. - str - The stderr after the copy command executed successfully. + tuple + Tuple containing return code, standard output and standard + error from the datasets API. Raises ------ - USSCmdExecError - When any exception is raised during the conversion. + ZOAUImportError + When there's an issue calling the datasets API. """ - module = AnsibleModuleHelper(argument_spec={}) - src = _validate_path(src) - dest = _validate_data_set_name(dest) - if ds_type == "PO": - cp_uss2mvs = "cp -CM -F rec {0} \"//'{1}'\"".format(quote(src), dest) - else: - cp_uss2mvs = "cp -F rec {0} \"//'{1}'\"".format(quote(src), dest) - if is_binary: - cp_uss2mvs = cp_uss2mvs.replace("rec", "bin", 1) - rc, out, err = module.run_command(cp_uss2mvs, errors='replace') - if rc: - raise USSCmdExecError(cp_uss2mvs, rc, out, err) - return rc, out, err - - -def copy_ps2uss(src, dest, is_binary=False): - """Copy a PS data set to a uss file. - - Parameters - ---------- - src : str - The MVS data set to be copied, it must be a PS data set - or a PDS(E) member. - dest : str - The destination uss file. - - Keyword Parameters - ------------------ - is_binary : bool - Whether the file to be copied contains binary data. - - Returns - ------- - bool - The return code after the copy command executed successfully. - str - The stdout after the copy command executed successfully. - str - The stderr after the copy command executed successfully. + copy_args = { + "options": "" + } - Raises - ------ - USSCmdExecError - When any exception is raised during the conversion. - """ - module = AnsibleModuleHelper(argument_spec={}) - src = _validate_data_set_name(src) - dest = _validate_path(dest) - cp_ps2uss = "cp -F rec \"//'{0}'\" {1}".format(src, quote(dest)) if is_binary: - cp_ps2uss = cp_ps2uss.replace("rec", "bin", 1) - rc, out, err = module.run_command(cp_ps2uss, errors='replace') - if rc: - raise USSCmdExecError(cp_ps2uss, rc, out, err) - return rc, out, err - - -def copy_pds2uss(src, dest, is_binary=False, asa_text=False): - """Copy the whole PDS(E) to a uss path. - - Parameters - ---------- - src : str - The MVS data set to be copied, it must be a PDS(E) data set. - dest : str - The destination uss path. - - Keyword Parameters - ------------------ - is_binary : bool - Whether the file to be copied contains binary data. - asa_text : bool - Whether the file to be copied contains ASA control - characters. - - Returns - ------- - bool - The return code after the USS command executed successfully. - str - The stdout after the USS command executed successfully. - str - The stderr after the USS command executed successfully. - - Raises - ------ - USSCmdExecError - When any exception is raised during the conversion. - """ - module = AnsibleModuleHelper(argument_spec={}) - src = _validate_data_set_name(src) - dest = _validate_path(dest) - - cp_pds2uss = "cp -U -F rec \"//'{0}'\" {1}".format(src, quote(dest)) - - # When dealing with ASA control chars, each record follows a - # different format than what '-F rec' means, so we remove it - # to allow the system to leave the control chars in the - # destination. - if asa_text: - cp_pds2uss = cp_pds2uss.replace("-F rec", "", 1) - elif is_binary: - cp_pds2uss = cp_pds2uss.replace("rec", "bin", 1) + copy_args["options"] = "-B" - rc, out, err = module.run_command(cp_pds2uss, errors='replace') - if rc: - raise USSCmdExecError(cp_pds2uss, rc, out, err) + try: + datasets.copy(source=src, target=dest, **copy_args) + except zoau_exceptions.ZOAUException as copy_exception: + # Returning the exception content instead of raising it + # since a lot of code that uses this function expects it + # so they can decide what to do in case of an error. + return copy_exception.response.rc, \ + copy_exception.response.stdout_response, \ + copy_exception.response.stderr_response - return rc, out, err + return 0, "", "" def copy_gdg2uss(src, dest, is_binary=False, asa_text=False): @@ -264,81 +168,6 @@ def copy_gdg2uss(src, dest, is_binary=False, asa_text=False): return True -def copy_uss2uss_binary(src, dest): - """Copy a USS file to a USS location in binary mode. - - Parameters - ---------- - src : str - The source USS path. - dest : str - The destination USS path. - - Returns - ------- - bool - The return code after the USS command executed successfully. - str - The stdout after the USS command executed successfully. - str - The stderr after the USS command executed successfully. - - Raises - ------ - USSCmdExecError - When any exception is raised during the conversion. - """ - module = AnsibleModuleHelper(argument_spec={}) - src = _validate_path(src) - dest = _validate_path(dest) - cp_uss2uss = "cp -F bin {0} {1}".format(quote(src), quote(dest)) - rc, out, err = module.run_command(cp_uss2uss, errors='replace') - if rc: - raise USSCmdExecError(cp_uss2uss, rc, out, err) - return rc, out, err - - -def copy_mvs2mvs(src, dest, is_binary=False): - """Copy an MVS source to MVS target. - - Parameters - ---------- - src : str - Name of source data set. - dest : str - Name of destination data set. - - Keyword Parameters - ------------------ - is_binary : bool - Whether the data set to be copied contains binary data. - - Returns - ------- - bool - The return code after the USS command executed successfully. - str - The stdout after the USS command executed successfully. - str - The stderr after the USS command executed successfully. - - Raises - ------ - USSCmdExecError - When any exception is raised during the conversion. - """ - module = AnsibleModuleHelper(argument_spec={}) - src = _validate_data_set_name(src) - dest = _validate_data_set_name(dest) - cp_mvs2mvs = "cp -F rec \"//'{0}'\" \"//'{1}'\"".format(src, dest) - if is_binary: - cp_mvs2mvs = cp_mvs2mvs.replace("rec", "bin", 1) - rc, out, err = module.run_command(cp_mvs2mvs, errors='replace') - if rc: - raise USSCmdExecError(cp_mvs2mvs, rc, out, err) - return rc, out, err - - def copy_vsam_ps(src, dest, tmphlq=None): """Copy a VSAM(KSDS) data set to a PS data set vise versa. diff --git a/plugins/module_utils/encode.py b/plugins/module_utils/encode.py index 9bdac056a..4530c730d 100644 --- a/plugins/module_utils/encode.py +++ b/plugins/module_utils/encode.py @@ -510,10 +510,10 @@ def mvs_convert_encoding( if src_type == "PS": temp_src_fo = NamedTemporaryFile() temp_src = temp_src_fo.name - rc, out, err = copy.copy_ps2uss(src, temp_src) + rc, out, err = copy.copy_uss_mvs(src, temp_src) if src_type == "PO": temp_src = mkdtemp() - rc, out, err = copy.copy_pds2uss(src, temp_src) + rc, out, err = copy.copy_uss_mvs(src, temp_src) if src_type == "KSDS": reclen, space_u = self.listdsi_data_set(src.upper(), tmphlq=tmphlq) # RDW takes the first 4 bytes in the VB format, hence we need to add an extra buffer to the vsam max recl. @@ -522,7 +522,7 @@ def mvs_convert_encoding( rc, out, err = copy.copy_vsam_ps(src.upper(), temp_ps, tmphlq=tmphlq) temp_src_fo = NamedTemporaryFile() temp_src = temp_src_fo.name - rc, out, err = copy.copy_ps2uss(temp_ps, temp_src) + rc, out, err = copy.copy_uss_mvs(temp_ps, temp_src) if dest_type == "PS" or dest_type == "KSDS": temp_dest_fo = NamedTemporaryFile() temp_dest = temp_dest_fo.name @@ -538,17 +538,17 @@ def mvs_convert_encoding( # RDW takes the first 4 bytes or records in the VB format, hence we need to add an extra buffer to the vsam max recl. reclen += 4 temp_ps = self.temp_data_set(reclen, space_u) - rc, out, err = copy.copy_uss2mvs(temp_dest, temp_ps, "PS") + rc, out, err = copy.copy_uss_mvs(temp_dest, temp_ps) rc, out, err = copy.copy_vsam_ps(temp_ps, dest.upper(), tmphlq=tmphlq) convert_rc = True elif dest_type == "PO": for (dir, subdir, files) in walk(temp_dest): for file in files: temp_file = path.join(validation.validate_safe_path(dir), validation.validate_safe_path(file)) - rc, out, err = copy.copy_uss2mvs(temp_file, dest, "PO") + rc, out, err = copy.copy_uss_mvs(temp_file, dest) convert_rc = True else: - rc, out, err = copy.copy_uss2mvs(temp_dest, dest, dest_type) + rc, out, err = copy.copy_uss_mvs(temp_dest, dest) convert_rc = True except Exception: raise diff --git a/plugins/modules/zos_copy.py b/plugins/modules/zos_copy.py index 4ec3c4bb7..1fc28e3d3 100644 --- a/plugins/modules/zos_copy.py +++ b/plugins/modules/zos_copy.py @@ -1657,7 +1657,7 @@ def _copy_to_file(self, src, dest, content_copy, conv_path): new_src = conv_path or src try: if self.is_binary: - copy.copy_uss2uss_binary(new_src, dest) + copy.copy_uss_mvs(new_src, dest, is_binary=True) else: opts = dict() opts["options"] = "" @@ -1931,11 +1931,10 @@ def _mvs_copy_to_uss( stderr=response.stderr_response ) else: - copy.copy_pds2uss( + copy.copy_uss_mvs( src, dest, - is_binary=self.is_binary, - asa_text=self.asa_text + is_binary=self.is_binary ) except CopyOperationError as err: raise err diff --git a/plugins/modules/zos_mount.py b/plugins/modules/zos_mount.py index 0b1377d31..962a137d8 100644 --- a/plugins/modules/zos_mount.py +++ b/plugins/modules/zos_mount.py @@ -548,8 +548,7 @@ backup as Backup, ) from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.copy import ( - copy_ps2uss, - copy_uss2mvs, + copy_uss_mvs ) @@ -1045,7 +1044,7 @@ def run_module(module, arg_def): tmp_file_filename = tmp_file.name tmp_file.close() - copy_ps2uss(data_store, tmp_file_filename, False) + copy_uss_mvs(data_store, tmp_file_filename, is_binary=False) module.run_command( "chtag -tc ISO8859-1 " + tmp_file_filename, use_unsafe_shell=False, errors='replace' @@ -1076,7 +1075,7 @@ def run_module(module, arg_def): module.run_command( "mrm " + data_store, use_unsafe_shell=False, errors='replace' ) - copy_uss2mvs(tmp_file_filename, data_store, "", True) + copy_uss_mvs(tmp_file_filename, data_store, is_binary=True) if os.path.isfile(tmp_file_filename): os.unlink(tmp_file_filename)