Sanity test fixes, review comments, logging & test improvements

galaxy.yml

@@ -8,7 +8,7 @@ namespace: ibm
 name: ibm_zos_cics
 
 # The collection version
-version: 2.2.0
+version: 2.1.0
 
 # Collection README file
 readme: README.md

plugins/module_utils/_data_set_utils.py

@@ -274,66 +274,54 @@ def _execute_command(command):
     module = AnsibleModuleHelper(argument_spec={})
     return module.run_command(command)
 
+
 def _submit_jcl(jcl_uss_path, job_name):
-    executions =[]
+    executions = []
     command = "jsub -f '{0}'".format(jcl_uss_path)
 
-    rc, stdout,stderr = _execute_command(command)
+    rc, stdout, stderr = _execute_command(command)
     executions.append(
-    _execution(
-        name="Submit jcl job {0} for {1}".format(jcl_uss_path, job_name),
-        rc=rc,
-        stdout=stdout,
-        stderr=stderr))
+        _execution(
+            name="Submit JCL job for {0}".format(job_name),
+            rc=rc,
+            stdout=stdout,
+            stderr=stderr))
     if rc != 0:
         raise MVSExecutionException(
-            "RC {0} when submitting jcl from {1}".format(
+            "RC {0} when submitting JCL from {1}".format(
                 rc, jcl_uss_path), executions)
     return executions
 
 
 def _get_job_output(job_id, job_name):
     executions = []
 
-    try:
-        jobs = job_output(job_id=job_id, job_name=job_name)
-
-        #There should only be one job found for the JCL submitted
-        if (len(jobs) != 1):
-            raise MVSExecutionException(
-                "Query for job status for {0} with job id {1} returned more than one result. Jobs returned: {2}".format(
-                    job_name,
-                    job_id, 
-                    jobs), executions)
-
-        executions.append(
-            _execution(
-                name="Get job output for {0}".format(job_id),
-                rc=jobs[0].get("ret_code").get("code"),
-                stdout=jobs[0].get("ret_code").get("msg", ""),
-                stderr=jobs[0].get("ret_code").get("msg_txt", "")))
-    except Exception as e:
-        raise MVSExecutionException(
-            "Query for {0} job submitted under Job ID {1} failed. An exception occured: {2}".format(
-            job_name, 
-            job_id,
-            e), executions)
-
-    try:
-        # job output fails to get the ddname content in its response. Call direct into zoau to do it
-        for i in range(len(jobs[0].get("ddnames"))):
-            dd_executions, job_stdout = _get_job_dd(job_id, jobs[0]["ddnames"][i]["ddname"])
-
-            #Put the content back in the job response object
-            jobs[0]["ddnames"][i]["content"] = job_stdout
-            executions.append(dd_executions)
-
-        return jobs[0], executions
-    except Exception as e:
+    jobs = job_output(job_id=job_id, job_name=job_name)
+
+    # There should only be one job found for the JCL submitted
+    if (len(jobs) != 1):
         raise MVSExecutionException(
-            "Could not get all job DDs for {0}. An exception occured: {1}".format(
-            job_name, 
-            e), executions)
+            "Query for job status for {0} with job ID {1} returned more than one result. Jobs returned: {2}".format(
+                job_name,
+                job_id,
+                jobs), executions)
+
+    executions.append(
+        _execution(
+            name="Get job output for {0}".format(job_id),
+            rc=jobs[0]["ret_code"].get("code"),
+            stdout=jobs[0]["ret_code"].get("msg", ""),
+            stderr=jobs[0]["ret_code"].get("msg_txt", "")))
+
+    # job output fails to get the ddname content in its response. Call direct into zoau to do it
+    for i in range(len(jobs[0].get("ddnames"))):
+        dd_executions, job_stdout = _get_job_dd(job_id, jobs[0]["ddnames"][i]["ddname"])
+
+        # Put the content back in the job response object
+        jobs[0]["ddnames"][i]["content"] = job_stdout
+        executions.append(dd_executions)
+
+    return jobs[0], executions
 
 
 def _get_job_dd(job_id, dd_name):
@@ -343,7 +331,7 @@ def _get_job_dd(job_id, dd_name):
     rc, stdout, stderr = _execute_command(command)
     executions.append(
         _execution(
-            name="Get job dd {0} output for {1}".format(dd_name, job_id),
+            name="Get job DD {0} output for {1}".format(dd_name, job_id),
             rc=rc,
             stdout=stdout,
             stderr=stderr))

plugins/module_utils/_global_catalog.py

@@ -6,15 +6,16 @@
 # FOR INTERNAL USE IN THE COLLECTION ONLY.
 
 from __future__ import (absolute_import, division, print_function)
-import os
+from os import remove
 __metaclass__ = type
 
 
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import MVSExecutionException, _execution
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import MVS_CMD_RETRY_ATTEMPTS, _submit_jcl, _get_job_output
-import tempfile
+from tempfile import NamedTemporaryFile
 from time import sleep
 
+
 def _get_value_from_line(line):  # type: (list[str]) -> str | None
     val = None
     if len(line) == 1:
@@ -55,15 +56,16 @@ def _get_catalog_records(stdout):  # type: (str) -> tuple[str | None, str | None
 
     return (autostart_override, nextstart)
 
+
 def _create_dfhrmutl_jcl(location, sdfhload, cmd=""):
     steplib_line = f"//STEPLIB  DD DSNAME={sdfhload},DISP=SHR"
     dfhgcd_line = f"//DFHGCD   DD DSNAME={location},DISP=OLD"
     # Validate line lengths
-    _validate_line_length(steplib_line, sdfhload)
-    _validate_line_length(dfhgcd_line, location)
-    
+    _validate_line_length(steplib_line, sdfhload, "cics_data_sets")
+    _validate_line_length(dfhgcd_line, location, "region_data_sets")
+
     jcl = ""
-    if (cmd ==""):
+    if (cmd == ""):
         jcl = f'''
 //DFHRMUTL JOB
 //RMUTL    EXEC PGM=DFHRMUTL,REGION=1M
@@ -86,37 +88,32 @@ def _create_dfhrmutl_jcl(location, sdfhload, cmd=""):
 '''
 
     # Create a temporary file
-    with tempfile.NamedTemporaryFile(mode='w+', delete=False) as fp:
+    with NamedTemporaryFile(mode='w+', delete=False) as fp:
         fp.write(jcl)
         fp.flush()
 
         # Get the temporary file's name
         qualified_file_path = fp.name
     return qualified_file_path
 
-def _validate_line_length(line, name):
+
+def _validate_line_length(line, name, parameter):
     """
     Validates that the JCL line does not exceed MAX_LINE_LENGTH.
     Raises ValueError if validation fails.
     """
     if len(line) > MAX_LINE_LENGTH:
-        raise ValueError(f"{name} line exceeds {MAX_LINE_LENGTH} characters: {len(line)}")
+        raise ValueError(f"Value supplied for {parameter} ({name}) is {len(line) - MAX_LINE_LENGTH} characters too long")
     return True
 
-def _validate_name_params(param):
-    if any(len(part) > MAX_NAME_LENGTH for part in param.split('.')):
-        raise ValueError(f"One or more parts of {param} exceeds MAX_NAME_LENGTH:{MAX_NAME_LENGTH}")
-    return True
 
 def _run_dfhrmutl(
         location,  # type: str
         sdfhload,  # type: str
         cmd=""  # type: str
 ):
-   # type: (...) -> tuple[list[dict[str, str| int]], tuple[str | None, str | None]] | list[dict[str, str| int]]
-    _validate_name_params(location)
-    _validate_name_params(sdfhload)
-
+    # type: (...) -> tuple[list[dict[str, str| int]], tuple[str | None, str | None]] | list[dict[str, str| int]]
+
     qualified_file_path = _create_dfhrmutl_jcl(
         location,
         sdfhload,
@@ -126,11 +123,11 @@ def _run_dfhrmutl(
     try:
         for x in range(MVS_CMD_RETRY_ATTEMPTS):
             dfhrmutl_response, jcl_executions = _execute_dfhrmutl(qualified_file_path)
-            dfhrmutl_rc = dfhrmutl_response.get("ret_code").get("code")
+            dfhrmutl_rc = dfhrmutl_response["ret_code"].get("code")
 
             allContent = []
             for ddname in dfhrmutl_response.get("ddnames"):
-                    allContent += ddname.get("content")
+                allContent += ddname.get("content")
             stdout_raw = "".join(allContent)
 
             executions.append(jcl_executions)
@@ -141,8 +138,8 @@ def _run_dfhrmutl(
                         x + 1),
                     rc=dfhrmutl_rc,
                     stdout=stdout_raw,
-                    stderr=dfhrmutl_response.get("ret_code").get("msg_txt", "")))
-            
+                    stderr=dfhrmutl_response["ret_code"].get("msg_txt", "")))
+
             if dfhrmutl_rc not in (0, 16):
                 raise MVSExecutionException(
                     "DFHRMUTL failed with RC {0}".format(
@@ -171,16 +168,16 @@ def _run_dfhrmutl(
         if cmd != "":
             return executions
     finally:
-        os.remove(qualified_file_path)
+        remove(qualified_file_path)
     return executions, _get_catalog_records(stdout_raw)
 
 
 def _execute_dfhrmutl(rmutl_jcl_path, job_name="DFHRMUTL"):
     executions = _submit_jcl(rmutl_jcl_path, job_name)
     job_id = executions[0].get("stdout").strip()
 
-    #Give RMUTL a second to run
-    sleep(1)
+    # Give RMUTL a second to run
+    sleep(JOB_SUBMIT_WAIT_TIME)
 
     job, job_executions = _get_job_output(job_id, job_name)
     executions.append(job_executions)
@@ -209,6 +206,7 @@ def _get_idcams_cmd_gcd(dataset):   # type: (dict) -> dict
     return defaults
 
 
+# IDCAMS Consts
 RECORD_COUNT_DEFAULT = 4089
 RECORD_SIZE_DEFAULT = 32760
 CONTROL_INTERVAL_SIZE_DEFAULT = 32768
@@ -217,5 +215,7 @@ def _get_idcams_cmd_gcd(dataset):   # type: (dict) -> dict
 CI_PERCENT = 10
 CA_PERCENT = 10
 SHARE_CROSSREGION = 2
+
+# RMUTL Consts
 MAX_LINE_LENGTH = 72
-MAX_NAME_LENGTH = 8
+JOB_SUBMIT_WAIT_TIME = 1

tests/sanity/ignore-2.17.txt

@@ -0,0 +1,15 @@
+plugins/modules/cmci_get.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/cmci_action.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/cmci_create.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/cmci_delete.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/cmci_update.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/global_catalog.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/local_catalog.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/local_request_queue.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/td_intrapartition.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/aux_temp_storage.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/aux_trace.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/csd.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/transaction_dump.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/region_jcl.py validate-modules:missing-gplv3-license # Licence is Apache-2.0
+plugins/modules/stop_region.py validate-modules:missing-gplv3-license # Licence is Apache-2.0

tests/unit/helpers/data_set_helper.py

@@ -419,31 +419,54 @@ def CSDUP_add_group_stdout(data_set_name):
         DFH5109 I END OF DFHCSDUP UTILITY JOB. HIGHEST RETURN CODE WAS: 0
     """.format(data_set_name)
 
+
+def get_rmutl_executions():
+    job = get_sample_job_output()
+    executions = [
+        _execution(
+            name="Submit JCL job for {0}".format(job["job_id"]),
+            rc=0,
+            stdout="",
+            stderr=""
+        ),
+        get_job_output_execution()
+    ]
+
+    for dd in job.get("ddnames"):
+        executions.append(get_job_dd_output_execution(ddname=dd["content"], job_id=job["job_id"]))
+
+    return executions
+
+
 def get_job_output_execution(rc=0):
     job = get_sample_job_output()
     return _execution(
         name="Get job output for {0}".format(job["job_id"]),
         rc=rc,
-        stdout=job.get("ret_code").get("msg"),
-        stderr=job.get("ret_code").get("msg_txt")
+        stdout=job["ret_code"]["msg"],
+        stderr=job["ret_code"]["msg_txt"]
     )
 
-def _get_job_dd_output(ddname, job_id):
+
+def get_job_dd_output(ddname, job_id):
     return (
-        _get_job_dd_output_execution(rc=0, ddname=ddname, job_id=job_id),
+        get_job_dd_output_execution(rc=0, ddname=ddname, job_id=job_id),
         RMUTL_stdout("AUTOINIT", job_id)
     )
 
-def _get_job_dd_output_execution(rc = 0, ddname="", job_id=""):
+
+def get_job_dd_output_execution(rc=0, ddname="", job_id=""):
     return _execution(
         name=JOB_DD_return_name(ddname, job_id),
         rc=rc,
         stdout=RMUTL_stdout("AUTOINIT", job_id),
         stderr="CC"
     )
 
+
 def JOB_DD_return_name(ddname, job_id):
-    return "Get job dd {0} output for {1}".format(ddname, job_id)
+    return f"Get job DD {ddname} output for {job_id}"
+
 
 def get_sample_job_output(content="", rc=0, err="CC"):
     return {
@@ -453,7 +476,7 @@ def get_sample_job_output(content="", rc=0, err="CC"):
             {
                 "byte_count": 0,
                 "content": [
-                    "{0}".format(content)
+                    content
                 ],
                 "ddname": "JESMSGLG",
                 "id": "?",
@@ -510,10 +533,11 @@ def get_sample_job_output(content="", rc=0, err="CC"):
         "system": ""
     }
 
-def _get_sample_job_output_with_content():
+
+def get_sample_job_output_with_content():
     job = get_sample_job_output()
     for dd in job.get("ddnames"):
-        dd["content"] = RMUTL_stdout("AUTOINIT", job.get("job_id"))
+        dd["content"] = RMUTL_stdout("AUTOINIT", job["job_id"])
     return job