diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..971dbc3a --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "ms-python.black-formatter", + "ms-python.isort", + "ms-python.vscode-pylance" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 5432f1ee..57c96298 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,13 +1,13 @@ { - "editor.formatOnSave": true, - "python.formatting.provider": "black", - "rust-analyzer.linkedProjects": ["./src/secrets/Cargo.toml"], "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true - } + "source.organizeImports": "explicit" + }, + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true }, "python.testing.pytestArgs": ["tests"], + "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "rust-analyzer.linkedProjects": ["./src/secrets/Cargo.toml"], } diff --git a/CHANGELOG.md b/CHANGELOG.md index e0fa88b4..98ca4a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to the Zowe Client Python SDK will be documented in this file. +## Recent Changes + +### Bug Fixes + +- Fixed `Files.download_dsn` and `Files.download_binary_dsn` failing to write contents to disk [#179](https://github.com/zowe/zowe-client-python-sdk/issues/179) +- Fixed `Files.delete_data_set` and `Files.list_dsn_members` so they encode URLs correctly +- Fixed `Files.upload_to_uss` displaying an unclosed file warning +- Fixed loading environment variables when there is no schema file in current directory + +### Enhancements + +- Added method `Files.download_uss` to download USS files to disk +- Added support to `Tso` class for loading TSO profile properties + ## `1.0.0-dev13` ### Bug Fixes diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 42bb1daf..581927ce 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -152,11 +152,9 @@ def validate_schema(self) -> None: # validate the $schema property if path_schema_json: - validate_config_json(self.jsonc, path_schema_json, cwd = self.location) + validate_config_json(self.jsonc, path_schema_json, cwd=self.location) - def schema_list( - self, - ) -> list: + def schema_list(self, cwd=None) -> list: """ Loads the schema properties in a sorted order according to the priority @@ -180,7 +178,7 @@ def schema_list( schema_json = json.load(f) elif not os.path.isabs(schema): - schema = os.path.join(self.location, schema) + schema = os.path.join(self.location or cwd, schema) with open(schema) as f: schema_json = json.load(f) else: diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 20a6b38c..7c0c78d1 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -114,9 +114,7 @@ def config_filepath(self) -> Optional[str]: return self.project_config.filepath @staticmethod - def get_env( - cfg: ConfigFile, - ) -> dict: + def get_env(cfg: ConfigFile, cwd=None) -> dict: """ Maps the env variables to the profile properties @@ -127,7 +125,7 @@ def get_env( Containing profile properties from env variables (prop: value) """ - props = cfg.schema_list() + props = cfg.schema_list(cwd) if props == []: return {} @@ -255,6 +253,7 @@ def load( profiles_merged: dict = {} cfg_name = None cfg_schema = None + cfg_schema_dir = None for cfg_layer in (self.project_user_config, self.project_config, self.global_user_config, self.global_config): if cfg_layer.profiles is None: @@ -272,6 +271,7 @@ def load( cfg_name = cfg_layer.name if not cfg_schema and cfg_layer.schema_property: cfg_schema = cfg_layer.schema_property + cfg_schema_dir = cfg_layer._location usrProject = self.project_user_config.profiles or {} project = self.project_config.profiles or {} @@ -299,7 +299,7 @@ def load( missing_secure_props.extend(profile_loaded.missing_secure_props) if override_with_env: - env_var = {**self.get_env(cfg)} + env_var = {**self.get_env(cfg, cfg_schema_dir)} if profile_type != BASE_PROFILE: profile_props = { diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py index 26f49370..efe9d539 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py @@ -12,7 +12,6 @@ import os -import shutil from zowe.core_for_zowe_sdk import SdkApi from zowe.core_for_zowe_sdk.exceptions import FileNotFound @@ -140,12 +139,8 @@ def list_dsn_members(self, dataset_name, member_pattern=None, member_start=None, additional_parms["start"] = member_start if member_pattern is not None: additional_parms["pattern"] = member_pattern - url = "{}ds/{}/member".format(self.request_endpoint, dataset_name) - separator = "?" - for k, v in additional_parms.items(): - url = "{}{}{}={}".format(url, separator, k, v) - separator = "&" - custom_args["url"] = self._encode_uri_component(url) + custom_args["params"] = additional_parms + custom_args["url"] = "{}ds/{}/member".format(self.request_endpoint, self._encode_uri_component(dataset_name)) custom_args["headers"]["X-IBM-Max-Items"] = "{}".format(limit) custom_args["headers"]["X-IBM-Attributes"] = attributes response_json = self.request_handler.perform_request("GET", custom_args) @@ -443,13 +438,13 @@ def get_dsn_content_streamed(self, dataset_name): Returns ------- - raw - A raw socket response + response + A response object from the requests library """ custom_args = self._create_custom_request_arguments() custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) - raw_response = self.request_handler.perform_streamed_request("GET", custom_args) - return raw_response + response = self.request_handler.perform_streamed_request("GET", custom_args) + return response def get_dsn_binary_content(self, dataset_name, with_prefixes=False): """ @@ -462,8 +457,8 @@ def get_dsn_binary_content(self, dataset_name, with_prefixes=False): default: False Returns ------- - bytes - The contents of the dataset with no transformation + response + A response object from the requests library """ custom_args = self._create_custom_request_arguments() custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) @@ -472,8 +467,8 @@ def get_dsn_binary_content(self, dataset_name, with_prefixes=False): custom_args["headers"]["X-IBM-Data-Type"] = "record" else: custom_args["headers"]["X-IBM-Data-Type"] = "binary" - content = self.request_handler.perform_request("GET", custom_args) - return content + response = self.request_handler.perform_request("GET", custom_args) + return response def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False): """ @@ -486,8 +481,8 @@ def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False): default: False Returns ------- - raw - The raw socket response + response + A response object from the requests library """ custom_args = self._create_custom_request_arguments() custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) @@ -496,8 +491,8 @@ def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False): custom_args["headers"]["X-IBM-Data-Type"] = "record" else: custom_args["headers"]["X-IBM-Data-Type"] = "binary" - content = self.request_handler.perform_streamed_request("GET", custom_args) - return content + response = self.request_handler.perform_streamed_request("GET", custom_args) + return response def write_to_dsn(self, dataset_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING): """Write content to an existing dataset. @@ -516,9 +511,10 @@ def write_to_dsn(self, dataset_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING def download_dsn(self, dataset_name, output_file): """Retrieve the contents of a dataset and saves it to a given file.""" - raw_response = self.get_dsn_content_streamed(dataset_name) + response = self.get_dsn_content_streamed(dataset_name) with open(output_file, "w", encoding="utf-8") as f: - shutil.copyfileobj(raw_response, f) + for chunk in response.iter_content(chunk_size=4096, decode_unicode=True): + f.write(chunk) def download_binary_dsn(self, dataset_name, output_file, with_prefixes=False): """Retrieve the contents of a binary dataset and saves it to a given file. @@ -529,15 +525,11 @@ def download_binary_dsn(self, dataset_name, output_file, with_prefixes=False): output_file:str - Name of the local file to create with_prefixes:boolean - If true, include a four big endian bytes record length prefix. The default is False - - Returns - ------- - bytes - Binary content of the dataset. """ - content = self.get_dsn_binary_content_streamed(dataset_name, with_prefixes=with_prefixes) + response = self.get_dsn_binary_content_streamed(dataset_name, with_prefixes=with_prefixes) with open(output_file, "wb") as f: - shutil.copyfileobj(content, f) + for chunk in response.iter_content(chunk_size=4096): + f.write(chunk) def upload_file_to_dsn(self, input_file, dataset_name, encoding=_ZOWE_FILES_DEFAULT_ENCODING): """Upload contents of a given file and uploads it to a dataset.""" @@ -564,21 +556,42 @@ def write_to_uss(self, filepath_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODIN def upload_file_to_uss(self, input_file, filepath_name, encoding=_ZOWE_FILES_DEFAULT_ENCODING): """Upload contents of a given file and uploads it to UNIX file""" if os.path.isfile(input_file): - in_file = open(input_file, "r", encoding="utf-8") - file_contents = in_file.read() - response_json = self.write_to_uss(filepath_name, file_contents) + with open(input_file, "r", encoding="utf-8") as in_file: + response_json = self.write_to_uss(filepath_name, in_file) else: raise FileNotFound(input_file) + def get_file_content_streamed(self, file_path, binary=False): + """Retrieve the contents of a given USS file streamed. + + Returns + ------- + response + A response object from the requests library + """ + custom_args = self._create_custom_request_arguments() + custom_args["url"] = "{}fs/{}".format(self.request_endpoint, self._encode_uri_component(file_path.lstrip("/"))) + if binary: + custom_args["headers"]["X-IBM-Data-Type"] = "binary" + response = self.request_handler.perform_streamed_request("GET", custom_args) + return response + + def download_uss(self, file_path, output_file, binary=False): + """Retrieve the contents of a USS file and saves it to a local file.""" + response = self.get_file_content_streamed(file_path, binary) + with open(output_file, "wb" if binary else "w", encoding="utf-8") as f: + for chunk in response.iter_content(chunk_size=4096, decode_unicode=not binary): + f.write(chunk) + def delete_data_set(self, dataset_name, volume=None, member_name=None): """Deletes a sequential or partitioned data.""" custom_args = self._create_custom_request_arguments() if member_name is not None: dataset_name = f"{dataset_name}({member_name})" - url = "{}ds/{}".format(self.request_endpoint, dataset_name) + url = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) if volume is not None: - url = "{}ds/-{}/{}".format(self.request_endpoint, volume, dataset_name) - custom_args["url"] = self._encode_uri_component(url) + url = "{}ds/-{}/{}".format(self.request_endpoint, volume, self._encode_uri_component(dataset_name)) + custom_args["url"] = url response_json = self.request_handler.perform_request("DELETE", custom_args, expected_code=[200, 202, 204]) return response_json diff --git a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py index 4f497ce9..f2573b6a 100644 --- a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py +++ b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py @@ -267,9 +267,8 @@ def submit_from_local_file(self, jcl_path): A JSON containing the result of the request execution """ if os.path.isfile(jcl_path): - jcl_file = open(jcl_path, "r", encoding="utf-8") - file_content = jcl_file.read() - jcl_file.close() + with open(jcl_path, "r", encoding="utf-8") as jcl_file: + file_content = jcl_file.read() return self.submit_plaintext(file_content) else: raise FileNotFoundError("Provided argument is not a file path {}".format(jcl_path)) @@ -388,32 +387,30 @@ def get_job_output_as_files(self, status, output_dir): A JSON containing the result of the request execution """ - _job_name = status["jobname"] - _job_id = status["jobid"] - _job_correlator = status["job-correlator"] - - _output_dir = os.path.join(output_dir, _job_name, _job_id) - os.makedirs(_output_dir, exist_ok=True) - _output_file = os.path.join(output_dir, _job_name, _job_id, "jcl.txt") - _data_spool_file = self.get_jcl_text(_job_correlator) - _dataset_content = _data_spool_file["response"] - _out_file = open(_output_file, "w", encoding="utf-8") - _out_file.write(_dataset_content) - _out_file.close() - - _spool = self.get_spool_files(_job_correlator) - for _spool_file in _spool: - _stepname = _spool_file["stepname"] - _ddname = _spool_file["ddname"] - _spoolfile_id = _spool_file["id"] - _output_dir = os.path.join(output_dir, _job_name, _job_id, _stepname) - os.makedirs(_output_dir, exist_ok=True) - - _output_file = os.path.join(output_dir, _job_name, _job_id, _stepname, _ddname) - _data_spool_file = self.get_spool_file_contents(_job_correlator, _spoolfile_id) - _dataset_content = _data_spool_file["response"] - _out_file = open(_output_file, "w", encoding="utf-8") - _out_file.write(_dataset_content) - _out_file.close() + job_name = status["jobname"] + job_id = status["jobid"] + job_correlator = status["job-correlator"] + + output_dir = os.path.join(output_dir, job_name, job_id) + os.makedirs(output_dir, exist_ok=True) + output_file = os.path.join(output_dir, job_name, job_id, "jcl.txt") + data_spool_file = self.get_jcl_text(job_correlator) + dataset_content = data_spool_file["response"] + with open(output_file, "w", encoding="utf-8") as out_file: + out_file.write(dataset_content) + + spool = self.get_spool_files(job_correlator) + for spool_file in spool: + stepname = spool_file["stepname"] + ddname = spool_file["ddname"] + spoolfile_id = spool_file["id"] + output_dir = os.path.join(output_dir, job_name, job_id, stepname) + os.makedirs(output_dir, exist_ok=True) + + output_file = os.path.join(output_dir, job_name, job_id, stepname, ddname) + data_spool_file = self.get_spool_file_contents(job_correlator, spoolfile_id) + dataset_content = data_spool_file["response"] + with open(output_file, "w", encoding="utf-8") as out_file: + out_file.write(dataset_content) return diff --git a/src/zos_tso/zowe/zos_tso_for_zowe_sdk/tso.py b/src/zos_tso/zowe/zos_tso_for_zowe_sdk/tso.py index 07972314..76ed950b 100644 --- a/src/zos_tso/zowe/zos_tso_for_zowe_sdk/tso.py +++ b/src/zos_tso/zowe/zos_tso_for_zowe_sdk/tso.py @@ -27,7 +27,7 @@ class Tso(SdkApi): Constant for the session not found tso message id """ - def __init__(self, connection): + def __init__(self, connection, tso_profile=None): """ Construct a Tso object. @@ -38,6 +38,7 @@ def __init__(self, connection): """ super().__init__(connection, "/zosmf/tsoApp/tso") self.session_not_found = constants["TsoSessionNotFound"] + self.tso_profile = tso_profile or {} def issue_command(self, command): """Issues a TSO command. @@ -63,13 +64,13 @@ def issue_command(self, command): def start_tso_session( self, - proc="IZUFPROC", - chset="697", - cpage="1047", - rows="204", - cols="160", - rsize="4096", - acct="DEFAULT", + proc=None, + chset=None, + cpage=None, + rows=None, + cols=None, + rsize=None, + acct=None, ): """Start a TSO session. @@ -97,13 +98,13 @@ def start_tso_session( """ custom_args = self._create_custom_request_arguments() custom_args["params"] = { - "proc": proc, - "chset": chset, - "cpage": cpage, - "rows": rows, - "cols": cols, - "rsize": rsize, - "acct": acct, + "proc": proc or self.tso_profile.get("logonProcedure", "IZUFPROC"), + "chset": chset or self.tso_profile.get("characterSet", "697"), + "cpage": cpage or self.tso_profile.get("codePage", "1047"), + "rows": rows or self.tso_profile.get("rows", "204"), + "cols": cols or self.tso_profile.get("columns", "160"), + "rsize": rsize or self.tso_profile.get("regionSize", "4096"), + "acct": acct or self.tso_profile.get("account", "DEFAULT"), } response_json = self.request_handler.perform_request("POST", custom_args) return response_json["servletKey"] diff --git a/tests/integration/fixtures/files.json b/tests/integration/fixtures/files.json index a963adc9..3c36029d 100644 --- a/tests/integration/fixtures/files.json +++ b/tests/integration/fixtures/files.json @@ -2,7 +2,10 @@ "TEST_HLQ": "ZOWE", "TEST_PDS": "ZOWE.TESTS.JCL", "TEST_MEMBER": "IEFBR14T", + "TEST_MEMBER_NEW": "TESTNEW", "TEST1_ZFS": "TEST1.ZFS", "TEST2_ZFS": "TEST2.ZFS", - "TEST_USS": "/tmp/zowe-test.txt" + "TEST_USS": "/tmp/zowe-test.txt", + "TEST_USS_MOUNT": "/tmp/zowe/mount", + "TEST_USS_NEW": "/tmp/zowe-test-new.txt" } \ No newline at end of file diff --git a/tests/integration/test_zos_console.py b/tests/integration/test_zos_console.py index 4423982b..33e8d804 100644 --- a/tests/integration/test_zos_console.py +++ b/tests/integration/test_zos_console.py @@ -10,7 +10,7 @@ class TestConsoleIntegration(unittest.TestCase): def setUp(self): """Setup fixtures for Console class.""" - test_profile = ProfileManager().load(profile_type="zosmf") + test_profile = ProfileManager(show_warnings=False).load(profile_type="zosmf") self.console = Console(test_profile) def test_console_command_time_should_return_time(self): diff --git a/tests/integration/test_zos_files.py b/tests/integration/test_zos_files.py index bba80195..14592936 100644 --- a/tests/integration/test_zos_files.py +++ b/tests/integration/test_zos_files.py @@ -9,6 +9,7 @@ FIXTURES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") FILES_FIXTURES_PATH = os.path.join(FIXTURES_PATH, "files.json") +SAMPLE_JCL_FIXTURE_PATH = os.path.join(FIXTURES_PATH, "sample.jcl") class TestFilesIntegration(unittest.TestCase): @@ -16,16 +17,18 @@ class TestFilesIntegration(unittest.TestCase): def setUp(self): """Setup fixtures for Files class.""" - test_profile = ProfileManager().load(profile_type="zosmf") + test_profile = ProfileManager(show_warnings=False).load(profile_type="zosmf") self.user_name = test_profile["user"] with open(FILES_FIXTURES_PATH, "r") as fixtures_json: self.files_fixtures = json.load(fixtures_json) self.files = Files(test_profile) self.test_member_jcl = f'{self.files_fixtures["TEST_PDS"]}({self.files_fixtures["TEST_MEMBER"]})' self.test_member_generic = f'{self.files_fixtures["TEST_PDS"]}(TEST)' + self.test_ds_upload = f'{self.files_fixtures["TEST_PDS"]}({self.files_fixtures["TEST_MEMBER_NEW"]})' + self.test_uss_upload = self.files_fixtures["TEST_USS_NEW"] self.test1_zfs_file_system = f'{self.user_name}.{self.files_fixtures["TEST1_ZFS"]}' self.test2_zfs_file_system = f'{self.user_name}.{self.files_fixtures["TEST2_ZFS"]}' - self.create_zfs_options = {"perms": 755, "cylsPri": 10, "cylsSec": 2, "timeout": 20, "volumes": ["VPMVSC"]} + self.create_zfs_options = {"perms": 755, "cylsPri": 10, "cylsSec": 2, "timeout": 20} self.mount_zfs_file_system_options = {"fs-type": "ZFS", "mode": "rdonly"} def test_list_dsn_should_return_a_list_of_datasets(self): @@ -65,15 +68,20 @@ def test_get_dsn_content_should_return_content_from_dataset(self): command_output = self.files.get_dsn_content(self.test_member_jcl) self.assertIsInstance(command_output["response"], str) - def test_get_dsn_content_streamed_should_return_a_raw_response_content(self): - """Executing get_dsn_content_streamed should return raw socket response from the server.""" + def test_get_dsn_content_streamed_should_return_response_content(self): + """Executing get_dsn_content_streamed should return response object from the server.""" command_output = self.files.get_dsn_content_streamed(self.test_member_jcl) - self.assertIsInstance(command_output, urllib3.response.HTTPResponse) + self.assertIsInstance(command_output.raw, urllib3.response.HTTPResponse) - def test_get_dsn_binary_content_streamed_should_return_a_raw_response_content(self): - """Executing get_dsn_binary_content_streamed should return raw socket response from the server.""" + def test_get_dsn_binary_content_streamed_should_return_response_content(self): + """Executing get_dsn_binary_content_streamed should return response object from the server.""" command_output = self.files.get_dsn_binary_content_streamed(self.test_member_jcl) - self.assertIsInstance(command_output, urllib3.response.HTTPResponse) + self.assertIsInstance(command_output.raw, urllib3.response.HTTPResponse) + + def test_get_file_content_streamed_should_return_response_content(self): + """Executing get_dsn_binary_content_streamed should return response object from the server.""" + command_output = self.files.get_file_content_streamed(self.files_fixtures["TEST_USS"]) + self.assertIsInstance(command_output.raw, urllib3.response.HTTPResponse) def test_write_to_dsn_should_be_possible(self): """Executing write_to_dsn should be possible.""" @@ -83,7 +91,7 @@ def test_write_to_dsn_should_be_possible(self): def test_copy_uss_to_dataset_should_be_possible(self): """Executing copy_uss_to_dataset should be possible.""" command_output = self.files.copy_uss_to_dataset( - self.files_fixtures["TEST_USS"], "ZOWE.TESTS.JCL(TEST2)", replace=True + self.files_fixtures["TEST_USS"], self.files_fixtures["TEST_PDS"] + "(TEST2)", replace=True ) self.assertTrue(command_output["response"] == "") @@ -102,7 +110,7 @@ def test_copy_dataset_or_member_should_be_possible(self): def test_mount_unmount_zfs_file_system(self): """Mounting a zfs filesystem should be possible""" username = self.user_name.lower() - mount_point = f"/u/{username}/mount" # Assuming a dir called mount exist in zOS USS + mount_point = self.files_fixtures["TEST_USS_MOUNT"] # Create a zfs file system zfs_file_system = self.files.create_zFS_file_system(self.test2_zfs_file_system, self.create_zfs_options) @@ -114,7 +122,7 @@ def test_mount_unmount_zfs_file_system(self): self.assertTrue(command_output["response"] == "") # List a zfs file system - command_output = self.files.list_unix_file_systems(file_system_name=self.test2_zfs_file_system) + command_output = self.files.list_unix_file_systems(file_system_name=self.test2_zfs_file_system.upper()) self.assertTrue(len(command_output["items"]) > 0) # Unmount file system @@ -125,4 +133,28 @@ def test_mount_unmount_zfs_file_system(self): command_output = self.files.delete_zFS_file_system(self.test2_zfs_file_system) self.assertTrue(command_output["response"] == "") - # TODO implement tests for download/upload datasets + def test_upload_download_delete_dataset(self): + self.files.upload_file_to_dsn(SAMPLE_JCL_FIXTURE_PATH, self.test_ds_upload) + self.files.download_dsn(self.test_ds_upload, SAMPLE_JCL_FIXTURE_PATH + ".tmp") + + with open(SAMPLE_JCL_FIXTURE_PATH, "r") as in_file: + old_file_content = in_file.read() + with open(SAMPLE_JCL_FIXTURE_PATH + ".tmp", "r") as in_file: + new_file_content = in_file.read().rstrip() + self.assertEqual(old_file_content, new_file_content) + + self.files.delete_data_set(self.files_fixtures["TEST_PDS"], member_name=self.files_fixtures["TEST_MEMBER_NEW"]) + os.unlink(SAMPLE_JCL_FIXTURE_PATH + ".tmp") + + def test_upload_download_delete_uss(self): + self.files.upload_file_to_uss(SAMPLE_JCL_FIXTURE_PATH, self.test_uss_upload) + self.files.download_uss(self.test_uss_upload, SAMPLE_JCL_FIXTURE_PATH + ".tmp") + + with open(SAMPLE_JCL_FIXTURE_PATH, "r") as in_file: + old_file_content = in_file.read() + with open(SAMPLE_JCL_FIXTURE_PATH + ".tmp", "r") as in_file: + new_file_content = in_file.read() + self.assertEqual(old_file_content, new_file_content) + + self.files.delete_uss(self.test_uss_upload) + os.unlink(SAMPLE_JCL_FIXTURE_PATH + ".tmp") diff --git a/tests/integration/test_zos_jobs.py b/tests/integration/test_zos_jobs.py index e27c5863..be32e324 100644 --- a/tests/integration/test_zos_jobs.py +++ b/tests/integration/test_zos_jobs.py @@ -16,7 +16,7 @@ class TestJobsIntegration(unittest.TestCase): def setUp(self): """Setup fixtures for Jobs class.""" - test_profile = ProfileManager().load(profile_type="zosmf") + test_profile = ProfileManager(show_warnings=False).load(profile_type="zosmf") with open(JOBS_FIXTURES_JSON_JSON_PATH, "r") as fixtures_json: self.jobs_fixtures_json = json.load(fixtures_json) self.jobs = Jobs(test_profile) diff --git a/tests/integration/test_zos_tso.py b/tests/integration/test_zos_tso.py index 6df85a5c..01f27ab3 100644 --- a/tests/integration/test_zos_tso.py +++ b/tests/integration/test_zos_tso.py @@ -10,8 +10,8 @@ class TestTsoIntegration(unittest.TestCase): def setUp(self): """Setup fixtures for Tso class.""" - test_profile = ProfileManager().load(profile_type="zosmf") - self.tso = Tso(test_profile) + test_profile = ProfileManager(show_warnings=False).load(profile_type="zosmf") + self.tso = Tso(test_profile, {"account": "IZUACCT"}) def test_issue_command_should_return_valid_response(self): """Executing the issue_command method should return a valid response from TSO""" diff --git a/tests/integration/test_zosmf.py b/tests/integration/test_zosmf.py index 3d8a87ab..87b93c1b 100644 --- a/tests/integration/test_zosmf.py +++ b/tests/integration/test_zosmf.py @@ -10,7 +10,7 @@ class TestZosmfIntegration(unittest.TestCase): def setUp(self): """Setup fixtures for Zosmf class.""" - test_profile = ProfileManager().load(profile_type="zosmf") + test_profile = ProfileManager(show_warnings=False).load(profile_type="zosmf") self.zosmf = Zosmf(test_profile) def test_get_info_should_return_valid_response(self): diff --git a/tests/unit/test_zos_files.py b/tests/unit/test_zos_files.py index acd3478f..504d9285 100644 --- a/tests/unit/test_zos_files.py +++ b/tests/unit/test_zos_files.py @@ -356,38 +356,40 @@ def test_rename_dataset_member_parametrized(self): with self.assertRaises(ValueError) as e_info: files_test_profile.rename_dataset_member(*test_case[0]) self.assertEqual(str(e_info.exception), "Invalid value for enq.") - - @mock.patch("requests.Session.send") + + @mock.patch("requests.Session.send") def test_create_data_set_accept_valid_recfm(self, mock_send_request): """Test if create dataset does accept all accepted record formats""" mock_send_request.return_value = mock.Mock(headers={"Content-Type": "application/json"}, status_code=201) for recfm in ["F", "FB", "V", "VB", "U", "FBA", "FBM", "VBA", "VBM"]: Files(self.test_profile).create_data_set( - "DSNAME123", options={ + "DSNAME123", + options={ "alcunit": "CYL", "dsorg": "PO", "primary": 1, "dirblk": 5, "recfm": recfm, "blksize": 6160, - "lrecl": 80 - } + "lrecl": 80, + }, ) mock_send_request.assert_called() - + def test_create_data_set_does_not_accept_invalid_recfm(self): """Test if create dataset raises an error for invalid record formats""" with self.assertRaises(KeyError): Files(self.test_profile).create_data_set( - "DSNAME123", options={ + "DSNAME123", + options={ "alcunit": "CYL", "dsorg": "PO", "primary": 1, "dirblk": 5, "recfm": "XX", "blksize": 6160, - "lrecl": 80 - } + "lrecl": 80, + }, ) def test_create_data_set_raises_error_without_required_arguments(self): diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 23445756..7b1d1188 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -184,8 +184,8 @@ def setUp(self): self.original_file_path = os.path.join(FIXTURES_PATH, "zowe.config.json") self.original_user_file_path = os.path.join(FIXTURES_PATH, "zowe.config.user.json") self.original_nested_file_path = os.path.join(FIXTURES_PATH, "nested.zowe.config.json") + self.original_nested_user_file_path = os.path.join(FIXTURES_PATH, "nested.zowe.config.user.json") self.original_schema_file_path = os.path.join(FIXTURES_PATH, "zowe.schema.json") - self.original_invalidUri_schema_file_path = os.path.join(FIXTURES_PATH, "invalidUri.zowe.schema.json") loader = importlib.util.find_spec("jsonschema") module_path = loader.origin @@ -194,6 +194,7 @@ def setUp(self): self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) + self.fs.add_real_file(self.original_nested_user_file_path) self.fs.add_real_file(self.original_schema_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" @@ -226,7 +227,6 @@ def test_autodiscovery_and_base_profile_loading(self, get_pass_func): # Setup - copy profile to fake filesystem created by pyfakefs cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) self.setUpCreds( @@ -294,6 +294,7 @@ def test_custom_file_and_custom_profile_loading_with_nested_profile(self, get_pa # Setup - copy profile to fake filesystem created by pyfakefs custom_file_path = os.path.join(self.custom_dir, self.custom_filename) shutil.copy(self.original_nested_file_path, custom_file_path) + shutil.copy(self.original_nested_user_file_path, custom_file_path.replace(".json", ".user.json")) self.setUpCreds( custom_file_path, @@ -309,7 +310,13 @@ def test_custom_file_and_custom_profile_loading_with_nested_profile(self, get_pa props: dict = prof_manager.load(profile_name="lpar1.zosmf", validate_schema=False) self.assertEqual(prof_manager.config_filepath, custom_file_path) - expected_props = {"host": "example1.com", "rejectUnauthorized": True, "port": 443} + expected_props = { + "host": "example1.com", + "rejectUnauthorized": True, + "port": 443, + "user": "admin", + "password": "password1", + } self.assertEqual(props, expected_props) @mock.patch("zowe.secrets_for_zowe_sdk.keyring.get_password", side_effect=keyring_get_password) @@ -322,9 +329,8 @@ def test_profile_loading_with_user_overridden_properties(self, get_pass_func): cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) - shutil.copy(self.original_user_file_path, cwd_up_dir_path) + shutil.copy(self.original_user_file_path, cwd_up_file_path.replace(".json", ".user.json")) self.setUpCreds( cwd_up_file_path, @@ -360,7 +366,6 @@ def test_profile_loading_exception(self, get_pass_func): # Setup cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, f"{self.custom_appname}.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) # Test @@ -414,7 +419,6 @@ def test_load_secure_props(self, retrieve_cred_func): # Setup - copy profile to fake filesystem created by pyfakefs cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) credential = { cwd_up_file_path: {"profiles.base.properties.user": "user", "profiles.base.properties.password": "password"} @@ -461,7 +465,9 @@ def side_effect(*args, **kwargs): delete_pass_func.assert_has_calls(expected_calls) @mock.patch("sys.platform", "win32") - @mock.patch("zowe.secrets_for_zowe_sdk.keyring.get_password", side_effect=["password", None, "part1", "part2\0", None]) + @mock.patch( + "zowe.secrets_for_zowe_sdk.keyring.get_password", side_effect=["password", None, "part1", "part2\0", None] + ) def test_retrieve_credential(self, get_pass_func): """ Test the _retrieve_credential method for retrieving credentials from keyring. @@ -470,13 +476,17 @@ def test_retrieve_credential(self, get_pass_func): # Scenario 1: Retrieve password directly expected_password1 = "password" - retrieve_credential1 = credential_manager._get_credential(constants["ZoweServiceName"], constants["ZoweAccountName"]) + retrieve_credential1 = credential_manager._get_credential( + constants["ZoweServiceName"], constants["ZoweAccountName"] + ) self.assertEqual(retrieve_credential1, expected_password1) get_pass_func.assert_called_with(constants["ZoweServiceName"], constants["ZoweAccountName"]) # Scenario 2: Retrieve password in parts expected_password2 = "part1part2" - retrieve_credential2 = credential_manager._get_credential(constants["ZoweServiceName"], constants["ZoweAccountName"]) + retrieve_credential2 = credential_manager._get_credential( + constants["ZoweServiceName"], constants["ZoweAccountName"] + ) self.assertEqual(retrieve_credential2, expected_password2) get_pass_func.assert_any_call(constants["ZoweServiceName"], constants["ZoweAccountName"]) get_pass_func.assert_any_call(constants["ZoweServiceName"], f"{constants['ZoweAccountName']}-1") @@ -504,7 +514,6 @@ def test_save_secure_props_normal_credential(self, delete_pass_func, retrieve_cr # Setup - copy profile to fake filesystem created by pyfakefs cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) credential = { cwd_up_file_path: { @@ -521,7 +530,9 @@ def test_save_secure_props_normal_credential(self, delete_pass_func, retrieve_cr # delete the existing credential delete_pass_func.return_value = None # Verify the keyring function call - set_pass_func.assert_called_once_with(constants["ZoweServiceName"], constants["ZoweAccountName"], encoded_credential) + set_pass_func.assert_called_once_with( + constants["ZoweServiceName"], constants["ZoweAccountName"], encoded_credential + ) @mock.patch("sys.platform", "win32") @mock.patch("zowe.core_for_zowe_sdk.CredentialManager._get_credential") @@ -531,7 +542,6 @@ def test_save_secure_props_exceed_limit(self, delete_pass_func, set_pass_func, r # Setup - copy profile to fake filesystem created by pyfakefs cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) credential = { cwd_up_file_path: { @@ -552,9 +562,7 @@ def test_save_secure_props_exceed_limit(self, delete_pass_func, set_pass_func, r expected_calls = [] chunk_size = constants["WIN32_CRED_MAX_STRING_LENGTH"] - chunks = [ - encoded_credential[i : i + chunk_size] for i in range(0, len(encoded_credential), chunk_size) - ] + chunks = [encoded_credential[i : i + chunk_size] for i in range(0, len(encoded_credential), chunk_size)] for index, chunk in enumerate(chunks, start=1): field_name = f"{constants['ZoweAccountName']}-{index}" expected_calls.append(mock.call(constants["ZoweServiceName"], field_name, chunk)) @@ -569,7 +577,6 @@ def test_profile_loading_with_valid_schema(self, get_pass_func): custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") shutil.copy(self.original_nested_file_path, custom_file_path) shutil.copy(self.original_schema_file_path, self.custom_dir) - os.chdir(self.custom_dir) self.setUpCreds( custom_file_path, @@ -592,7 +599,6 @@ def test_profile_loading_with_invalid_schema(self, get_pass_func): # Setup - copy profile to fake filesystem created by pyfakefs with self.assertRaises(ValidationError): custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") - os.chdir(self.custom_dir) with open(self.original_file_path, "r") as f: original_config = commentjson.load(f) original_config["$schema"] = "invalid.zowe.schema.json" @@ -624,7 +630,6 @@ def test_profile_loading_with_invalid_schema_internet_URI(self, get_pass_func): # Setup - copy profile to fake filesystem created by pyfakefs with self.assertRaises(SchemaError): custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") - os.chdir(self.custom_dir) with open(self.original_file_path, "r") as f: original_config = commentjson.load(f) original_config["$schema"] = "invalidUri.zowe.schema.json" @@ -659,9 +664,9 @@ def test_profile_loading_with_env_variables(self, get_pass_func): # Setup - copy profile to fake filesystem created by pyfakefs os.environ["ZOWE_OPT_HOST"] = "aaditya" custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") + custom_schema_file_path = os.path.join(self.custom_dir, "zowe.schema.json") shutil.copy(self.original_nested_file_path, custom_file_path) - shutil.copy(self.original_schema_file_path, self.custom_dir) - os.chdir(self.custom_dir) + shutil.copy(self.original_schema_file_path, custom_schema_file_path) self.setUpCreds( custom_file_path, @@ -790,7 +795,6 @@ def test_config_file_set_property(self, get_pass_func, mock_is_secure, mock_find """ cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) self.setUpCreds(cwd_up_file_path, {"profiles.zosmf.properties.user": "admin"}) config_file = ConfigFile(name="zowe_abcd", type="User Config", profiles={}) @@ -830,7 +834,6 @@ def test_config_file_set_profile_and_save(self, get_pass_func): """ cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) self.setUpCreds( cwd_up_file_path, {"profiles.zosmf.properties.user": "abc", "profiles.zosmf.properties.password": "def"} @@ -876,7 +879,6 @@ def test_config_file_save(self, mock_save_secure_props): """ cwd_up_dir_path = os.path.dirname(CWD) cwd_up_file_path = os.path.join(cwd_up_dir_path, "zowe.config.json") - os.chdir(CWD) shutil.copy(self.original_file_path, cwd_up_file_path) profile_data = { "lpar1": { @@ -900,41 +902,42 @@ def test_config_file_save(self, mock_save_secure_props): ) -class TestValidateConfigJsonClass(unittest.TestCase): +class TestValidateConfigJsonClass(TestCase): """Testing the validate_config_json function""" + def setUp(self): + self.setUpPyfakefs() + + self.original_file_path = os.path.join(FIXTURES_PATH, "zowe.config.json") + self.original_schema_file_path = os.path.join(FIXTURES_PATH, "zowe.schema.json") + self.fs.add_real_file(self.original_file_path) + self.fs.add_real_file(self.original_schema_file_path) + def test_validate_config_json_valid(self): """Test validate_config_json with valid config.json matching schema.json""" - path_to_config = FIXTURES_PATH + "/zowe.config.json" - path_to_schema = FIXTURES_PATH + "/zowe.schema.json" - - config_json = commentjson.load(open(path_to_config)) - schema_json = commentjson.load(open(path_to_schema)) + config_json = commentjson.load(open(self.original_file_path)) + schema_json = commentjson.load(open(self.original_schema_file_path)) expected = validate(config_json, schema_json) - result = validate_config_json(path_to_config, path_to_schema, cwd=FIXTURES_PATH) + result = validate_config_json(self.original_file_path, self.original_schema_file_path, cwd=FIXTURES_PATH) self.assertEqual(result, expected) def test_validate_config_json_invalid(self): """Test validate_config_json with invalid config.json that does not match schema.json""" - path_to_config = FIXTURES_PATH + "/zowe.config.json" - path_to_schema = FIXTURES_PATH + "/zowe.schema.json" - path_to_invalid_config = "invalid.zowe.config.json" - path_to_invalid_schema = "invalid.zowe.schema.json" - custom_dir = os.path.dirname(FIXTURES_PATH) - custom_file_path = os.path.join(custom_dir, "zowe.config.json") - os.chdir(custom_dir) - with open(path_to_config, "r") as f: + path_to_invalid_config = os.path.join(custom_dir, "invalid.zowe.config.json") + path_to_invalid_schema = os.path.join(custom_dir, "invalid.zowe.schema.json") + + with open(self.original_file_path, "r") as f: original_config = commentjson.load(f) original_config["$schema"] = "invalid.zowe.schema.json" original_config["profiles"]["zosmf"]["properties"]["port"] = "10443" - with open(os.path.join(custom_dir, "invalid.zowe.config.json"), "w") as f: + with open(path_to_invalid_config, "w") as f: commentjson.dump(original_config, f) - with open(path_to_schema, "r") as f: + with open(self.original_schema_file_path, "r") as f: original_schema = commentjson.load(f) - with open(os.path.join(custom_dir, "invalid.zowe.schema.json"), "w") as f: + with open(path_to_invalid_schema, "w") as f: commentjson.dump(original_schema, f) invalid_config_json = commentjson.load(open(path_to_invalid_config)) invalid_schema_json = commentjson.load(open(path_to_invalid_schema))