diff --git a/src/test_workflow/integ_test/distribution.py b/src/test_workflow/integ_test/distribution.py index 93fbb35efa..991eb00f69 100644 --- a/src/test_workflow/integ_test/distribution.py +++ b/src/test_workflow/integ_test/distribution.py @@ -4,7 +4,8 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. - +import logging +import os.path from abc import ABC, abstractmethod @@ -69,3 +70,22 @@ def uninstall(self) -> None: Allow distribution to do proper cleanup """ pass + + def configure_jvm_options(self, options: list) -> None: + jvm_config_path = os.path.join(os.path.dirname(self.config_path), 'jvm.options') + try: + with open(jvm_config_path, 'r') as file: + file_content = file.read() + + modified_content = file_content + + for jvm_old, jvm_new in options: + modified_content = modified_content.replace(jvm_old, jvm_new) + + with open(jvm_config_path, 'w') as file: + file.write(modified_content) + logging.info("Configured JVM options") + except FileNotFoundError: + logging.error("File not found.") + except Exception as e: + logging.error(f"An error occurred:{e}") diff --git a/src/test_workflow/integ_test/distribution_deb.py b/src/test_workflow/integ_test/distribution_deb.py index 47c3cf3f56..45136ac59e 100644 --- a/src/test_workflow/integ_test/distribution_deb.py +++ b/src/test_workflow/integ_test/distribution_deb.py @@ -46,7 +46,7 @@ def install(self, bundle_name: str) -> None: '--install', bundle_name, '&&', - f'sudo chmod 0666 {self.config_path}', + f'sudo chmod 0666 {self.config_path} {os.path.dirname(self.config_path)}/jvm.options', '&&', f'sudo chmod 0755 {os.path.dirname(self.config_path)} {self.log_dir}', '&&', diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index 9a8573eeca..4f6a961579 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -48,7 +48,7 @@ def install(self, bundle_name: str) -> None: '-y', bundle_name, '&&', - f'sudo chmod 0666 {self.config_path}', + f'sudo chmod 0666 {self.config_path} {os.path.dirname(self.config_path)}/jvm.options', '&&', f'sudo chmod 0755 {os.path.dirname(self.config_path)} {self.log_dir}', '&&', diff --git a/src/test_workflow/integ_test/service_opensearch.py b/src/test_workflow/integ_test/service_opensearch.py index 86e7d4b7b7..8a5bf59aa5 100644 --- a/src/test_workflow/integ_test/service_opensearch.py +++ b/src/test_workflow/integ_test/service_opensearch.py @@ -44,6 +44,7 @@ def __init__( def start(self) -> None: self.dist.install(self.download()) + self.dist.configure_jvm_options([("-Xms1g", "-Xms2g"), ("-Xmx1g", "-Xmx2g")]) self.opensearch_yml_path = self.dist.config_path self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "opensearch-security") diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_deb.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_deb.py index da2106e126..12dfc9a9ab 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_deb.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_deb.py @@ -47,7 +47,7 @@ def test_install(self, check_call_mock: Mock) -> None: "sudo dpkg --purge opensearch && " "sudo env OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123! " "dpkg --install opensearch.deb && " - f"sudo chmod 0666 {self.distribution_deb.config_path} && " + f"sudo chmod 0666 {self.distribution_deb.config_path} {os.path.dirname(self.distribution_deb.config_path)}/jvm.options && " f"sudo chmod 0755 {os.path.dirname(self.distribution_deb.config_path)} {self.distribution_deb.log_dir} && " f"sudo usermod -a -G opensearch `whoami` && " f"sudo usermod -a -G adm `whoami`" diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py index beec76b285..ebf762d7ca 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py @@ -47,7 +47,7 @@ def test_install(self, check_call_mock: Mock) -> None: "sudo yum remove -y opensearch && " "sudo env OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123! " "yum install -y opensearch.rpm && " - f"sudo chmod 0666 {self.distribution_rpm.config_path} && " + f"sudo chmod 0666 {self.distribution_rpm.config_path} {os.path.dirname(self.distribution_rpm.config_path)}/jvm.options && " f"sudo chmod 0755 {os.path.dirname(self.distribution_rpm.config_path)} {self.distribution_rpm.log_dir} && " f"sudo usermod -a -G opensearch `whoami` && " f"sudo usermod -a -G adm `whoami`" diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py index b17a379b4f..cc17548e1c 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py @@ -37,8 +37,8 @@ def setUp(self) -> None: @patch("builtins.open", new_callable=mock_open) @patch("yaml.dump") @patch("tarfile.open") - def test_start(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Mock, mock_pid: Mock, mock_process: Mock) -> None: - + def test_start(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Mock, mock_pid: Mock, + mock_process: Mock) -> None: dependency_installer = MagicMock() service = ServiceOpenSearch( @@ -62,12 +62,18 @@ def test_start(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Mock, # call test target function service.start() - mock_process.assert_called_once_with("export OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123! && ./opensearch-tar-install.sh", os.path.join(self.work_dir, "opensearch-1.1.0"), False) + mock_process.assert_called_once_with( + "export OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123! && ./opensearch-tar-install.sh", + os.path.join(self.work_dir, "opensearch-1.1.0"), False) mock_dump.assert_called_once_with(self.additional_config) - mock_file.assert_called_once_with(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a") - mock_file.return_value.write.assert_called_once_with(mock_dump_result) + assert mock_file.call_count == 3 + mock_file.assert_any_call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "r") + mock_file.assert_any_call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "w") + mock_file.assert_any_call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a") + + mock_file.return_value.write.assert_has_calls([call(''), call(mock_dump_result)]) dependency_installer.download_dist.assert_called_once_with(self.work_dir) mock_tarfile_open.assert_called_once_with(bundle_full_name, "r:gz") @@ -81,8 +87,8 @@ def test_start(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Mock, @patch("builtins.open", new_callable=mock_open) @patch("yaml.dump") @patch("tarfile.open") - def test_start_security_disabled(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Any, mock_pid: Mock, mock_process: Mock, mock_os_isdir: Mock) -> None: - + def test_start_security_disabled(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Any, mock_pid: Mock, + mock_process: Mock, mock_os_isdir: Mock) -> None: dependency_installer = MagicMock() service = ServiceOpenSearch( @@ -107,9 +113,12 @@ def test_start_security_disabled(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file_handler_for_security = mock_open().return_value mock_file_handler_for_additional_config = mock_open().return_value + mock_file_handler_for_jvm_read = mock_open().return_value + mock_file_handler_for_jvm_write = mock_open().return_value # open() will be called twice, one for disabling security, second for additional_config - mock_file.side_effect = [mock_file_handler_for_security, mock_file_handler_for_additional_config] + mock_file.side_effect = [mock_file_handler_for_jvm_read, mock_file_handler_for_jvm_write, + mock_file_handler_for_security, mock_file_handler_for_additional_config] mock_os_isdir.return_value = True @@ -118,10 +127,14 @@ def test_start_security_disabled(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_dump.assert_has_calls([call({"plugins.security.disabled": "true"}), call(self.additional_config)]) + assert mock_file.call_count == 4 mock_file.assert_has_calls( - [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a")], - [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a")], + [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "r"), + call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "w"), + call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a"), + call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a"), ] ) + mock_file_handler_for_security.write.assert_called_once_with(mock_dump_result_for_security) mock_file_handler_for_additional_config.write.assert_called_once_with(mock_dump_result_for_additional_config) @@ -131,8 +144,8 @@ def test_start_security_disabled(self, mock_tarfile_open: Mock, mock_dump: Mock, @patch("builtins.open", new_callable=mock_open) @patch("yaml.dump") @patch("tarfile.open") - def test_start_security_disabled_and_not_installed(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Any, mock_pid: Mock, mock_process: Mock, mock_os_isdir: Mock) -> None: - + def test_start_security_disabled_and_not_installed(self, mock_tarfile_open: Mock, mock_dump: Mock, mock_file: Any, + mock_pid: Mock, mock_process: Mock, mock_os_isdir: Mock) -> None: dependency_installer = MagicMock() service = ServiceOpenSearch( @@ -156,9 +169,12 @@ def test_start_security_disabled_and_not_installed(self, mock_tarfile_open: Mock mock_file_handler_for_security = mock_open().return_value mock_file_handler_for_additional_config = mock_open().return_value + mock_file_handler_for_jvm_read = mock_open().return_value + mock_file_handler_for_jvm_write = mock_open().return_value # open() will be called twice, one for disabling security, second for additional_config - mock_file.side_effect = [mock_file_handler_for_additional_config] + mock_file.side_effect = [mock_file_handler_for_jvm_read, mock_file_handler_for_jvm_write, + mock_file_handler_for_additional_config] mock_os_isdir.return_value = False @@ -167,23 +183,29 @@ def test_start_security_disabled_and_not_installed(self, mock_tarfile_open: Mock mock_dump.assert_has_calls([call(self.additional_config)]) + assert mock_file.call_count == 3 + mock_file.assert_has_calls( - [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a")], - [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a")], + [call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "r"), + call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "jvm.options"), "w"), + call(os.path.join(self.work_dir, "opensearch-1.1.0", "config", "opensearch.yml"), "a") + ] ) mock_file_handler_for_security.write.assert_not_called() mock_file_handler_for_additional_config.write.assert_called_once_with(mock_dump_result_for_additional_config) @patch("test_workflow.integ_test.service.Process.terminate", return_value=123) @patch('test_workflow.integ_test.service.Process.started', new_callable=PropertyMock, return_value=True) - @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data") - @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, return_value="test stderr_data") + @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, + return_value="test stdout_data") + @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, + return_value="test stderr_data") def test_terminate( - self, - mock_process_stderr_data: Mock, - mock_process_stdout_data: Mock, - mock_process_started: Mock, - mock_process_terminate: Mock + self, + mock_process_stderr_data: Mock, + mock_process_stdout_data: Mock, + mock_process_started: Mock, + mock_process_terminate: Mock ) -> None: service = ServiceOpenSearch( self.version, @@ -201,7 +223,8 @@ def test_terminate( self.assertEqual(termination_result.return_code, 123) self.assertEqual(termination_result.stdout_data, "test stdout_data") self.assertEqual(termination_result.stderr_data, "test stderr_data") - self.assertEqual(termination_result.log_files, {"opensearch-service-logs": os.path.join("test_work_dir", "opensearch-1.1.0", "logs")}) + self.assertEqual(termination_result.log_files, + {"opensearch-service-logs": os.path.join("test_work_dir", "opensearch-1.1.0", "logs")}) @patch("test_workflow.integ_test.service.Process.terminate") @patch('test_workflow.integ_test.service.Process.started', new_callable=PropertyMock, return_value=False) @@ -281,9 +304,13 @@ def test_get_service_response(self, mock_url: Mock, mock_requests_get: Mock) -> @patch("time.sleep") @patch.object(ServiceOpenSearch, "service_alive", return_value=True) - @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data") - @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, return_value="test stderr_data") - def test_wait_for_service_succeed_on_first_attemp(self, mock_process_stderr_data: Mock, mock_process_stdout_data: Mock, mock_service_alive: Mock, mock_time_sleep: Mock) -> None: + @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, + return_value="test stdout_data") + @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, + return_value="test stderr_data") + def test_wait_for_service_succeed_on_first_attemp(self, mock_process_stderr_data: Mock, + mock_process_stdout_data: Mock, mock_service_alive: Mock, + mock_time_sleep: Mock) -> None: service = ServiceOpenSearch( self.version, self.distribution, @@ -303,15 +330,17 @@ def test_wait_for_service_succeed_on_first_attemp(self, mock_process_stderr_data @patch('test_workflow.integ_test.service.logging.error') @patch("time.sleep") @patch.object(ServiceOpenSearch, "service_alive", return_value=False) - @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data") - @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, return_value="test stderr_data") + @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, + return_value="test stdout_data") + @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, + return_value="test stderr_data") def test_wait_for_service_always_fail_without_exception( - self, - mock_process_stderr_data: Mock, - mock_process_stdout_data: Mock, - mock_service_alive: Mock, - mock_time_sleep: Mock, - mock_logging_error: Mock + self, + mock_process_stderr_data: Mock, + mock_process_stdout_data: Mock, + mock_service_alive: Mock, + mock_time_sleep: Mock, + mock_logging_error: Mock ) -> None: service = ServiceOpenSearch( self.version, @@ -333,17 +362,18 @@ def test_wait_for_service_always_fail_without_exception( @patch('test_workflow.integ_test.service.logging.error') @patch("time.sleep") @patch.object(ServiceOpenSearch, "service_alive", side_effect=requests.exceptions.ConnectionError()) - @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data") - @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, return_value="test stderr_data") + @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, + return_value="test stdout_data") + @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, + return_value="test stderr_data") def test_wait_for_service_always_fail_with_exception( - self, - mock_process_stderr_data: Mock, - mock_process_stdout_data: Mock, - mock_service_alive: Mock, - mock_time_sleep: Mock, - mock_logging_error: Mock + self, + mock_process_stderr_data: Mock, + mock_process_stdout_data: Mock, + mock_service_alive: Mock, + mock_time_sleep: Mock, + mock_logging_error: Mock ) -> None: - service = ServiceOpenSearch( self.version, self.distribution, @@ -366,14 +396,16 @@ def test_wait_for_service_always_fail_with_exception( ServiceOpenSearch, "service_alive", side_effect=[requests.exceptions.ConnectionError(), requests.exceptions.ConnectionError(), True]) - @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, return_value="test stdout_data") - @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, return_value="test stderr_data") + @patch('test_workflow.integ_test.service.Process.stdout_data', new_callable=PropertyMock, + return_value="test stdout_data") + @patch('test_workflow.integ_test.service.Process.stderr_data', new_callable=PropertyMock, + return_value="test stderr_data") def test_wait_for_service_suceed_on_third_attempt( - self, - mock_process_stderr_data: Mock, - mock_process_stdout_data: Mock, - mock_service_alive: Mock, - mock_time_sleep: Mock + self, + mock_process_stderr_data: Mock, + mock_process_stdout_data: Mock, + mock_service_alive: Mock, + mock_time_sleep: Mock ) -> None: service = ServiceOpenSearch( self.version,