From 9425c4c13baaadb4d210a69118ac7d4056b2dae3 Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Mon, 27 May 2019 13:03:12 +0200 Subject: [PATCH] Add tests for opencas.py Signed-off-by: Jan Musial --- .gitignore | 3 + test/utils_tests/opencas-py-tests/__init__.py | 0 test/utils_tests/opencas-py-tests/conftest.py | 15 + test/utils_tests/opencas-py-tests/helpers.py | 119 +++++ .../opencas-py-tests/test_cas_config_01.py | 487 ++++++++++++++++++ .../test_cas_config_cache_01.py | 402 +++++++++++++++ .../test_cas_config_core_01.py | 154 ++++++ .../opencas-py-tests/test_casadm_01.py | 53 ++ utils/opencas.py | 15 +- 9 files changed, 1240 insertions(+), 8 deletions(-) create mode 100644 test/utils_tests/opencas-py-tests/__init__.py create mode 100644 test/utils_tests/opencas-py-tests/conftest.py create mode 100644 test/utils_tests/opencas-py-tests/helpers.py create mode 100644 test/utils_tests/opencas-py-tests/test_cas_config_01.py create mode 100644 test/utils_tests/opencas-py-tests/test_cas_config_cache_01.py create mode 100644 test/utils_tests/opencas-py-tests/test_cas_config_core_01.py create mode 100644 test/utils_tests/opencas-py-tests/test_casadm_01.py diff --git a/.gitignore b/.gitignore index bff028b25..e39af3eb6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ Module.symvers Module.markers *.mod.c modules.order +__pycache__/ +*.py[cod] +*$py.class diff --git a/test/utils_tests/opencas-py-tests/__init__.py b/test/utils_tests/opencas-py-tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/utils_tests/opencas-py-tests/conftest.py b/test/utils_tests/opencas-py-tests/conftest.py new file mode 100644 index 000000000..989b56122 --- /dev/null +++ b/test/utils_tests/opencas-py-tests/conftest.py @@ -0,0 +1,15 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import sys + + +def pytest_configure(config): + try: + import helpers + except ImportError: + raise Exception("Couldn't import helpers") + + sys.path.append(helpers.find_repo_root() + "/utils") diff --git a/test/utils_tests/opencas-py-tests/helpers.py b/test/utils_tests/opencas-py-tests/helpers.py new file mode 100644 index 000000000..dca9fe55d --- /dev/null +++ b/test/utils_tests/opencas-py-tests/helpers.py @@ -0,0 +1,119 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import mock +import re +import os +from io import StringIO +from textwrap import dedent + + +def find_repo_root(): + path = os.getcwd() + + while os.path.realpath(path) != "/": + if ".git" in os.listdir(path): + return path + + path = os.path.dirname(path) + + raise Exception( + "Couldn't find repository root - unable to locate opencas.py" + ) + + +def get_process_mock(return_value, stdout, stderr): + process_mock = mock.Mock() + attrs = { + "wait.return_value": return_value, + "communicate.return_value": (stdout, stderr), + } + process_mock.configure_mock(**attrs) + + return process_mock + + +def get_mock_os_exists(existing_files): + return lambda x: x in existing_files + + +def get_hashed_config_list(conf): + """ Convert list of config lines to list of config lines hashes, drop empty lines """ + hashed_conf = [get_conf_line_hash(x) for x in conf] + + return [x for x in hashed_conf if x] + + +def get_conf_line_hash(line): + """ + Removes whitespace, lowercases, comments and sorts cache params if present. + Returns empty line for comment-only lines + + We don't care about order of params and kinds of whitespace in config lines + so normalize it to compare. We do care about case in paths, but to simplify + testing we pretend we don't. + """ + + def sort_cache_params(params): + return ",".join(sorted(params.split(","))) + + line = line.split("#")[0] + + cache_params_pattern = re.compile(r"(.*?\s)(\S+=\S+)") + match = cache_params_pattern.search(line) + if match: + sorted_params = sort_cache_params(match.group(2)) + line = match.group(1) + sorted_params + + return "".join(line.lower().split()) + + +class MockConfigFile(object): + def __init__(self, buffer=""): + self.set_contents(buffer) + + def __enter__(self): + return self.buffer + + def __exit__(self, *args, **kwargs): + self.set_contents(self.buffer.getvalue()) + + def __call__(self, path, mode): + if mode == "w": + self.buffer = StringIO() + + return self + + def read(self): + return self.buffer.read() + + def write(self, str): + return self.buffer.write(str) + + def close(self): + self.set_contents(self.buffer.getvalue()) + + def readline(self): + return self.buffer.readline() + + def __next__(self): + return self.buffer.__next__() + + def __iter__(self): + return self + + def set_contents(self, buffer): + self.buffer = StringIO(dedent(buffer).strip()) + + +class CopyableMock(mock.Mock): + def __init__(self, *args, **kwargs): + super(CopyableMock, self).__init__(*args, **kwargs) + self.copies = [] + + def __deepcopy__(self, memo): + copy = mock.Mock(spec=self) + self.copies += [copy] + return copy diff --git a/test/utils_tests/opencas-py-tests/test_cas_config_01.py b/test/utils_tests/opencas-py-tests/test_cas_config_01.py new file mode 100644 index 000000000..cfe206e93 --- /dev/null +++ b/test/utils_tests/opencas-py-tests/test_cas_config_01.py @@ -0,0 +1,487 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest +from mock import patch, mock_open +from textwrap import dedent +import helpers as h + +import opencas + + +@patch("builtins.open", new_callable=mock_open) +def test_cas_config_from_file_exception(mock_file): + mock_file.raises = ValueError() + + with pytest.raises(Exception): + opencas.cas_config.from_file("/dummy/file.conf") + + mock_file.assert_called_once_with("/dummy/file.conf", "r") + + +@patch( + "builtins.open", + new_callable=h.MockConfigFile, + buffer=""" + [caches] + 1 /dev/nvme0n1 WT + [cores] + 1 1 /dev/sdc + """, +) +def test_cas_config_from_file_no_vertag(mock_file): + with pytest.raises(ValueError): + opencas.cas_config.from_file("/dummy/file.conf") + + +@patch( + "builtins.open", + new_callable=h.MockConfigFile, + buffer=""" + version=03.08.00 + #[caches] + #1 /dev/nvme0n1 WT + #[cores] + #1 1 /dev/sdc + """, +) +@patch("opencas.cas_config.core_config.from_line") +@patch("opencas.cas_config.cache_config.from_line") +@patch("opencas.cas_config.insert_core") +@patch("opencas.cas_config.insert_cache") +def test_cas_config_from_file_comments_only( + mock_insert_cache, + mock_insert_core, + mock_cache_from_line, + mock_core_from_line, + mock_file, +): + + config = opencas.cas_config.from_file("/dummy/file.conf") + + mock_cache_from_line.assert_not_called() + mock_core_from_line.assert_not_called() + mock_insert_cache.assert_not_called() + mock_insert_core.assert_not_called() + + assert config.is_empty() + + +ConflictingConfigException = opencas.cas_config.ConflictingConfigException +AlreadyConfiguredException = opencas.cas_config.AlreadyConfiguredException + + +@pytest.mark.parametrize( + "caches_config,cores_config,exception", + [ + ( + [ + "1 /dev/dummy0n1 WT", + "2 /dev/dummy0n1 WT", + "3 /dev/dummy0n1 WT", + ], + ["1 1 /dev/dummyc"], + ConflictingConfigException, + ), + ( + ["1 /dev/dummyc WT"], + ["1 1 /dev/dummyc"], + ConflictingConfigException, + ), + ( + ["1 /dev/dummya WT", "1 /dev/dummy0n1 WT"], + ["1 1 /dev/dummyc"], + ConflictingConfigException, + ), + ( + ["1 /dev/dummya WT", "1 /dev/dummya WT"], + ["1 1 /dev/dummyc"], + AlreadyConfiguredException, + ), + ( + ["1 /dev/dummya WT"], + ["1 1 /dev/dummyc", "1 1 /dev/dummyc"], + AlreadyConfiguredException, + ), + ( + ["2 /dev/dummya WT"], + ["1 1 /dev/dummyc", "2 1 /dev/dummyb"], + KeyError, + ), + ( + ["1 /dev/dummya WT", "2 /dev/dummy0n1 WT"], + ["1 1 /dev/dummyc", "2 1 /dev/dummyc"], + ConflictingConfigException, + ), + ], +) +@patch("builtins.open", new_callable=h.MockConfigFile) +@patch("opencas.cas_config.cache_config.validate_config") +@patch("opencas.cas_config.core_config.validate_config") +def test_cas_config_from_file_inconsistent_configs( + mock_validate_core, + mock_validate_cache, + mock_file, + caches_config, + cores_config, + exception, +): + + mock_file.set_contents( + dedent( + """ + version=3.8.0 + [caches] + {0} + [cores] + {1} + """ + ).format("\n".join(caches_config), "\n".join(cores_config)) + ) + + with pytest.raises(exception): + opencas.cas_config.from_file("/dummy/file.conf") + + +@patch( + "builtins.open", + new_callable=h.MockConfigFile, + buffer=""" + version=3.8.0 + [caches] + 1 /dev/dummy0n1 WT + [cores] + 1 1 /dev/dummyc + """, +) +@patch("opencas.cas_config.cache_config.validate_config") +@patch("opencas.cas_config.core_config.validate_config") +def test_cas_config_is_empty_non_empty( + mock_validate_core, mock_validate_cache, mock_file +): + + config = opencas.cas_config.from_file("/dummy/file.conf") + + assert not config.is_empty() + + +def test_cas_config_double_add_cache(): + config = opencas.cas_config() + + cache = opencas.cas_config.cache_config(1, "/dev/dummy", "WT") + config.insert_cache(cache) + + with pytest.raises(AlreadyConfiguredException): + config.insert_cache(cache) + + +def test_cas_config_double_add_core(): + config = opencas.cas_config() + cache = opencas.cas_config.cache_config(1, "/dev/dummy1", "WT") + config.insert_cache(cache) + + core = opencas.cas_config.core_config(1, 1, "/dev/dummy") + config.insert_core(core) + + with pytest.raises(AlreadyConfiguredException): + config.insert_core(core) + + +def test_cas_config_insert_core_no_cache(): + config = opencas.cas_config() + + core = opencas.cas_config.core_config(1, 1, "/dev/dummy") + + with pytest.raises(KeyError): + config.insert_core(core) + + +@patch("os.path.realpath") +def test_cas_config_add_same_cache_symlinked_01(mock_realpath): + mock_realpath.side_effect = ( + lambda x: "/dev/dummy1" if x == "/dev/dummy_link" else x + ) + + config = opencas.cas_config() + cache = opencas.cas_config.cache_config(1, "/dev/dummy1", "WT") + config.insert_cache(cache) + + cache_symlinked = opencas.cas_config.cache_config( + 2, "/dev/dummy_link", "WB" + ) + + with pytest.raises(ConflictingConfigException): + config.insert_cache(cache_symlinked) + + +@patch("os.path.realpath") +def test_cas_config_add_same_cache_symlinked_02(mock_realpath): + mock_realpath.side_effect = ( + lambda x: "/dev/dummy1" if x == "/dev/dummy_link" else x + ) + + config = opencas.cas_config() + cache = opencas.cas_config.cache_config(1, "/dev/dummy1", "WT") + config.insert_cache(cache) + + cache_symlinked = opencas.cas_config.cache_config( + 1, "/dev/dummy_link", "WB" + ) + + with pytest.raises(AlreadyConfiguredException): + config.insert_cache(cache_symlinked) + + +@patch("os.path.realpath") +def test_cas_config_add_same_core_symlinked_01(mock_realpath): + mock_realpath.side_effect = ( + lambda x: "/dev/dummy1" if x == "/dev/dummy_link" else x + ) + + config = opencas.cas_config() + config.insert_cache( + opencas.cas_config.cache_config(1, "/dev/dummy_cache", "WB") + ) + core = opencas.cas_config.core_config(1, 1, "/dev/dummy1") + config.insert_core(core) + + core_symlinked = opencas.cas_config.core_config(1, 2, "/dev/dummy_link") + + with pytest.raises(ConflictingConfigException): + config.insert_core(core_symlinked) + + +@patch("os.path.realpath") +def test_cas_config_add_same_core_symlinked_02(mock_realpath): + mock_realpath.side_effect = ( + lambda x: "/dev/dummy1" if x == "/dev/dummy_link" else x + ) + + config = opencas.cas_config() + config.insert_cache( + opencas.cas_config.cache_config(1, "/dev/dummy_cache", "WB") + ) + core = opencas.cas_config.core_config(1, 1, "/dev/dummy1") + config.insert_core(core) + + core_symlinked = opencas.cas_config.core_config(1, 1, "/dev/dummy_link") + + with pytest.raises(AlreadyConfiguredException): + config.insert_core(core_symlinked) + + +@patch("os.path.realpath") +@patch("os.listdir") +def test_cas_config_get_by_id_path(mock_listdir, mock_realpath): + mock_listdir.return_value = [ + "wwn-1337deadbeef-x0x0", + "wwn-1337deadbeef-x0x0-part1", + "nvme-INTEL_SSDAAAABBBBBCCC_0984547ASDDJHHHFH", + ] + mock_realpath.side_effect = ( + lambda x: "/dev/dummy1" + if x == "/dev/disk/by-id/wwn-1337deadbeef-x0x0-part1" + else x + ) + + path = opencas.cas_config.get_by_id_path("/dev/dummy1") + + assert path == "/dev/disk/by-id/wwn-1337deadbeef-x0x0-part1" + + +@patch("os.path.realpath") +@patch("os.listdir") +def test_cas_config_get_by_id_path_not_found(mock_listdir, mock_realpath): + mock_listdir.return_value = [ + "wwn-1337deadbeef-x0x0", + "wwn-1337deadbeef-x0x0-part1", + "nvme-INTEL_SSDAAAABBBBBCCC_0984547ASDDJHHHFH", + ] + mock_realpath.side_effect = lambda x: x + + with pytest.raises(ValueError): + path = opencas.cas_config.get_by_id_path("/dev/dummy1") + + +@pytest.mark.parametrize( + "caches_config,cores_config", + [ + ( + [ + "1 /dev/dummy0n1 WT", + "2 /dev/dummy0n2 WT", + "3 /dev/dummy0n3 WT", + ], + ["1 1 /dev/dummyc"], + ), + ([], []), + ( + [ + "1 /dev/dummy0n1 WT", + "2 /dev/dummy0n2 WT", + "3 /dev/dummy0n3 WT", + ], + [ + "1 1 /dev/dummyc1", + "2 200 /dev/dummyc2", + "3 100 /dev/dummyc3", + ], + ), + ( + [ + "1 /dev/dummy0n1 WT cleaning_policy=acp", + "2 /dev/dummy0n2 pt ioclass_file=mango.csv", + "3 /dev/dummy0n3 WA cache_line_size=16", + "4 /dev/dummyc wb cache_line_size=16,ioclass_file=mango.csv,cleaning_policy=nop", + ], + [], + ), + ], +) +@patch("builtins.open", new_callable=h.MockConfigFile) +@patch("opencas.cas_config.cache_config.validate_config") +@patch("opencas.cas_config.core_config.validate_config") +def test_cas_config_from_file_to_file( + mock_validate_core, + mock_validate_cache, + mock_file, + caches_config, + cores_config, +): + """ + 1. Read config from mocked file with parametrized caches and cores section + 2. Serialize config back to mocked file + 3. Check if serialized file is proper config file and the same content-wise + as the initial file. Specifically check: + * Version tag is present in first line + * There is only one of each [caches] and [cores] sections marking + * [cores] section comes after [caches] + * sets of caches and cores are equal before and after test + """ + + mock_file.set_contents( + dedent( + """ + version=3.8.0 + [caches] + {0} + [cores] + {1} + """ + ).format("\n".join(caches_config), "\n".join(cores_config)) + ) + + config = opencas.cas_config.from_file("/dummy/file.conf") + + config.write("/dummy/file.conf") + + f = mock_file("/dummy/file.conf", "r") + contents_hashed = h.get_hashed_config_list(f) + + assert contents_hashed[0] == "version=3.8.0" + assert contents_hashed.count("[caches]") == 1 + assert contents_hashed.count("[cores]") == 1 + + caches_index = contents_hashed.index("[caches]") + cores_index = contents_hashed.index("[cores]") + + assert cores_index > caches_index + + caches_hashed = h.get_hashed_config_list(caches_config) + cores_hashed = h.get_hashed_config_list(cores_config) + + assert set(caches_hashed) == set( + contents_hashed[caches_index + 1 : cores_index] + ) + assert set(cores_hashed) == set(contents_hashed[cores_index + 1 :]) + + +@pytest.mark.parametrize( + "caches_config,cores_config", + [ + ( + [ + "1 /dev/dummy0n1 WT", + "2 /dev/dummy0n2 WT", + "3 /dev/dummy0n3 WT", + ], + ["1 1 /dev/dummyc"], + ), + ([], []), + ( + [ + "1 /dev/dummy0n1 WT", + "2 /dev/dummy0n2 WT", + "3 /dev/dummy0n3 WT", + ], + [ + "1 1 /dev/dummyc1", + "2 200 /dev/dummyc2", + "3 100 /dev/dummyc3", + ], + ), + ( + [ + "1 /dev/dummy0n1 WT cleaning_policy=acp", + "2 /dev/dummy0n2 pt ioclass_file=mango.csv", + "3 /dev/dummy0n3 WA cache_line_size=16", + "4 /dev/dummyc wb cache_line_size=16,ioclass_file=mango.csv,cleaning_policy=nop", + ], + [], + ), + ], +) +@patch("builtins.open", new_callable=h.MockConfigFile) +@patch("opencas.cas_config.cache_config.validate_config") +@patch("opencas.cas_config.core_config.validate_config") +def test_cas_config_from_file_insert_cache_insert_core_to_file( + mock_validate_core, + mock_validate_cache, + mock_file, + caches_config, + cores_config, +): + """ + 1. Read config from mocked file with parametrized caches and cores section + 2. Add one core and one cache to config + 3. Serialize config back to mocked file + 4. Compare that config file after serialization is same content-wise with + initial + added core and cache + """ + + mock_file.set_contents( + dedent( + """ + version=3.8.0 + [caches] + {0} + [cores] + {1} + """ + ).format("\n".join(caches_config), "\n".join(cores_config)) + ) + + config = opencas.cas_config.from_file("/dummy/file.conf") + + config.insert_cache(opencas.cas_config.cache_config(5, "/dev/mango", "WT")) + config.insert_core(opencas.cas_config.core_config(5, 1, "/dev/mango_core")) + + config.write("/dummy/file.conf") + + f = mock_file("/dummy/file.conf", "r") + contents_hashed = h.get_hashed_config_list(f) + + caches_index = contents_hashed.index("[caches]") + cores_index = contents_hashed.index("[cores]") + + caches_hashed = h.get_hashed_config_list(caches_config) + cores_hashed = h.get_hashed_config_list(cores_config) + + assert set(contents_hashed[caches_index + 1 : cores_index]) - set( + caches_hashed + ) == set(["5/dev/mangowt"]) + assert set(contents_hashed[cores_index + 1 :]) - set(cores_hashed) == set( + ["51/dev/mango_core"] + ) diff --git a/test/utils_tests/opencas-py-tests/test_cas_config_cache_01.py b/test/utils_tests/opencas-py-tests/test_cas_config_cache_01.py new file mode 100644 index 000000000..720b0cd50 --- /dev/null +++ b/test/utils_tests/opencas-py-tests/test_cas_config_cache_01.py @@ -0,0 +1,402 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest +import mock +import stat + +import helpers as h +import opencas + + +@pytest.mark.parametrize( + "line", + [ + "", + " ", + "#", + " # ", + "TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwgc2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdWEu", + " # ? } { ! ", + "1 /dev/nvme0n1 WT 1 2 3", + "1 /dev/nvme0n1 WT ioclass_file=ioclass.csv ,cache_line_size=4", + ], +) +@mock.patch("opencas.cas_config.cache_config.validate_config") +def test_cache_config_from_line_parsing_checks_01(mock_validate, line): + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "line", + [ + "1 /dev/nvme0n1 WT", + "1 /dev/nvme0n1 WT ioclass_file=ioclass.csv,cache_line_size=4", + ], +) +@mock.patch("opencas.cas_config.cache_config.validate_config") +def test_cache_config_from_line_parsing_checks_02(mock_validate, line): + opencas.cas_config.cache_config.from_line(line) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_cache_config_from_line_device_is_directory( + mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/home/user/catpictures"] + ) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFDIR) + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line( + "1 /home/user/catpictures WT" + ) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_cache_config_from_line_device_not_present( + mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists([]) + mock_stat.side_effect = OSError() + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line("1 /dev/nvme0n1 WT") + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +@mock.patch("subprocess.Popen") +def test_cache_config_from_line_device_with_partitions( + mock_popen, mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + mock_popen.return_value = h.get_process_mock(0, "sda\nsda1\nsda2", "") + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line("1 /dev/sda WT") + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +@mock.patch("subprocess.Popen") +def test_cache_config_validate_device_with_partitions( + mock_popen, mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + mock_popen.return_value = h.get_process_mock(0, "sda\nsda1\nsda2", "") + + cache = opencas.cas_config.cache_config( + cache_id="1", device="/dev/sda", cache_mode="WT", params=dict() + ) + + with pytest.raises(ValueError): + cache.validate_config(False) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +@mock.patch("subprocess.Popen") +def test_cache_config_validate_force_device_with_partitions( + mock_popen, mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + mock_popen.return_value = h.get_process_mock(0, "sda\nsda1\nsda2", "") + + cache = opencas.cas_config.cache_config( + cache_id="1", device="/dev/sda", cache_mode="WT", params=dict() + ) + + with pytest.raises(ValueError): + cache.validate_config(True) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +@mock.patch("subprocess.Popen") +def test_cache_config_from_line_device_without_partitions( + mock_popen, mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + mock_popen.return_value = h.get_process_mock(0, "sda\n", "") + + opencas.cas_config.cache_config.from_line("1 /dev/sda WT") + + +@pytest.mark.parametrize("device", ["/dev/cas1-1", "/dev/cas1-300"]) +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_cache_config_from_line_recursive_multilevel( + mock_stat, mock_path_exists, device +): + mock_path_exists.side_effect = h.get_mock_os_exists([]) + mock_stat.raises = OSError() + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line( + "1 {0} WT".format(device) + ) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_cache_config_from_line_multilevel(mock_stat, mock_path_exists): + mock_path_exists.side_effect = h.get_mock_os_exists([]) + mock_stat.raises = OSError() + + opencas.cas_config.cache_config.from_line("2 /dev/cas1-1 WT") + + +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_allow_incomplete(mock_check_block,): + opencas.cas_config.cache_config.from_line( + "1 /dev/sda WT", allow_incomplete=True + ) + + assert not mock_check_block.called + + +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_missing_ioclass_file( + mock_check_block, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/nvme0n1"]) + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line( + "11 /dev/nvme0n1 WT ioclass_file=ioclass.csv,cleaning_policy=nop,cache_line_size=4" + ) + + +@pytest.mark.parametrize( + "params", + [ + "ioclass_file=", + "ioclass_file=asdf", + "ioclass_file=ioclass.csv,ioclass_file=ioclass.csv", + "cleaning_policy=nop,cleaning_policy=acp", + "cleaning_policy=", + "cleaning_policy=INVALID", + "ioclass_file=ioclass.csv, cleaning_policy=nop", + "cache_line_size=4,cache_line_size=8", + "cache_line_size=", + "cache_line_size=0", + "cache_line_size=4k", + "cache_line_size=4kb", + "cache_line_size=256", + "cache_line_size=-1", + "cache_line_size=four", + "cache_line_size=128", + ], +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_parameter_validation_01( + mock_check_block, mock_device_empty, mock_path_exists, params +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "1 /dev/sda WT {0}".format(params) + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "params", + [ + "ioclass_file=ioclass.csv", + "cleaning_policy=acp", + "cleaning_policy=nop", + "cleaning_policy=alru", + "cleaning_policy=AlRu", + "ioclass_file=ioclass.csv,cleaning_policy=nop", + "cache_line_size=4", + "cache_line_size=8", + "cache_line_size=16", + "cache_line_size=32", + "cache_line_size=64", + "cache_line_size=4,cleaning_policy=nop", + "ioclass_file=ioclass.csv,cache_line_size=4,cleaning_policy=nop", + ], +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_parameter_validation_02( + mock_check_block, mock_device_empty, mock_path_exists, params +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "1 /dev/sda WT {0}".format(params) + + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "mode", + [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "ioclass_file=ioclass.csv,cache_line_size=4,cleaning_policy=nop", + " ", + " $$# ", + "PT$$# ", + ], +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_cache_mode_validation_01( + mock_check_block, mock_device_empty, mock_path_exists, mode +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "1 /dev/sda {0}".format(mode) + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "mode", ["wt", "WT", "pt", "PT", "wb", "WB", "wa", "WA", "wA", "Wa"] +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_cache_mode_validation_02( + mock_check_block, mock_device_empty, mock_path_exists, mode +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "1 /dev/sda {0}".format(mode) + + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "cache_id", + [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "lizard", + "", + "#", + "-1", + "3.14", + "3,14", + "3 14", + "0", + "16385", + "99999999999", + ], +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_cache_id_validation_01( + mock_check_block, mock_device_empty, mock_path_exists, cache_id +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "{0} /dev/sda WT".format(cache_id) + + with pytest.raises(ValueError): + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize("cache_id", ["1", "16384", "123"]) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_from_line_cache_id_validation_02( + mock_check_block, mock_device_empty, mock_path_exists, cache_id +): + mock_path_exists.side_effect = h.get_mock_os_exists( + ["/dev/sda", "ioclass.csv"] + ) + + line = "{0} /dev/sda WT".format(cache_id) + + opencas.cas_config.cache_config.from_line(line) + + +@pytest.mark.parametrize( + "params", + [ + { + "cache_id": "1", + "device": "/dev/nvme0n1", + "cache_mode": "WT", + "ioclass_file": "ioclass.csv", + "cleaning_policy": "acp", + "cache_line_size": "4", + }, + { + "cache_id": "16384", + "device": "/dev/nvme0n1p1", + "cache_mode": "wb", + "ioclass_file": "ioclass.csv", + "cleaning_policy": "nop", + "cache_line_size": "64", + }, + {"cache_id": "100", "device": "/dev/sda", "cache_mode": "wb"}, + { + "cache_id": "2", + "device": "/dev/dm-1", + "cache_mode": "wb", + "cleaning_policy": "nop", + "cache_line_size": "64", + }, + { + "cache_id": "1", + "device": "/dev/nvme0n1", + "cache_mode": "WT", + "cache_line_size": "4", + }, + ], +) +@mock.patch("os.path.exists") +@mock.patch("opencas.cas_config.cache_config.check_cache_device_empty") +@mock.patch("opencas.cas_config.check_block_device") +def test_cache_config_to_line_from_line( + mock_check_block, mock_device_empty, mock_path_exists, params +): + mock_path_exists.side_effect = h.get_mock_os_exists( + [params["device"], "ioclass.csv"] + ) + + cache_reference = opencas.cas_config.cache_config(**params) + + cache_reference.validate_config(False) + + cache_after = opencas.cas_config.cache_config.from_line( + cache_reference.to_line() + ) + + assert cache_after.cache_id == cache_reference.cache_id + assert cache_after.device == cache_reference.device + assert str.lower(cache_after.cache_mode) == str.lower( + cache_reference.cache_mode + ) + assert cache_after.params == cache_reference.params diff --git a/test/utils_tests/opencas-py-tests/test_cas_config_core_01.py b/test/utils_tests/opencas-py-tests/test_cas_config_core_01.py new file mode 100644 index 000000000..bfbbcaea1 --- /dev/null +++ b/test/utils_tests/opencas-py-tests/test_cas_config_core_01.py @@ -0,0 +1,154 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest +import mock +import stat + +import helpers as h +import opencas + + +@pytest.mark.parametrize( + "line", + [ + "", + " ", + "#", + " # ", + "TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwgc2VkIGRvIGVpdXNtb2QgdGVtcG9yIGluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdWEu", + " # ? } { ! ", + "1 1 /dev/sda /dev/sdb", + "1 2 1 /dev/sda ", + ], +) +@mock.patch("opencas.cas_config.core_config.validate_config") +def test_core_config_from_line_parsing_checks_01(mock_validate, line): + with pytest.raises(ValueError): + opencas.cas_config.core_config.from_line(line) + + +@pytest.mark.parametrize("line", ["1 1 /dev/sda", "1 1 /dev/sda "]) +@mock.patch("opencas.cas_config.core_config.validate_config") +def test_core_config_from_line_parsing_checks_02(mock_validate, line): + opencas.cas_config.core_config.from_line(line) + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_core_config_from_line_device_is_directory( + mock_stat, mock_path_exists +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/home/user/stuff"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFDIR) + + with pytest.raises(ValueError): + opencas.cas_config.core_config.from_line("1 1 /home/user/stuff") + + +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_core_config_from_line_device_not_present(mock_stat, mock_path_exists): + mock_path_exists.side_effect = h.get_mock_os_exists([]) + mock_stat.side_effect = ValueError() + + with pytest.raises(ValueError): + opencas.cas_config.core_config.from_line("1 1 /dev/sda") + + +def test_core_config_from_line_recursive_multilevel(): + with pytest.raises(ValueError): + opencas.cas_config.core_config.from_line("1 1 /dev/cas1-1") + + +def test_core_config_from_line_multilevel(): + opencas.cas_config.core_config.from_line("1 1 /dev/cas2-1") + + +@mock.patch("opencas.cas_config.check_block_device") +def test_core_config_from_line_allow_incomplete(mock_check_block,): + opencas.cas_config.core_config.from_line( + "1 1 /dev/sda", allow_incomplete=True + ) + + assert not mock_check_block.called + + +@pytest.mark.parametrize( + "cache_id,core_id", + [ + ("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "bbbbbbb"), + ("lizard", "chicken"), + ("0", "0"), + ("0", "100"), + ("0", "-1"), + ("-1", "0"), + ("-1", "1"), + ("-1", "-1"), + ("16385", "4095"), + ("16384", "4096"), + ("0", "0"), + ("1", "-1"), + ], +) +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_core_config_from_line_cache_id_validation_01( + mock_stat, mock_path_exists, cache_id, core_id +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + + line = "{0} {1} /dev/sda".format(cache_id, core_id) + + with pytest.raises(ValueError): + opencas.cas_config.core_config.from_line(line) + + +@pytest.mark.parametrize( + "cache_id,core_id", [("16384", "4095"), ("1", "0"), ("1", "10")] +) +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_core_config_from_line_cache_id_validation_02( + mock_stat, mock_path_exists, cache_id, core_id +): + mock_path_exists.side_effect = h.get_mock_os_exists(["/dev/sda"]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + + line = "{0} {1} /dev/sda".format(cache_id, core_id) + + opencas.cas_config.core_config.from_line(line) + + +@pytest.mark.parametrize( + "cache_id,core_id,device", + [ + ("1", "1", "/dev/sda"), + ("16384", "4095", "/dev/sda1"), + ("16384", "0", "/dev/nvme0n1p"), + ("100", "5", "/dev/dm-10"), + ], +) +@mock.patch("os.path.exists") +@mock.patch("os.stat") +def test_core_config_from_line_cache_id_validation( + mock_stat, mock_path_exists, cache_id, core_id, device +): + mock_path_exists.side_effect = h.get_mock_os_exists([device]) + mock_stat.return_value = mock.Mock(st_mode=stat.S_IFBLK) + + core_reference = opencas.cas_config.core_config( + cache_id=cache_id, core_id=core_id, path=device + ) + + core_reference.validate_config() + + core_after = opencas.cas_config.core_config.from_line( + core_reference.to_line() + ) + assert core_after.cache_id == core_reference.cache_id + assert core_after.core_id == core_reference.core_id + assert core_after.device == core_reference.device diff --git a/test/utils_tests/opencas-py-tests/test_casadm_01.py b/test/utils_tests/opencas-py-tests/test_casadm_01.py new file mode 100644 index 000000000..279ab2d68 --- /dev/null +++ b/test/utils_tests/opencas-py-tests/test_casadm_01.py @@ -0,0 +1,53 @@ +# +# Copyright(c) 2012-2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest +import subprocess +import mock + +from opencas import casadm +from helpers import get_process_mock + + +@mock.patch("subprocess.Popen") +def test_run_cmd_01(mock_popen): + mock_popen.return_value = get_process_mock(0, "successes", "errors") + result = casadm.run_cmd(["casadm", "-L"]) + + assert result.exit_code == 0 + assert result.stdout == "successes" + assert result.stderr == "errors" + mock_popen.assert_called_once_with( + ["casadm", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + +@mock.patch("subprocess.Popen") +def test_run_cmd_02(mock_popen): + mock_popen.return_value = get_process_mock(4, "successes", "errors") + with pytest.raises(casadm.CasadmError): + casadm.run_cmd(["casadm", "-L"]) + + +@mock.patch("subprocess.Popen") +def test_get_version_01(mock_popen): + mock_popen.return_value = get_process_mock(0, "0.0.1", "errors") + result = casadm.get_version() + + assert result.exit_code == 0 + assert result.stdout == "0.0.1" + assert result.stderr == "errors" + mock_popen.assert_called_once_with( + [casadm.casadm_path, "--version", "--output-format", "csv"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +@mock.patch("subprocess.Popen") +def test_get_version_02(mock_popen): + mock_popen.return_value = get_process_mock(4, "successes", "errors") + with pytest.raises(casadm.CasadmError): + casadm.get_version() diff --git a/utils/opencas.py b/utils/opencas.py index 6068f4e67..7a9a52700 100644 --- a/utils/opencas.py +++ b/utils/opencas.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # # Copyright(c) 2012-2019 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause-Clear @@ -223,7 +222,7 @@ def validate_config(self, force, allow_incomplete=False): type(self).check_cache_id_valid(self.cache_id) self.check_recursive() self.check_cache_mode_valid(self.cache_mode) - for param_name, param_value in self.params.iteritems(): + for param_name, param_value in self.params.items(): self.validate_parameter(param_name, param_value) if not allow_incomplete: @@ -289,7 +288,7 @@ def to_line(self): ret = '{0}\t{1}\t{2}'.format(self.cache_id, self.device, self.cache_mode) if len(self.params) > 0: i = 0 - for param, value in self.params.iteritems(): + for param, value in self.params.items(): if i > 0: ret += ',' else: @@ -408,14 +407,14 @@ def insert_cache(self, new_cache_config): raise cas_config.AlreadyConfiguredException( 'Cache already configured') - for cache_id, cache in self.caches.iteritems(): + for cache_id, cache in self.caches.items(): if cache_id != new_cache_config.cache_id: if (os.path.realpath(new_cache_config.device) == os.path.realpath(cache.device)): raise cas_config.ConflictingConfigException( 'This cache device is already configured as a cache') - for _, core in cache.cores.iteritems(): + for _, core in cache.cores.items(): if (os.path.realpath(core.device) == os.path.realpath(new_cache_config.device)): raise cas_config.ConflictingConfigException( @@ -433,13 +432,13 @@ def insert_core(self, new_core_config): raise KeyError('Cache id {0} doesn\'t exist'.format(new_core_config.cache_id)) try: - for cache_id, cache in self.caches.iteritems(): + for cache_id, cache in self.caches.items(): if (os.path.realpath(cache.device) == os.path.realpath(new_core_config.device)): raise cas_config.ConflictingConfigException( 'Core device already configured as a cache') - for core_id, core in cache.cores.iteritems(): + for core_id, core in cache.cores.items(): if (cache_id == new_core_config.cache_id and core_id == new_core_config.core_id): if (os.path.realpath(core.device) @@ -478,7 +477,7 @@ def write(self, config_file): conf.write('# This config was automatically generated\n') conf.write('[caches]\n') - for _, cache in self.caches.iteritems(): + for _, cache in self.caches.items(): conf.write(cache.to_line()) conf.write('\n[cores]\n')