From ebcac2257caf28d36198e18eb6811619a9f6f258 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 May 2019 13:02:09 +0100 Subject: [PATCH 1/5] Move update_config from templates to files --- .../{templates/update_config.j2 => files/update_config.py} | 0 roles/slurm/tasks/elastic.yml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename roles/slurm/{templates/update_config.j2 => files/update_config.py} (100%) diff --git a/roles/slurm/templates/update_config.j2 b/roles/slurm/files/update_config.py similarity index 100% rename from roles/slurm/templates/update_config.j2 rename to roles/slurm/files/update_config.py diff --git a/roles/slurm/tasks/elastic.yml b/roles/slurm/tasks/elastic.yml index c37c5e4b..8fc89652 100644 --- a/roles/slurm/tasks/elastic.yml +++ b/roles/slurm/tasks/elastic.yml @@ -60,8 +60,8 @@ mode: 0755 - name: configure update_config script - template: - src: update_config.j2 + copy: + src: update_config.py dest: /usr/local/bin/update_config mode: 0755 From 8a1f309c335c58a906db9632d9809823122ce0a1 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 May 2019 13:06:23 +0100 Subject: [PATCH 2/5] Rename Ansible task to reflect that it just copies --- roles/slurm/tasks/elastic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/tasks/elastic.yml b/roles/slurm/tasks/elastic.yml index 8fc89652..58d86d6b 100644 --- a/roles/slurm/tasks/elastic.yml +++ b/roles/slurm/tasks/elastic.yml @@ -59,7 +59,7 @@ dest: /usr/local/bin/stopnode mode: 0755 -- name: configure update_config script +- name: install update_config script copy: src: update_config.py dest: /usr/local/bin/update_config From a53da1c5b106c9098186ce5810b061b3d1ed6467 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 May 2019 13:36:35 +0100 Subject: [PATCH 3/5] Move script part into if __name == "__main__" --- roles/slurm/files/update_config.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/roles/slurm/files/update_config.py b/roles/slurm/files/update_config.py index 39a21842..172a0409 100644 --- a/roles/slurm/files/update_config.py +++ b/roles/slurm/files/update_config.py @@ -73,18 +73,19 @@ def get_node_configs(limits, shapes, mgmt_info): yield create_slurmconf_line(i, shape_info, shape, ad) -# TODO Make sure that any nodes which are no longer managed due to service limit reductions are terminated. +if __name__ == "__main__": + # TODO Make sure that any nodes which are no longer managed due to service limit reductions are terminated. -slurm_conf_filename = "/mnt/shared/etc/slurm/slurm.conf" + slurm_conf_filename = "/mnt/shared/etc/slurm/slurm.conf" -node_config = "\n".join(get_node_configs(get_limits(), get_shapes(), get_mgmt_info())) + node_config = "\n".join(get_node_configs(get_limits(), get_shapes(), get_mgmt_info())) -chop = re.compile('(?<=# STARTNODES\n)(.*?)(?=\n?# ENDNODES)', re.DOTALL) + chop = re.compile('(?<=# STARTNODES\n)(.*?)(?=\n?# ENDNODES)', re.DOTALL) -with open(slurm_conf_filename) as f: - all_config = f.read() + with open(slurm_conf_filename) as f: + all_config = f.read() -new_config = chop.sub('{}'.format(node_config), all_config) + new_config = chop.sub('{}'.format(node_config), all_config) -with open(slurm_conf_filename, "w") as f: - f.write(new_config) + with open(slurm_conf_filename, "w") as f: + f.write(new_config) From 1a33f134af18ae0cf6d6f6ae010962030c4eff7c Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 May 2019 13:46:56 +0100 Subject: [PATCH 4/5] Attempt to check the schema of the limits.yaml file --- roles/slurm/files/test_update_config.py | 47 +++++++++++++++++++++++++ roles/slurm/files/update_config.py | 21 +++++++++-- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 roles/slurm/files/test_update_config.py diff --git a/roles/slurm/files/test_update_config.py b/roles/slurm/files/test_update_config.py new file mode 100644 index 00000000..99f437b2 --- /dev/null +++ b/roles/slurm/files/test_update_config.py @@ -0,0 +1,47 @@ +import pytest +import requests_mock +import yaml + +import update_config + + +@pytest.mark.parametrize( + "data", + [ + (""" + VM.Standard2.1: + 1: 1 + 2: 1 + 3: 1 + VM.Standard2.2: + 1: 1 + 2: 1 + 3: 1 + """), + ] +) +def test_get_limits(mocker, data): + mocker.patch("update_config.load_yaml", return_value=yaml.safe_load(data)) + assert isinstance(update_config.get_limits(), dict) + + +@pytest.mark.parametrize( + "data,error", + [ + (""" + VM.Standard2.1: + 1: 1 + 2: 1 + 3: 1 + VM.Standard2.2: + 1: 1 + 2: 1 + 3: 1 + """, + SyntaxError), + ] +) +def test_get_limits_errors(mocker, data, error): + mocker.patch("update_config.load_yaml", return_value=yaml.safe_load(data)) + with pytest.raises(error): + update_config.get_limits() diff --git a/roles/slurm/files/update_config.py b/roles/slurm/files/update_config.py index 172a0409..e58107f1 100644 --- a/roles/slurm/files/update_config.py +++ b/roles/slurm/files/update_config.py @@ -16,7 +16,19 @@ def get_limits() -> Dict[str, Dict[str, str]]: Until OCI has an API to fetch service limits, we have to hard-code them in a file. """ - return load_yaml("limits.yaml") + limits = load_yaml("limits.yaml") + for mappings in limits.values(): + if not isinstance(mappings, dict): + raise SyntaxError + for ad, count in mappings.items(): + if not isinstance(ad, int): + raise SyntaxError + if not isinstance(count, int): + raise SyntaxError + for shape in limits: + if not re.match(r"", shape): + raise ValueError + return limits def get_shapes() -> Dict[str, Dict[str, str]]: @@ -78,7 +90,12 @@ def get_node_configs(limits, shapes, mgmt_info): slurm_conf_filename = "/mnt/shared/etc/slurm/slurm.conf" - node_config = "\n".join(get_node_configs(get_limits(), get_shapes(), get_mgmt_info())) + try: + limits = get_limits() + except SyntaxError: + print("ERROR: Syntax error in `limits.yaml`.") + exit(1) + node_config = "\n".join(get_node_configs(limits, get_shapes(), get_mgmt_info())) chop = re.compile('(?<=# STARTNODES\n)(.*?)(?=\n?# ENDNODES)', re.DOTALL) From 1968a54bf4f515716abaa080fc06e7c6996a0df9 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 May 2019 13:56:12 +0100 Subject: [PATCH 5/5] Check format of shape names --- roles/slurm/files/update_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/update_config.py b/roles/slurm/files/update_config.py index e58107f1..a361e0b5 100644 --- a/roles/slurm/files/update_config.py +++ b/roles/slurm/files/update_config.py @@ -26,7 +26,7 @@ def get_limits() -> Dict[str, Dict[str, str]]: if not isinstance(count, int): raise SyntaxError for shape in limits: - if not re.match(r"", shape): + if re.match(r"\w{2}\.", shape) is None: raise ValueError return limits