From 190287659cb92126b7687af666cbae15693de599 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 28 Nov 2023 15:49:55 -0500 Subject: [PATCH 1/9] IS-13: add basic API tests Successfully tested against: https://github.com/garethsb/nmos-cpp/commit/04ac4fd0893828a3d197b1c6c12f7c70e8990dda https://github.com/AMWA-TV/is-13/commit/8655e0ce019341a21fc607c9d91ccea9b8a0db7b --- README.md | 1 + nmostesting/Config.py | 11 ++++++++++ nmostesting/NMOSTesting.py | 9 ++++++++ nmostesting/suites/IS1301Test.py | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 nmostesting/suites/IS1301Test.py diff --git a/README.md b/README.md index 01bed2c7..febf3e81 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ The following test suites are currently supported. | IS-09-01 | IS-09 System API | | (X) | | System Parameters Server | | IS-09-02 | IS-09 System API Discovery | X | | | | | IS-10-01 | IS-10 Authorization API | | | | Authorization Server | +| IS-13-01 | IS-13 Annotation API | X | | | | | - | BCP-002-01 Natural Grouping | X | | | Included in IS-04 Node API suite | | - | BCP-002-02 Asset Distinguishing Information | X | | | Included in IS-04 Node API suite | | BCP-003-01 | BCP-003-01 Secure Communication | X | X | | See [Testing TLS](docs/2.2.%20Usage%20-%20Testing%20BCP-003-01%20TLS.md) | diff --git a/nmostesting/Config.py b/nmostesting/Config.py index c663ec91..0a2240b3 100644 --- a/nmostesting/Config.py +++ b/nmostesting/Config.py @@ -280,6 +280,17 @@ } } }, + "is-13": { + "repo": "is-13", + "versions": ["v1.0"], + "default_version": "v1.0", + "apis": { + "annotation": { + "name": "Annotation API", + "raml": "AnnotationAPI.raml" + } + } + }, "bcp-002-01": { "repo": "bcp-002-01", "versions": ["v1.0"], diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index e4860827..b5dee403 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -82,6 +82,7 @@ from .suites import IS0901Test from .suites import IS0902Test # from .suites import IS1001Test +from .suites import IS1301Test from .suites import BCP00301Test from .suites import BCP0060101Test from .suites import BCP0060102Test @@ -340,6 +341,14 @@ # }], # "class": IS1001Test.IS1001Test # }, + "IS-13-01": { + "name": "IS-13 Annotation API", + "specs": [{ + "spec_key": "is-13", + "api_key": "annotation" + }], + "class": IS1301Test.IS1301Test, + }, "BCP-003-01": { "name": "BCP-003-01 Secure Communication", "specs": [{ diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py new file mode 100644 index 00000000..e66756b8 --- /dev/null +++ b/nmostesting/suites/IS1301Test.py @@ -0,0 +1,35 @@ +# Copyright (C) 2023 Advanced Media Workflow Association +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..GenericTest import GenericTest, NMOSTestException +from ..TestHelper import compare_json + +ANNOTATION_API_KEY = "annotation" + + +class IS1301Test(GenericTest): + """ + Runs IS-13-Test + """ + def __init__(self, apis, **kwargs): + GenericTest.__init__(self, apis, **kwargs) + self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] + + def test_01(self, test): + """ 1st annotation test """ + + if compare_json({}, {}): + return test.PASS() + else: + return test.FAIL("IO Resource does not correctly reflect the API resources") From 8a4c1bf2150c4ab32310bda33ee95af4d91cd369 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Wed, 29 Nov 2023 10:05:08 -0500 Subject: [PATCH 2/9] Output more explicit error messages When possible, TestResult.detail should be displayed in stderr since the raised exception prevents from the report creation. --- nmostesting/GenericTest.py | 2 +- nmostesting/NMOSTesting.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nmostesting/GenericTest.py b/nmostesting/GenericTest.py index 951eb1fa..ecb73131 100644 --- a/nmostesting/GenericTest.py +++ b/nmostesting/GenericTest.py @@ -671,7 +671,7 @@ def check_api_resource(self, test, resource, response_code, api, path): schema = self.get_schema(api, resource[1]["method"], resource[0], response.status_code) if not schema: - raise NMOSTestException(test.MANUAL("Test suite unable to locate schema")) + raise NMOSTestException(test.MANUAL(f"Test suite unable to locate schema for resource:{resource}")) return self.check_response(schema, resource[1]["method"], response) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index b5dee403..39906492 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -49,7 +49,7 @@ from .DNS import DNS from .GenericTest import NMOSInitException from . import ControllerTest -from .TestResult import TestStates +from .TestResult import TestStates, TestResult from .TestHelper import get_default_ip from .NMOSUtils import DEFAULT_ARGS from .CRL import CRL, CRL_API @@ -632,7 +632,7 @@ def run_tests(test, endpoints, test_selection=["all"]): try: result = test_obj.run_tests(test_selection) except Exception as ex: - print(" * ERROR: {}".format(ex)) + print(" * ERROR while running {}: {}".format(test_selection, ex)) raise ex finally: core_app.config['TEST_ACTIVE'] = False @@ -970,7 +970,8 @@ def run_noninteractive_tests(args): else: exit_code = print_test_results(results, endpoints, args) except Exception as e: - print(" * ERROR: {}".format(str(e))) + print(" * ERROR raw: {}".format(e.args)) + print(" * ERROR in non-interactive tests: {}".format(str(e) if not isinstance(e.args[0], TestResult) else e.args[0].detail)) exit_code = ExitCodes.ERROR return exit_code From 9d9241235b937a043609bd8423041ce5271f94b2 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 22 Jan 2024 17:41:57 -0500 Subject: [PATCH 3/9] IS-13: add tests for self & devices resources For each resource&objects: - Read initial value and store - Reset default value, check timestamp and store - Write max-length and check value+timestamp - Write >max-length and check value+timestamp - Reset default value and compare - Restore initial value Apply to objects: label, description, tags TODO: check IS04 --- nmostesting/suites/IS1301Test.py | 184 ++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 5 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index e66756b8..d8478662 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -17,6 +17,27 @@ ANNOTATION_API_KEY = "annotation" +from ..import TestHelper +import re +import copy + +ANNOTATION_API_KEY = "annotation" + +RESOURCES = ["self", "devices", "senders", "receivers"] +OBJECTS = ["label", "description", "tags"] + +# const for label&description-related tests +STRING_OVER_MAX_VALUE = ''.join(['X' for i in range(100)]) +STRING_MAX_VALUE = STRING_OVER_MAX_VALUE[:64] # this is the max length tolerated + +# const for tags-related tests +TAGS_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} +TAGS_MAX_VALUE = TAGS_OVER_MAX_VALUE.copy() +TAGS_OVER_MAX_VALUE.pop('tech') # must have a max of 5 + +def get_ts_from_version(version): + """ Convert the 'version' object (string) into float """ + return float(re.sub(':', '.', version)) class IS1301Test(GenericTest): """ @@ -26,10 +47,163 @@ def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] - def test_01(self, test): - """ 1st annotation test """ + def set_up_tests(self): + """ + FAKE_ORIG = { + 'description': 'fake_orig_desc', + 'label': 'fake_orig_label', + 'tags': { + 'location': ['fake_location'] + } + } + for resource in RESOURCES: + url = "{}{}{}".format(self.annotation_url, 'node/', resource) + TestHelper.do_request("PATCH", url, json=FAKE_ORIG) + """ + pass + + def get_resource(self, url): + """ Get a resource """ + valid, r = self.do_request("GET", url) + if valid and r.status_code == 200: + try: + return True, r.json() + except Exception as e: + return False, e.msg + else: + return False, "GET Resquest FAIL" + + def set_resource(self, url, new, prev): + """ Patch a resource with one ore several object values """ + object = list(new.keys())[0] + + valid, r = TestHelper.do_request("PATCH", url, json=new) + if valid and r.status_code == 200: + try: + resp = r.json() + except Exception as e: + return False, e.msg + else: + return False, "PATCH max Resquest FAIL" + + if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): + return False, "new version FAIL" + + if new[object] is not None: # NOT a reset + if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): + return False, f"new {object} FAIL" + elif resp[object] != new[object]: + return False, f"new {object} FAIL" + + # TODO this is reflected in IS04 + + return True, resp + + def log(self, msg): + print(msg) + + def do_test(self, test, resource, object): + """ + Perform the test sequence for a resource: + + - Read initial value and store + - Reset default value, check timestamp and store + - Write max-length and check value+timestamp + - Write >max-length and check value+timestamp + - Reset default value and compare + - Restore initial value + """ - if compare_json({}, {}): - return test.PASS() + url = "{}{}{}".format(self.annotation_url, 'node/', resource) + if resource != "self": # get first of the list of devices, receivers, senders + valid, r = self.get_resource(url) + if valid: + print(f" Possible endpoint: {r}") + index = r[0] + url = "{}{}{}".format(url, '/', index) + else: + return test.FAIL(f"Can't find any {resource}") + + msg = "save initial" + valid, r = self.get_resource(url) + if valid: + prev = r + initial = copy.copy(r) + initial.pop('id') + initial.pop('version') + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "reset to default and save" + valid, r = self.set_resource(url, {object: None}, prev) + if valid: + default = prev = r + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "set max value and expected complete response" + value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE + valid, r = self.set_resource(url, {object: value}, prev) + if valid: + prev = r + self.log(f" {msg}: {r}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "set >max value and expect truncated response" + value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE + valid, r = self.set_resource(url, {object: value}, prev) + if valid: + prev = r + self.log(f" {msg}: {r}") + if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE): + return test.FAIL(f"Can't {msg} {resource}/{object}") + elif r[object] != STRING_MAX_VALUE: + return test.FAIL(f"Can't {msg} {resource}/{object}") + else: + return test.FAIL(f"Can't {msg} {resource}/{object}") + + msg = "reset again and compare" + valid, r = self.set_resource(url, {object: None}, prev) + if valid: + self.log(f" {msg}: {r}") + if object == "tags" and not TestHelper.compare_json(default[object], r[object]): + return test.FAIL("Second reset give a different default.") + elif default[object] != r[object]: + return test.FAIL("Second reset give a different default.") + prev = r else: - return test.FAIL("IO Resource does not correctly reflect the API resources") + return test.FAIL(f"Can't {msg} {resource}/{object}") + + # restore initial for courtesy + self.set_resource(url, initial, prev) + + return test.PASS() + + def test_01_01(self, test): + """ Annotation test: self/label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "self", "label") + + def test_01_02(self, test): + """ Annotation test: self/description (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "self", "description") + + def test_01_03(self, test): + """ Annotation test: self/tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "self", "tags") + + def test_02_01(self, test): + """ Annotation test: devices/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "devices", "label") + + def test_02_02(self, test): + """Annotation test: devices/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "devices", "description") + + def test_02_03(self, test): + """Annotation test: devices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "devices", "tags") + +# TODO add receivers + senders From 9c3f07886792a0f1440cb5a3f0254064c4d703a7 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 4 Mar 2024 17:28:31 -0500 Subject: [PATCH 4/9] IS-13: test consistency with IS-04 --- nmostesting/NMOSTesting.py | 4 ++ nmostesting/suites/IS1301Test.py | 88 ++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index 39906492..bce92215 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -346,6 +346,10 @@ "specs": [{ "spec_key": "is-13", "api_key": "annotation" + }, { + "spec_key": "is-04", + "api_key": "node", + "disable_fields": ["host", "port"] }], "class": IS1301Test.IS1301Test, }, diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index d8478662..fd2e6313 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -15,13 +15,12 @@ from ..GenericTest import GenericTest, NMOSTestException from ..TestHelper import compare_json -ANNOTATION_API_KEY = "annotation" - from ..import TestHelper import re import copy ANNOTATION_API_KEY = "annotation" +NODE_API_KEY = "node" RESOURCES = ["self", "devices", "senders", "receivers"] OBJECTS = ["label", "description", "tags"] @@ -46,6 +45,7 @@ class IS1301Test(GenericTest): def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] + self.node_url = f"{self.apis[ANNOTATION_API_KEY]['base_url']}/x-nmos/node/{self.apis[NODE_API_KEY]['version']}/" def set_up_tests(self): """ @@ -73,35 +73,61 @@ def get_resource(self, url): else: return False, "GET Resquest FAIL" - def set_resource(self, url, new, prev): + def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ object = list(new.keys())[0] - valid, r = TestHelper.do_request("PATCH", url, json=new) - if valid and r.status_code == 200: - try: - resp = r.json() - except Exception as e: - return False, e.msg - else: - return False, "PATCH max Resquest FAIL" + valid, resp = TestHelper.do_request("PATCH", url, json=new) + if not valid: + return False, "PATCH Resquest FAIL" + + valid, resp = self.get_resource(url) + if not valid: + return False, "Get Resquest FAIL" + # check that the version (timestamp) has increased if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): return False, "new version FAIL" - + # check PATCH == GET if new[object] is not None: # NOT a reset if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): return False, f"new {object} FAIL" elif resp[object] != new[object]: return False, f"new {object} FAIL" - # TODO this is reflected in IS04 + # validate that it is reflected in IS04 + valid, node_resp = self.get_resource(node_url) + if not valid: + return False, "GET node FAIL" + if new[object] is not None: # NOT a reset + if object == "tags" and not TestHelper.compare_json(node_resp[object], new[object]): + return False, f"new node/.../{object} FAIL" + elif node_resp[object] != new[object]: + return False, f"new node/.../{object} FAIL" + if get_ts_from_version(node_resp['version']) != get_ts_from_version(resp['version']): + return False, "new node version FAIL" return True, resp def log(self, msg): print(msg) + def get_url(self, base_url, resource): + url = f"{base_url}{resource}" + if resource != "self": # get first of the list of devices, receivers, senders + valid, r = self.get_resource(url) + if valid: + if isinstance(r[0], str): # in annotation api + index = r[0] + elif isinstance(r[0], dict): # in node api + index = r[0]['id'] + else: + return None + url = f"{url}/{index}" + else: + return None + return url + def do_test(self, test, resource, object): """ Perform the test sequence for a resource: @@ -114,17 +140,15 @@ def do_test(self, test, resource, object): - Restore initial value """ - url = "{}{}{}".format(self.annotation_url, 'node/', resource) - if resource != "self": # get first of the list of devices, receivers, senders - valid, r = self.get_resource(url) - if valid: - print(f" Possible endpoint: {r}") - index = r[0] - url = "{}{}{}".format(url, '/', index) - else: - return test.FAIL(f"Can't find any {resource}") + url = self.get_url(f"{self.annotation_url}node/", resource) + if not url: + return test.FAIL(f"Can't get annotation url for {resource}") + + node_url = self.get_url(self.node_url, resource) + if not url: + return test.FAIL(f"Can't get node url for {resource}") - msg = "save initial" + msg = "SAVE initial" valid, r = self.get_resource(url) if valid: prev = r @@ -135,26 +159,26 @@ def do_test(self, test, resource, object): else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "reset to default and save" - valid, r = self.set_resource(url, {object: None}, prev) + msg = "RESET to default and save" + valid, r = self.set_resource(url, node_url, {object: None}, prev) if valid: default = prev = r self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "set max value and expected complete response" + msg = "SET MAX value and expected complete response" value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE - valid, r = self.set_resource(url, {object: value}, prev) + valid, r = self.set_resource(url, node_url, {object: value}, prev) if valid: prev = r self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "set >max value and expect truncated response" + msg = "SET >MAX value and expect truncated response" value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE - valid, r = self.set_resource(url, {object: value}, prev) + valid, r = self.set_resource(url, node_url, {object: value}, prev) if valid: prev = r self.log(f" {msg}: {r}") @@ -165,8 +189,8 @@ def do_test(self, test, resource, object): else: return test.FAIL(f"Can't {msg} {resource}/{object}") - msg = "reset again and compare" - valid, r = self.set_resource(url, {object: None}, prev) + msg = "RESET again and compare" + valid, r = self.set_resource(url, node_url, {object: None}, prev) if valid: self.log(f" {msg}: {r}") if object == "tags" and not TestHelper.compare_json(default[object], r[object]): @@ -178,7 +202,7 @@ def do_test(self, test, resource, object): return test.FAIL(f"Can't {msg} {resource}/{object}") # restore initial for courtesy - self.set_resource(url, initial, prev) + self.set_resource(url, node_url, initial, prev) return test.PASS() From 35a2c78b5d1fd8715a3d74c5c2ecbc6c4821172e Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Tue, 5 Mar 2024 10:29:38 -0500 Subject: [PATCH 5/9] is-13: cover senders and receivers --- nmostesting/suites/IS1301Test.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index fd2e6313..f8818392 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -230,4 +230,26 @@ def test_02_03(self, test): """Annotation test: devices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" return self.do_test(test, "devices", "tags") -# TODO add receivers + senders + def test_03_01(self, test): + """ Annotation test: senders/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "senders", "label") + + def test_03_02(self, test): + """Annotation test: senders/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "senders", "description") + + def test_03_03(self, test): + """Annotation test: sender/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "senders", "tags") + + def test_04_01(self, test): + """ Annotation test: receivers/../label (reset to default, set 64-byte value, set >64-byte, check IS04+version)""" + return self.do_test(test, "receivers", "label") + + def test_04_02(self, test): + """Annotation test: receivers/../description (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "receivers", "description") + + def test_04_03(self, test): + """Annotation test: receivers/sevices/../tags (reset to default, set 5 tags, set >5 tags, check IS04+version)""" + return self.do_test(test, "receivers", "tags") From 1f3b948a6ed99ccc215e2aeb20bf94a69a98abb7 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 15:33:27 -0400 Subject: [PATCH 6/9] IS-13: strip grouphint tag which pollutes the test --- nmostesting/suites/IS1301Test.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index f8818392..113b68f4 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -33,15 +33,25 @@ TAGS_OVER_MAX_VALUE = {'location': ['underground'], 'studio': ['42'], 'tech': ['John', 'Mike']} TAGS_MAX_VALUE = TAGS_OVER_MAX_VALUE.copy() TAGS_OVER_MAX_VALUE.pop('tech') # must have a max of 5 +TAGS_TO_BE_SKIPPED = 'urn:x-nmos:tag:grouphint/v1.0' + def get_ts_from_version(version): """ Convert the 'version' object (string) into float """ return float(re.sub(':', '.', version)) + +def strip_tags(tags): + if TAGS_TO_BE_SKIPPED in list(tags.keys()): + tags.pop(TAGS_TO_BE_SKIPPED) + return tags + + class IS1301Test(GenericTest): """ Runs IS-13-Test """ + def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.annotation_url = self.apis[ANNOTATION_API_KEY]["url"] @@ -90,8 +100,11 @@ def set_resource(self, url, node_url, new, prev): return False, "new version FAIL" # check PATCH == GET if new[object] is not None: # NOT a reset - if object == "tags" and not TestHelper.compare_json(resp[object], new[object]): - return False, f"new {object} FAIL" + if object == "tags": + if TAGS_TO_BE_SKIPPED in list(resp[object].keys()): + resp[object].pop(TAGS_TO_BE_SKIPPED) + if not TestHelper.compare_json(resp[object], new[object]): + return False, f"new {object} FAIL" elif resp[object] != new[object]: return False, f"new {object} FAIL" @@ -117,9 +130,9 @@ def get_url(self, base_url, resource): if resource != "self": # get first of the list of devices, receivers, senders valid, r = self.get_resource(url) if valid: - if isinstance(r[0], str): # in annotation api + if isinstance(r[0], str): # in annotation api index = r[0] - elif isinstance(r[0], dict): # in node api + elif isinstance(r[0], dict): # in node api index = r[0]['id'] else: return None @@ -155,6 +168,7 @@ def do_test(self, test, resource, object): initial = copy.copy(r) initial.pop('id') initial.pop('version') + initial['tags'] = strip_tags(initial['tags']) self.log(f" {msg}: {r}") else: return test.FAIL(f"Can't {msg} {resource}/{object}") From 3c8bbf4de3406d2d667bbc9cbf40cb3a900504cf Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 15:44:01 -0400 Subject: [PATCH 7/9] IS-13: improve error msg --- nmostesting/suites/IS1301Test.py | 116 +++++++++++++++++++------------ 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 113b68f4..1ee50c42 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -74,6 +74,7 @@ def set_up_tests(self): def get_resource(self, url): """ Get a resource """ + valid, r = self.do_request("GET", url) if valid and r.status_code == 200: try: @@ -83,6 +84,22 @@ def get_resource(self, url): else: return False, "GET Resquest FAIL" + def compare_resource(self, object, new, resp): + """ Compare string values (or "tags" dict) """ + + if new[object] is None: # this is a reset, new is null, skip + return True, "" + + if object == "tags": # tags needs to be stripped + resp[object] = strip_tags(resp[object]) + if not TestHelper.compare_json(resp[object], new[object]): + return False, f"{object} value FAIL" + + elif resp[object] != new[object]: + return False, f"{object} value FAIL" + + return True, "" + def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ object = list(new.keys())[0] @@ -91,34 +108,29 @@ def set_resource(self, url, node_url, new, prev): if not valid: return False, "PATCH Resquest FAIL" + # re-GET valid, resp = self.get_resource(url) if not valid: return False, "Get Resquest FAIL" - + # check PATCH == GET + valid, msg = self.compare_resource(object, new, resp) + if not valid: + return False, f"new {msg}" # check that the version (timestamp) has increased if get_ts_from_version(prev['version']) >= get_ts_from_version(resp['version']): - return False, "new version FAIL" - # check PATCH == GET - if new[object] is not None: # NOT a reset - if object == "tags": - if TAGS_TO_BE_SKIPPED in list(resp[object].keys()): - resp[object].pop(TAGS_TO_BE_SKIPPED) - if not TestHelper.compare_json(resp[object], new[object]): - return False, f"new {object} FAIL" - elif resp[object] != new[object]: - return False, f"new {object} FAIL" + return False, "new version (timestamp) FAIL" # validate that it is reflected in IS04 valid, node_resp = self.get_resource(node_url) if not valid: - return False, "GET node FAIL" - if new[object] is not None: # NOT a reset - if object == "tags" and not TestHelper.compare_json(node_resp[object], new[object]): - return False, f"new node/.../{object} FAIL" - elif node_resp[object] != new[object]: - return False, f"new node/.../{object} FAIL" + return False, "GET IS-04 Node FAIL" + # check PATCH == GET + valid, msg = self.compare_resource(object, new, resp) + if not valid: + return False, f"new IS-04/node/.../ {msg}" + # check that the version (timestamp) has increased if get_ts_from_version(node_resp['version']) != get_ts_from_version(resp['version']): - return False, "new node version FAIL" + return False, "new IS-04/node/.../version (timestamp) FAIL" return True, resp @@ -126,8 +138,13 @@ def log(self, msg): print(msg) def get_url(self, base_url, resource): + """ + Build the url for both annotation and node APIs which behaves differently. + For iterables resources (devices, senders, receivers), return the 1st element. + """ + url = f"{base_url}{resource}" - if resource != "self": # get first of the list of devices, receivers, senders + if resource != "self": valid, r = self.get_resource(url) if valid: if isinstance(r[0], str): # in annotation api @@ -139,6 +156,7 @@ def get_url(self, base_url, resource): url = f"{url}/{index}" else: return None + return url def do_test(self, test, resource, object): @@ -149,75 +167,83 @@ def do_test(self, test, resource, object): - Reset default value, check timestamp and store - Write max-length and check value+timestamp - Write >max-length and check value+timestamp - - Reset default value and compare + - Reset default value again and compare - Restore initial value """ url = self.get_url(f"{self.annotation_url}node/", resource) if not url: - return test.FAIL(f"Can't get annotation url for {resource}") + msg = f"Can't get annotation url for {resource}" + self.log(f" FAIL {msg}") + return test.FAIL(msg) node_url = self.get_url(self.node_url, resource) if not url: - return test.FAIL(f"Can't get node url for {resource}") + msg = f"Can't get node url for {resource}" + self.log(f" FAIL {msg}") + return test.FAIL(msg) - msg = "SAVE initial" valid, r = self.get_resource(url) + msg = f"SAVE initial: {r}" + self.log(f" {msg}") if valid: prev = r initial = copy.copy(r) initial.pop('id') initial.pop('version') initial['tags'] = strip_tags(initial['tags']) - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "RESET to default and save" valid, r = self.set_resource(url, node_url, {object: None}, prev) + msg = f"RESET to default and save: {r}" + self.log(f" {msg}") if valid: default = prev = r - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "SET MAX value and expected complete response" value = TAGS_MAX_VALUE if object == "tags" else STRING_MAX_VALUE valid, r = self.set_resource(url, node_url, {object: value}, prev) + msg = f"SET MAX value and expected complete response: {r}" + self.log(f" {msg}") if valid: prev = r - self.log(f" {msg}: {r}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "SET >MAX value and expect truncated response" value = TAGS_OVER_MAX_VALUE if object == "tags" else STRING_OVER_MAX_VALUE valid, r = self.set_resource(url, node_url, {object: value}, prev) + msg = f"SET >MAX value and expect truncated response: {r}" + self.log(f" {msg}") if valid: prev = r - self.log(f" {msg}: {r}") - if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE): - return test.FAIL(f"Can't {msg} {resource}/{object}") - elif r[object] != STRING_MAX_VALUE: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(f" {msg}") + if object == "tags" and not TestHelper.compare_json(r[object], TAGS_MAX_VALUE) or r[object] != STRING_MAX_VALUE: + return test.FAIL(f"Can't {msg}") else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") - msg = "RESET again and compare" valid, r = self.set_resource(url, node_url, {object: None}, prev) + msg = f"RESET again and compare: {r}" + self.log(f" {msg}") if valid: - self.log(f" {msg}: {r}") - if object == "tags" and not TestHelper.compare_json(default[object], r[object]): - return test.FAIL("Second reset give a different default.") - elif default[object] != r[object]: - return test.FAIL("Second reset give a different default.") + if object == "tags" and not TestHelper.compare_json(default[object], r[object]) or default[object] != r[object]: + self.log(" FAIL") + return test.FAIL("Second reset gives a different default value.") prev = r else: - return test.FAIL(f"Can't {msg} {resource}/{object}") + self.log(" FAIL") + return test.FAIL(f"Can't {msg}") # restore initial for courtesy self.set_resource(url, node_url, initial, prev) + self.log(" PASS") return test.PASS() def test_01_01(self, test): From 66642768835da979d14ef6813a02e9a11f7370d0 Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 16:40:03 -0400 Subject: [PATCH 8/9] IS-13: add a pause to accomodate the update propagation This seems to impact the version(timestamp) test. --- nmostesting/suites/IS1301Test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 1ee50c42..59ca04b7 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -18,6 +18,7 @@ from ..import TestHelper import re import copy +import time ANNOTATION_API_KEY = "annotation" NODE_API_KEY = "node" @@ -102,12 +103,15 @@ def compare_resource(self, object, new, resp): def set_resource(self, url, node_url, new, prev): """ Patch a resource with one ore several object values """ - object = list(new.keys())[0] + object = list(new.keys())[0] valid, resp = TestHelper.do_request("PATCH", url, json=new) if not valid: return False, "PATCH Resquest FAIL" + # pause to accomodate update propagation + time.sleep(0.1) + # re-GET valid, resp = self.get_resource(url) if not valid: From 5fafebb1529c0e62b44a1c381b5c3c7872f08f1a Mon Sep 17 00:00:00 2001 From: Patrick Keroulas Date: Mon, 15 Apr 2024 17:36:05 -0400 Subject: [PATCH 9/9] is-13: add test suite description --- nmostesting/suites/IS1301Test.py | 33 ++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/nmostesting/suites/IS1301Test.py b/nmostesting/suites/IS1301Test.py index 59ca04b7..eddf161d 100644 --- a/nmostesting/suites/IS1301Test.py +++ b/nmostesting/suites/IS1301Test.py @@ -12,6 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. + +""" +The script implements the IS-13 test suite as specified by the nmos-resource-labelling workgroup. +At the end of the test, the initial state of the tested unit is supposed to be restored but this +cannot be garanteed. + +In addition to the basic annotation API tests, this suite includes the test sequence: +For each resource type (self, devices, senders, receivers): + For each annotable object type (label, description, tags): + - Read initial value + - store + - Reset default value by sending null + - check value + timestamp + is-14/value + is-04 timestamp + - store + - Write max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Write >max-length + - check value + timestamp + is-14/value + is-04 timestamp + - Reset default value again + - check value + timestamp + is-14/value + is-04 timestamp + - compare with 1st reset + - Restore initial value +""" + from ..GenericTest import GenericTest, NMOSTestException from ..TestHelper import compare_json @@ -165,14 +189,7 @@ def get_url(self, base_url, resource): def do_test(self, test, resource, object): """ - Perform the test sequence for a resource: - - - Read initial value and store - - Reset default value, check timestamp and store - - Write max-length and check value+timestamp - - Write >max-length and check value+timestamp - - Reset default value again and compare - - Restore initial value + Perform the test sequence as documented in the file header """ url = self.get_url(f"{self.annotation_url}node/", resource)