diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index cf3e30b34a..8c1e4a411a 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -190,6 +190,7 @@ def adjust_request_kwargs(self, url, method, kwargs): return kwargs async def put_with_headers(self, url, **kwargs): + """Issues a PUT and returns the status code, headers, and JSON body.""" url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "PUT", kwargs) @@ -201,10 +202,12 @@ async def put_with_headers(self, url, **kwargs): ) async def put(self, url, **kwargs): + """Issues a PUT and returns the status code and JSON body.""" (status, _, json) = await self.put_with_headers(url, **kwargs) return status, json async def get_with_headers(self, url, **kwargs): + """Issues a GET and returns the status code, headers, and JSON body.""" url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "GET", kwargs) @@ -216,6 +219,7 @@ async def get_with_headers(self, url, **kwargs): ) async def get(self, url, **kwargs): + """Issues a GET and returns the status code and JSON body.""" (status, _, json) = await self.get_with_headers(url, **kwargs) return status, json @@ -227,6 +231,7 @@ async def post(self, url, **kwargs): return response.status, await response.json() async def delete_with_headers(self, url, **kwargs): + """Issues a DELETE and returns the status code, headers, and JSON body.""" url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "DELETE", kwargs) @@ -238,6 +243,7 @@ async def delete_with_headers(self, url, **kwargs): ) async def delete(self, url, **kwargs): + """Issues a DELETE and returns the status code and JSON body.""" (status, _, json) = await self.delete_with_headers(url, **kwargs) return status, json diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index fa2e72bb3d..2ac22b7059 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -1,18 +1,15 @@ import datetime from typing import Dict, List, Optional, Union, Set -from implicitdict import ImplicitDict import s2sphere -from uas_standards import Operation - -from monitoring.monitorlib.fetch.rid import RIDQuery, Subscription, ISA -from monitoring.monitorlib.rid import RIDVersion -from uas_standards.astm.f3411 import v19, v22a import uas_standards.astm.f3411.v19.api import uas_standards.astm.f3411.v19.constants import uas_standards.astm.f3411.v22a.api import uas_standards.astm.f3411.v22a.constants import yaml +from implicitdict import ImplicitDict +from uas_standards import Operation +from uas_standards.astm.f3411 import v19, v22a from yaml.representer import Representer from monitoring.monitorlib import ( @@ -21,6 +18,8 @@ rid_v1, rid_v2, ) +from monitoring.monitorlib.fetch.rid import RIDQuery, Subscription, ISA +from monitoring.monitorlib.rid import RIDVersion class ChangedSubscription(RIDQuery): @@ -94,22 +93,49 @@ def isas(self) -> List[ISA]: ) -def upsert_subscription( +def build_subscription_url( + sub_id: str, + sub_version: Optional[str], + rid_version: RIDVersion, +) -> (Operation, str): + """ + Build the required URL to create, get, update or delete a subscription on a DSS, + in accordance with the specified rid_version and sub_version, if it is available. + + Note that for mutations and deletions, sub_version must be provided. + """ + if rid_version == RIDVersion.f3411_19: + if sub_version is None: + op = v19.api.OPERATIONS[v19.api.OperationID.CreateSubscription] + return (op, op.path.format(id=sub_id)) + else: + op = v19.api.OPERATIONS[v19.api.OperationID.UpdateSubscription] + return (op, op.path.format(id=sub_id, version=sub_version)) + elif rid_version == RIDVersion.f3411_22a: + if sub_version is None: + op = v22a.api.OPERATIONS[v22a.api.OperationID.CreateSubscription] + return (op, op.path.format(id=sub_id)) + else: + op = v22a.api.OPERATIONS[v22a.api.OperationID.UpdateSubscription] + return (op, op.path.format(id=sub_id, version=sub_version)) + else: + raise NotImplementedError( + f"Cannot build subscription URL for RID version {rid_version}" + ) + + +def build_subscription_payload( + sub_id: str, area_vertices: List[s2sphere.LatLng], alt_lo: float, alt_hi: float, start_time: Optional[datetime.datetime], end_time: Optional[datetime.datetime], uss_base_url: str, - subscription_id: str, rid_version: RIDVersion, - utm_client: infrastructure.UTMClientSession, - subscription_version: Optional[str] = None, - participant_id: Optional[str] = None, -) -> ChangedSubscription: - mutation = "create" if subscription_version is None else "update" +) -> Dict[str, any]: if rid_version == RIDVersion.f3411_19: - body = { + return { "extents": rid_v1.make_volume_4d( area_vertices, alt_lo, @@ -118,18 +144,57 @@ def upsert_subscription( end_time, ), "callbacks": { - "identification_service_area_url": uss_base_url - + v19.api.OPERATIONS[ - v19.api.OperationID.PostIdentificationServiceArea - ].path[: -len("/{id}")] + "identification_service_area_url": rid_version.post_isa_url_of( + uss_base_url, + sub_id, + ), }, } - if subscription_version is None: - op = v19.api.OPERATIONS[v19.api.OperationID.CreateSubscription] - url = op.path.format(id=subscription_id) - else: - op = v19.api.OPERATIONS[v19.api.OperationID.UpdateSubscription] - url = op.path.format(id=subscription_id, version=subscription_version) + elif rid_version == RIDVersion.f3411_22a: + return { + "extents": rid_v2.make_volume_4d( + area_vertices, + alt_lo, + alt_hi, + start_time, + end_time, + ), + "uss_base_url": uss_base_url, + } + else: + raise NotImplementedError( + f"Cannot upsert subscription using RID version {rid_version}" + ) + + +def upsert_subscription( + area_vertices: List[s2sphere.LatLng], + alt_lo: float, + alt_hi: float, + start_time: Optional[datetime.datetime], + end_time: Optional[datetime.datetime], + uss_base_url: str, + subscription_id: str, + rid_version: RIDVersion, + utm_client: infrastructure.UTMClientSession, + subscription_version: Optional[str] = None, + participant_id: Optional[str] = None, +) -> ChangedSubscription: + mutation = "create" if subscription_version is None else "update" + (op, url) = build_subscription_url( + subscription_id, subscription_version, rid_version + ) + body = build_subscription_payload( + subscription_id, + area_vertices, + alt_lo, + alt_hi, + start_time, + end_time, + uss_base_url, + rid_version, + ) + if rid_version == RIDVersion.f3411_19: return ChangedSubscription( mutation=mutation, v19_query=fetch.query_and_describe( @@ -142,22 +207,6 @@ def upsert_subscription( ), ) elif rid_version == RIDVersion.f3411_22a: - body = { - "extents": rid_v2.make_volume_4d( - area_vertices, - alt_lo, - alt_hi, - start_time, - end_time, - ), - "uss_base_url": uss_base_url, - } - if subscription_version is None: - op = v22a.api.OPERATIONS[v22a.api.OperationID.CreateSubscription] - url = op.path.format(id=subscription_id) - else: - op = v22a.api.OPERATIONS[v22a.api.OperationID.UpdateSubscription] - url = op.path.format(id=subscription_id, version=subscription_version) return ChangedSubscription( mutation=mutation, v22a_query=fetch.query_and_describe( diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 3d2826de41..416ae7dae0 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -2,13 +2,13 @@ from enum import Enum import arrow - -from monitoring.monitorlib import schema_validation -from uas_standards.astm.f3411 import v19, v22a import uas_standards.astm.f3411.v19.api import uas_standards.astm.f3411.v19.constants import uas_standards.astm.f3411.v22a.api import uas_standards.astm.f3411.v22a.constants +from uas_standards.astm.f3411 import v19, v22a + +from monitoring.monitorlib import schema_validation class RIDVersion(str, Enum): @@ -84,6 +84,42 @@ def openapi_delete_isa_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") + @property + def openapi_get_subscription_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.GetSubscriptionResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.GetSubscriptionResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + + @property + def openapi_put_subscription_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.PutSubscriptionResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.PutSubscriptionResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + + @property + def openapi_delete_subscription_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.DeleteSubscriptionResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.DeleteSubscriptionResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + + @property + def openapi_search_subscriptions_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.SearchSubscriptionsResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.SearchSubscriptionsResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + @property def realtime_period(self) -> timedelta: if self == RIDVersion.f3411_19: @@ -237,3 +273,18 @@ def flights_url_of(self, base_url: str) -> str: return base_url + flights_path else: raise ValueError("Unsupported RID version '{}'".format(self)) + + def post_isa_url_of(self, base_url: str, sub_id: str) -> str: + if self == RIDVersion.f3411_19: + isa_path = v19.api.OPERATIONS[ + v19.api.OperationID.PostIdentificationServiceArea + ].path[: -len("/{id}")] + return base_url + isa_path + elif self == RIDVersion.f3411_22a: + # TODO: Urls returned by the DSS contain the ID, confirm this is as expected + isa_path = v22a.api.OPERATIONS[ + v22a.api.OperationID.PostIdentificationServiceArea + ].path.format(id=sub_id) + return base_url + isa_path + else: + raise ValueError("Unsupported RID version '{}'".format(self)) diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index ce0c43786f..025553a809 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -28,6 +28,10 @@ class F3411_19(str, Enum): DeleteIdentificationServiceAreaResponse = ( "components.schemas.DeleteIdentificationServiceAreaResponse" ) + GetSubscriptionResponse = "components.schemas.GetSubscriptionResponse" + PutSubscriptionResponse = "components.schemas.PutSubscriptionResponse" + DeleteSubscriptionResponse = "components.schemas.DeleteSubscriptionResponse" + SearchSubscriptionsResponse = "components.schemas.SearchSubscriptionsResponse" class F3411_22a(str, Enum): @@ -46,6 +50,10 @@ class F3411_22a(str, Enum): DeleteIdentificationServiceAreaResponse = ( "components.schemas.DeleteIdentificationServiceAreaResponse" ) + GetSubscriptionResponse = "components.schemas.GetSubscriptionResponse" + PutSubscriptionResponse = "components.schemas.PutSubscriptionResponse" + DeleteSubscriptionResponse = "components.schemas.DeleteSubscriptionResponse" + SearchSubscriptionsResponse = "components.schemas.SearchSubscriptionsResponse" class F3548_21(str, Enum): diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index d0124b10c6..bca87ef15d 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs): resource_type_code_descriptions: Dict[ResourceType, str] = {} -# Next code: 374 +# Next code: 375 def register_resource_type(code: int, description: str) -> ResourceType: """Register that the specified code refers to the described resource. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py index f8f1f0616b..cdc482b2cd 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py @@ -4,50 +4,88 @@ from typing import List, Dict import arrow +import loguru import requests from uas_standards.astm.f3411 import v19, v22a from monitoring.monitorlib.fetch import ( - rid as fetch, describe_request, Query, describe_aiohttp_response, ) -from monitoring.monitorlib.fetch.rid import FetchedISA +from monitoring.monitorlib.fetch.rid import ( + FetchedISA, + FetchedSubscription, + Subscription, +) from monitoring.monitorlib.infrastructure import AsyncUTMTestSession from monitoring.monitorlib.mutate import rid as mutate -from monitoring.monitorlib.mutate.rid import ChangedISA +from monitoring.monitorlib.mutate.rid import ChangedISA, ChangedSubscription from monitoring.monitorlib.rid import RIDVersion from monitoring.prober.infrastructure import register_resource_type from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource -from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.utils import ISAValidator +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.utils import ( + ISAValidator, + delete_isa_if_exists, + SubscriptionValidator, +) from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario from monitoring.uss_qualifier.suites.suite import ExecutionContext # Semaphore is added to limit the number of simultaneous requests. # Should we consider making these configurable through the scenario's parameters? -SEMAPHORE = asyncio.Semaphore(20) +ISA_SEMAPHORE = asyncio.Semaphore(20) THREAD_COUNT = 10 CREATE_ISAS_COUNT = 100 +# Subscriptions are currently created for the whole ISA footprint, so we can't have more than 10 +# If we want to create more we need to split the ISA footprint. +CREATE_SUBSCRIPTIONS_COUNT = 10 + class HeavyTrafficConcurrent(GenericTestScenario): - """Based on prober/rid/v1/test_isa_simple_heavy_traffic_concurrent.py from the legacy prober tool.""" + """ + Based on prober/rid/v1/test_isa_simple_heavy_traffic_concurrent.py from the legacy prober tool. + + Essentially, this scenario: + + - creates a certain amount of ISAs & subscriptions concurrently + - creates another amount of ISAs & subscriptions concurrently: verify that + previously created ISAs and subscriptions are mentioned in the creation responses + - fetches them concurrently and checks their content + - searches for them in a single query: all should be found + - deletes half of the ISAs and subscriptions concurrently + - deletes the other half of the ISAs and subscriptions concurrently: verify that + previously deleted subscriptions are not mentioned in the ISA deletion response anymore + - fetches them again concurrently: all requests should return a 404 + - searches for them again in a single query: none should be found + + + """ ISA_TYPE = register_resource_type(373, "ISA") + SUB_TYPE = register_resource_type(374, "Subscription") _isa_ids: List[str] + _sub_ids: List[str] + _isa_params: Dict[str, any] + _sub_params: Dict[str, any] + _isa_versions: Dict[str, str] + _current_subs: Dict[str, Subscription] + _async_session: AsyncUTMTestSession + _owner: str + def __init__( self, dss: DSSInstanceResource, @@ -60,7 +98,8 @@ def __init__( ) # TODO: delete once _delete_isa_if_exists updated to use dss_wrapper self._dss_wrapper = DSSWrapper(self, dss.dss_instance) - self._isa_versions: Dict[str, str] = {} + self._isa_versions = {} + self._current_subs = {} self._isa = isa.specification now = arrow.utcnow().datetime @@ -68,7 +107,6 @@ def __init__( self._isa_end_time = self._isa.shifted_time_end(now) self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] - # TODO confirm that the auth-adapter can be reused as-is (this seems to work but a confirmation would be good) # Note that when the test scenario ends prematurely, we may end up with an unclosed session. self._async_session = AsyncUTMTestSession( self._dss.base_url, self._dss.client.auth_adapter @@ -78,6 +116,18 @@ def __init__( # The base ID ends in 000: we simply increment it to generate the other IDs self._isa_ids = [f"{isa_base_id[:-3]}{i:03d}" for i in range(CREATE_ISAS_COUNT)] + sub_base_id = id_generator.id_factory.make_id(HeavyTrafficConcurrent.SUB_TYPE) + self._sub_ids = [ + f"{sub_base_id[:-3]}{i:03d}" for i in range(CREATE_SUBSCRIPTIONS_COUNT) + ] + + # Split ids into two collections + self._first_half_isas = self._isa_ids[: len(self._isa_ids) // 2] + self._first_half_subs = self._sub_ids[: len(self._sub_ids) // 2] + + self._second_half_isas = self._isa_ids[len(self._isa_ids) // 2 :] + self._second_half_subs = self._sub_ids[len(self._sub_ids) // 2 :] + # currently all params are the same: # we could improve the test by having unique parameters per ISA self._isa_params = dict( @@ -89,39 +139,72 @@ def __init__( alt_hi=self._isa.altitude_max, ) + self._sub_params = dict( + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + rid_version=self._dss.rid_version, + ) + + # TODO read from correct resource once the relevant PR is merged + self._owner = id_generator.subscriber + def run(self, context: ExecutionContext): self.begin_test_scenario(context) self.begin_test_case("Setup") self.begin_test_step("Ensure clean workspace") + self._delete_subscriptions_if_exists() self._delete_isas_if_exists() self.end_test_step() self.end_test_case() - self.begin_test_case("Concurrent Requests") + self.begin_test_case("Concurrent requests") + # TODO rename step to 'create entities" self.begin_test_step("Create ISA concurrently") - self._create_isas_concurrent_step() + self._create_entities_first_half_step() + self._create_entities_second_half_step() self.end_test_step() self.begin_test_step("Get ISAs concurrently") self._get_isas_by_id_concurrent_step() self.end_test_step() - self.begin_test_step("Search Available ISAs") + self.begin_test_step("Get subscriptions concurrently") + self._get_subscriptions_by_id_concurrent_step() + self.end_test_step() + + self.begin_test_step("Search available ISAs") self._search_area_step() self.end_test_step() + self.begin_test_step("Search subscriptions") + self._search_subscriptions_step() + self.end_test_step() + self.begin_test_step("Delete ISAs concurrently") - self._delete_isas() + self._delete_isas_step() + self.end_test_step() + + self.begin_test_step("Delete subscriptions concurrently") + self._delete_subscriptions_step() self.end_test_step() - self.begin_test_step("Access Deleted ISAs") - self._get_deleted_isas() + # TODO rename step to 'delete entities" + self.begin_test_step("Access deleted ISAs") + self._get_deleted_entities_step() self.end_test_step() - self.begin_test_step("Search Deleted ISAs") - self._search_deleted_isas() + self.begin_test_step("Search deleted ISAs") + self._search_deleted_isas_step() + self.end_test_step() + + self.begin_test_step("Search deleted subscriptions") + self._search_deleted_subscriptions_step() self.end_test_step() self.end_test_case() @@ -130,64 +213,48 @@ def run(self, context: ExecutionContext): def _delete_isas_if_exists(self): """Delete test ISAs if they exist. Done sequentially.""" for isa_id in self._isa_ids: - self._delete_isa_if_exists(isa_id) + delete_isa_if_exists( + self, + isa_id=isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + participant_id=self._dss_wrapper.participant_id, + ) - def _delete_isa_if_exists(self, isa_id): - fetched = fetch.isa( - isa_id, - rid_version=self._dss.rid_version, - session=self._dss.client, - participant_id=self._dss.participant_id, - ) - self.record_query(fetched.query) - with self.check("Successful ISA query", [self._dss.participant_id]) as check: - if not fetched.success and fetched.status_code != 404: - check.record_failed( - "ISA information could not be retrieved", - Severity.High, - f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", - query_timestamps=[fetched.query.request.timestamp], - ) + def _ensure_no_active_subs_exist(self): + """Ensure that we don't currently have any other active subscriptions at the DSS: + as there is a limit on how many simultaneous subscriptions we can create, + we want to avoid potentially reaching the limit during this scenario.""" - if fetched.success: - deleted = mutate.delete_isa( - isa_id, - fetched.isa.version, - self._dss.rid_version, - self._dss.client, - participant_id=self._dss.participant_id, + with self.check( + "Search for all subscriptions in ISA area", + [self._dss_wrapper.participant_id], + ) as check: + subs_in_area = self._dss_wrapper.search_subs( + check, + self._isa_area, ) - self.record_query(deleted.dss_query.query) - for subscriber_id, notification in deleted.notifications.items(): - self.record_query(notification.query) + + for sub_id, sub in subs_in_area.subscriptions.items(): with self.check( - "Removed pre-existing ISA", [self._dss.participant_id] + "Subscription can be deleted", [self._dss_wrapper.participant_id] ) as check: - if not deleted.dss_query.success: - check.record_failed( - "Could not delete pre-existing ISA", - Severity.High, - f"Attempting to delete ISA {isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}", - query_timestamps=[deleted.dss_query.query.request.timestamp], - ) - for subscriber_url, notification in deleted.notifications.items(): - # For checking the notifications, we ignore the request we made for the subscription that we created. - if self._isa.base_url not in subscriber_url: - pid = ( - notification.query.participant_id - if "participant_id" in notification.query - else None - ) - with self.check( - "Notified subscriber", [pid] if pid else [] - ) as check: - if not notification.success: - check.record_failed( - "Could not notify ISA subscriber", - Severity.Medium, - f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", - query_timestamps=[notification.query.request.timestamp], - ) + self._dss_wrapper.del_sub(check, sub_id, sub.version) + + def _delete_subscriptions_if_exists(self): + # Start by dropping any active sub + self._ensure_no_active_subs_exist() + # Check for subscriptions that will collide with our IDs and drop them + self._ensure_test_sub_ids_do_not_exist() + + def _ensure_test_sub_ids_do_not_exist(self): + """ + Ensures no subscription with the IDs we intend to use exist. + Note that expired subscriptions won't appear in searches, + which is why we need to explicitly test for their presence. + """ + for sub_id in self._sub_ids: + self._dss_wrapper.cleanup_sub(sub_id) def _get_isas_by_id_concurrent_step(self): loop = asyncio.get_event_loop() @@ -224,6 +291,72 @@ def _get_isas_by_id_concurrent_step(self): isa_id, fetched_isa, expected_version=self._isa_versions[isa_id] ) + def _get_subscriptions_by_id_concurrent_step(self): + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + asyncio.gather( + *[self._get_subscription(sub_id) for sub_id in self._sub_ids] + ) + ) + + results = typing.cast(Dict[str, FetchedSubscription], results) + + for _, fetched_sub in results: + self.record_query(fetched_sub.query) + + with self.check( + "Successful concurrent subscription queries", + [self._dss_wrapper.participant_id], + ) as main_check: + for sub_id, fetched_sub in results: + if fetched_sub.query.response.code != 200: + main_check.record_failed( + f"Subscription retrieval query failed for {sub_id}", + severity=Severity.High, + details=f"Subscription retrieval query for {sub_id} yielded code {fetched_sub.query.response.code}", + ) + + sub_validator = SubscriptionValidator( + main_check=main_check, + scenario=self, + sub_params=self._sub_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + owner=self._owner, + ) + + for sub_id, fetched_sub in results: + sub_validator.validate_fetched_subscription( + sub_id, + fetched_sub, + expected_version=self._current_subs[sub_id].version, + ) + + # With the creation of the second half of the ISAs, the notification indices of the + # already existing subscriptions are expected to be incremented by at least + # the amount of new ISAs: + with self.check( + "Notification indices incremented", [self._dss.participant_id] + ) as check: + for sub_id, fetched_sub in results: + # Only check the subscriptions created before the second ISAs are created + if sub_id in self._first_half_subs: + expected_notif_index = self._sub_notif_indices_before_second_half[ + sub_id + ] + len(self._second_half_isas) + # We don't check for equality, as other events may have caused the index to be increased. + if ( + fetched_sub.subscription.notification_index + < expected_notif_index + ): + check.record_failed( + f"Subscription {sub_id} notification index did not increment by at least {len(self._second_half_isas)}", + severity=Severity.High, + details=f"Subscription {sub_id} notification index was {fetched_sub.subscription.notification_index} " + f"when we expected at least {expected_notif_index}", + query_timestamps=[fetched_sub.query.request.timestamp], + ) + def _wrap_isa_get_query(self, q: Query) -> FetchedISA: """Wrap things into the correct utility class""" if self._dss.rid_version == RIDVersion.f3411_19: @@ -242,88 +375,130 @@ def _wrap_isa_put_query(self, q: Query, mutation: str) -> ChangedISA: else: raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + def _wrap_get_sub_query(self, q: Query) -> FetchedSubscription: + """Wrap things into the correct utility class""" + if self._dss.rid_version == RIDVersion.f3411_19: + return FetchedSubscription(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + return FetchedSubscription(v22a_query=q) + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + def _wrap_sub_put_query(self, q: Query, mutation: str) -> ChangedSubscription: + """Wrap things into the correct utility class""" + if self._dss.rid_version == RIDVersion.f3411_19: + return ChangedSubscription(mutation=mutation, v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + return ChangedSubscription(mutation=mutation, v22a_query=q) + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + async def _get_isa(self, isa_id): - async with SEMAPHORE: + async with ISA_SEMAPHORE: (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id) - # Build a `Request` object to register the query later on, - # although we don't need it to do the effective request here on the async_session - # This one is quite barebone and we need to check if anything needs to be added - r = requests.Request( - "GET", - url, - ) - prep = self._dss.client.prepare_request(r) - t0 = datetime.utcnow() - req_descr = describe_request(prep, t0) - status, headers, resp_json = await self._async_session.get_with_headers( - url=url, scope=self._read_scope() - ) - duration = datetime.utcnow() - t0 - rq = Query( - request=req_descr, - response=describe_aiohttp_response( - status, headers, resp_json, duration - ), - participant_id=self._dss.participant_id, - ) - return isa_id, self._wrap_isa_get_query(rq) + dq = await self._get_and_describe(url) + return isa_id, self._wrap_isa_get_query(dq) + + async def _get_subscription(self, sub_id): + (_, url) = mutate.build_subscription_url(sub_id, None, self._dss.rid_version) + dq = await self._get_and_describe(url) + return sub_id, self._wrap_get_sub_query(dq) + + async def _delete_subscription(self, sub_id, sub_version): + (_, url) = mutate.build_subscription_url( + sub_id, sub_version, self._dss.rid_version + ) + dq = await self._delete_and_describe(url, is_subscription_delete=True) + return sub_id, self._wrap_sub_put_query(dq, "delete") + + async def _get_and_describe(self, url): + r = requests.Request( + "GET", + url, + ) + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.get_with_headers( + url=url, scope=self._read_scope() + ) + duration = datetime.utcnow() - t0 + return Query( + request=req_descr, + response=describe_aiohttp_response(status, headers, resp_json, duration), + participant_id=self._dss.participant_id, + ) + + async def _put_and_describe(self, url, payload, is_subscription_put: bool = False): + r = requests.Request( + "PUT", + url, + json=payload, + ) + scope = self._write_scope() + # Although we effectively write to the DSS when creating a subscription, + # it expects a 'read' (or DisplayProvider) scope, or might fail. + if is_subscription_put: + scope = self._read_scope() + + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.put_with_headers( + url=url, json=payload, scope=scope + ) + duration = datetime.utcnow() - t0 + return Query( + request=req_descr, + response=describe_aiohttp_response(status, headers, resp_json, duration), + participant_id=self._dss.participant_id, + ) + + async def _delete_and_describe(self, url, is_subscription_delete: bool = False): + r = requests.Request( + "DELETE", + url, + ) + scope = self._write_scope() + # Although we effectively write to the DSS when deleting a subscription, + # it expects a 'read' (or DisplayProvider) scope, or might fail. + if is_subscription_delete: + scope = self._read_scope() + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.delete_with_headers( + url=url, scope=scope + ) + duration = datetime.utcnow() - t0 + return Query( + request=req_descr, + response=describe_aiohttp_response(status, headers, resp_json, duration), + participant_id=self._dss.participant_id, + ) + + async def _create_subscription(self, sub_id): + # No semaphore for subscriptions as we don't create that many: + (_, url) = mutate.build_subscription_url(sub_id, None, self._dss.rid_version) + payload = mutate.build_subscription_payload(sub_id, **self._sub_params) + dq = await self._put_and_describe(url, payload, is_subscription_put=True) + return sub_id, self._wrap_sub_put_query(dq, "create") async def _create_isa(self, isa_id): - async with SEMAPHORE: + async with ISA_SEMAPHORE: payload = mutate.build_isa_payload( **self._isa_params, rid_version=self._dss.rid_version, ) (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id) - r = requests.Request( - "PUT", - url, - json=payload, - ) - prep = self._dss.client.prepare_request(r) - t0 = datetime.utcnow() - req_descr = describe_request(prep, t0) - status, headers, resp_json = await self._async_session.put_with_headers( - url=url, json=payload, scope=self._write_scope() - ) - duration = datetime.utcnow() - t0 - rq = Query( - request=req_descr, - response=describe_aiohttp_response( - status, headers, resp_json, duration - ), - participant_id=self._dss.participant_id, - ) - ChangedISA( - mutation="create", - ) - return isa_id, self._wrap_isa_put_query(rq, "create") + dq = await self._put_and_describe(url, payload) + return isa_id, self._wrap_isa_put_query(dq, "create") async def _delete_isa(self, isa_id, isa_version): - async with SEMAPHORE: + async with ISA_SEMAPHORE: (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id, isa_version) - r = requests.Request( - "DELETE", - url, - ) - prep = self._dss.client.prepare_request(r) - t0 = datetime.utcnow() - req_descr = describe_request(prep, t0) - status, headers, resp_json = await self._async_session.delete_with_headers( - url=url, scope=self._write_scope() - ) - duration = datetime.utcnow() - t0 - rq = Query( - request=req_descr, - response=describe_aiohttp_response( - status, headers, resp_json, duration - ), - participant_id=self._dss.participant_id, - ) - ChangedISA( - mutation="create", - ) - return isa_id, self._wrap_isa_put_query(rq, "delete") + dq = await self._delete_and_describe(url) + return isa_id, self._wrap_isa_put_query(dq, "delete") def _write_scope(self): if self._dss.rid_version == RIDVersion.f3411_19: @@ -341,16 +516,149 @@ def _read_scope(self): else: raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") - def _create_isas_concurrent_step(self): - loop = asyncio.get_event_loop() - results = loop.run_until_complete( - asyncio.gather(*[self._create_isa(isa_id) for isa_id in self._isa_ids]) + def _create_entities_first_half_step(self): + """Create half of the ISAs and subscriptions concurrently. + Only check response format but not interactions between subscriptions and ISAs + """ + first_half_isas = self._isa_ids[: len(self._isa_ids) // 2] + first_half_subs = self._sub_ids[: len(self._sub_ids) // 2] + (isas, subs) = asyncio.get_event_loop().run_until_complete( + asyncio.gather( + self._create_isas_concurrently(first_half_isas), + self._create_subscriptions_concurrently(first_half_subs), + ) ) + # Record the query _after_ they ran in parallel. + # The scenario classes should not be considered thread-safe. + for _, created_isa in isas: + self.record_query(created_isa.query) + + for _, created_sub in subs: + self.record_query(created_sub.query) + + # Save the current subscription state: + for sub_id, changed_sub in subs: + self._current_subs[sub_id] = changed_sub.subscription + + def _create_entities_second_half_step(self): + """Create the other half of the ISAs and subscriptions concurrently. + Check that the responses at least mention the previously created ISAs and subscriptions + """ + second_half_isas = self._isa_ids[len(self._isa_ids) // 2 :] + second_half_subs = self._sub_ids[len(self._sub_ids) // 2 :] + + # Save the current notification indices + self._sub_notif_indices_before_second_half = { + sub_id: sub.notification_index for sub_id, sub in self._current_subs.items() + } + + (isas, subs) = asyncio.get_event_loop().run_until_complete( + asyncio.gather( + self._create_isas_concurrently(second_half_isas), + self._create_subscriptions_concurrently(second_half_subs), + ) + ) + isas = typing.cast(Dict[str, ChangedISA], isas) + subs = typing.cast(Dict[str, ChangedSubscription], subs) - results = typing.cast(Dict[str, ChangedISA], results) + # Record the query _after_ they ran in parallel. + # The scenario classes should not be considered thread-safe. + for _, created_isa in isas: + self.record_query(created_isa.query) - for _, fetched_isa in results: - self.record_query(fetched_isa.query) + for _, created_sub in subs: + self.record_query(created_sub.query) + + # Check ISAs mention the subscription we created earlier + with self.check( + "Created ISAs mention subscriptions known to exist", + [self._dss.participant_id], + ) as check: + for isa_id, new_isa in isas: + subs_for_new_isa = new_isa.sub_ids + for expected_sub_id in self._first_half_subs: + if expected_sub_id not in subs_for_new_isa: + check.record_failed( + f"ISA {isa_id} does not mention subscription {expected_sub_id}", + severity=Severity.High, + details=f"ISA {isa_id} was created after some subscriptions were successfully created " + f"for the same area, but does not mention them." + f"Expected at least subscriptions: {self._first_half_subs}, found {subs_for_new_isa}", + query_timestamps=[new_isa.query.request.timestamp], + ) + + # Check subscriptions mention the ISAs we created earlier + with self.check( + "Created subscriptions mention ISAs known to exist", + [self._dss.participant_id], + ) as check: + for sub_id, new_sub in subs: + isas_for_new_sub = [isa.id for isa in new_sub.isas] + for expected_isa_id in self._first_half_isas: + if expected_isa_id not in isas_for_new_sub: + check.record_failed( + f"Subscription {sub_id} does not mention ISA {expected_isa_id}", + severity=Severity.High, + details=f"Subscription {sub_id} was created after some ISAs were successfully created " + f"for the same area, but does not mention them." + f"Expected at least ISAs: {self._first_half_isas}, found {isas_for_new_sub}", + query_timestamps=[new_sub.query.request.timestamp], + ) + + async def _create_subscriptions_concurrently( + self, subs_to_create: List[str] + ) -> Dict[str, ChangedSubscription]: + results = await asyncio.gather( + *[self._create_subscription(sub_id) for sub_id in subs_to_create] + ) + results = typing.cast(Dict[str, ChangedSubscription], results) + + with self.check( + "Concurrent subscriptions creation", [self._dss_wrapper.participant_id] + ) as main_check: + for sub_id, changed_sub in results: + if changed_sub.query.response.code != 200: + loguru.logger.error( + f"Failed sub req: {changed_sub.query.request.json}" + ) + loguru.logger.error( + f"Failed sub rep: {changed_sub.query.response.body}" + ) + main_check.record_failed( + f"Subscription creation failed for {sub_id}", + severity=Severity.High, + details=f"Subscription creation for {sub_id} returned {changed_sub.query.response.code} " + f"Query request: {changed_sub.query.request.json} " + f"Query response: {changed_sub.query.response.json}", + ) + else: + self._current_subs[sub_id] = changed_sub.subscription + + isa_validator = SubscriptionValidator( + main_check=main_check, + scenario=self, + sub_params=self._sub_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + owner=self._owner, + ) + + for sub_id, changed_sub in results: + isa_validator.validate_mutated_subscription( + sub_id, changed_sub, previous_version=None + ) + pass + + return results + + async def _create_isas_concurrently( + self, isas_to_create: List[str] + ) -> Dict[str, ChangedISA]: + results = await asyncio.gather( + *[self._create_isa(isa_id) for isa_id in isas_to_create] + ) + + results = typing.cast(Dict[str, ChangedISA], results) with self.check( "Concurrent ISAs creation", [self._dss_wrapper.participant_id] @@ -378,6 +686,8 @@ def _create_isas_concurrent_step(self): isa_id, changed_isa, previous_version=None ) + return results + def _search_area_step(self): with self.check( "Successful ISAs search", [self._dss_wrapper.participant_id] @@ -411,7 +721,50 @@ def _search_area_step(self): isas, expected_versions=self._isa_versions ) - def _delete_isas(self): + def _search_subscriptions_step(self): + with self.check( + "Successful subscriptions search", [self._dss_wrapper.participant_id] + ) as main_check: + subs = self._dss_wrapper.search_subs( + main_check, + area=self._isa_area, + ) + + with self.check( + "Correct subscriptions returned by search", + [self._dss_wrapper.participant_id], + ) as sub_check: + for sub_id in self._sub_ids: + if sub_id not in subs.subscriptions.keys(): + sub_check.record_failed( + f"Subscriptions search did not return subscription {sub_id} that was previously created", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned subscriptions {subs.subscriptions.keys()} and is missing some of the created subscriptions", + query_timestamps=[subs.dss_query.query.request.timestamp], + ) + + sub_validator = SubscriptionValidator( + main_check=main_check, + scenario=self, + sub_params=self._sub_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + owner=self._owner, + ) + + sub_validator.validate_searched_subscriptions( + subs, current_subs=self._current_subs + ) + + def _delete_isas_step(self): + + # Before deleting the ISAS, take a snapshot of the subscriptions notification indices: + + # Save the current notification indices + self._sub_notif_indices_before_deletion = { + sub_id: sub.notification_index for sub_id, sub in self._current_subs.items() + } + loop = asyncio.get_event_loop() results = loop.run_until_complete( asyncio.gather( @@ -451,29 +804,125 @@ def _delete_isas(self): isa_id, changed_isa, expected_version=self._isa_versions[isa_id] ) - def _get_deleted_isas(self): - + def _delete_subscriptions_step(self): + """ + Delete all the subscriptions, concurrently, then check their respective notification indices. + """ loop = asyncio.get_event_loop() results = loop.run_until_complete( - asyncio.gather(*[self._get_isa(isa_id) for isa_id in self._isa_ids]) + asyncio.gather( + *[ + self._delete_subscription( + sub_id, self._current_subs[sub_id].version + ) + for sub_id in self._sub_ids + ] + ) ) - results = typing.cast(Dict[str, ChangedISA], results) + results = typing.cast(Dict[str, ChangedSubscription], results) - for _, fetched_isa in results: + for _, fetched_sub in results: + self.record_query(fetched_sub.query) + + with self.check( + "Subscriptions deletion query success", [self._dss_wrapper.participant_id] + ) as main_check: + for sub_id, deleted_sub in results: + if deleted_sub.query.response.code != 200: + main_check.record_failed( + f"Subscription deletion failed for {sub_id}", + severity=Severity.High, + details=f"Subscription deletion for {sub_id} returned {deleted_sub.query.response.code}", + ) + + sub_validator = SubscriptionValidator( + main_check=main_check, + scenario=self, + sub_params=self._sub_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + owner=self._owner, + ) + + for sub_id, changed_sub in results: + sub_validator.validate_deleted_subscription( + sub_id, + changed_sub, + expected_version=self._current_subs[sub_id].version, + ) + + # Check that all subscriptions' notification indices have been + # updated after the ISA's were deleted: + with self.check( + "Notification indices incremented", [self._dss.participant_id] + ) as check: + for sub_id, fetched_sub in results: + expected_notif_index = self._sub_notif_indices_before_deletion[ + sub_id + ] + len(self._isa_ids) + # We don't check for equality, as other events may have caused the index to be increased. + if fetched_sub.subscription.notification_index < expected_notif_index: + check.record_failed( + f"Subscription {sub_id} notification index did not increment by at least {len(self._isa_ids)}", + severity=Severity.High, + details=f"Subscription {sub_id} notification index was {fetched_sub.subscription.notification_index} " + f"when we expected at least {expected_notif_index}", + query_timestamps=[fetched_sub.query.request.timestamp], + ) + + def _get_deleted_entities_step(self): + """Queries the ISAs and subscriptions that were previously deleted and expects + all queries to fail. + """ + loop = asyncio.get_event_loop() + (isas, subs) = loop.run_until_complete( + asyncio.gather(self._get_deleted_isas(), self._get_deleted_subscriptions()) + ) + isas = typing.cast(Dict[str, ChangedISA], isas) + subs = typing.cast(Dict[str, ChangedSubscription], subs) + + for _, fetched_isa in isas: self.record_query(fetched_isa.query) + for _, fetched_sub in subs: + self.record_query(fetched_sub.query) + with self.check("ISAs not found", [self._dss_wrapper.participant_id]) as check: - for isa_id, fetched_isa in results: + for isa_id, fetched_isa in isas: if fetched_isa.status_code != 404: check.record_failed( f"ISA retrieval succeeded for {isa_id}", severity=Severity.High, - details=f"ISA retrieval for {isa_id} returned {fetched_isa.status_code}", + details=f"ISA retrieval for {isa_id} returned {fetched_isa.status_code} " + f"when we expected 404, as the ISA has been deleted", query_timestamps=[fetched_isa.query.request.timestamp], ) - def _search_deleted_isas(self): + with self.check( + "Subscriptions not found", [self._dss_wrapper.participant_id] + ) as check: + for sub_id, fetched_sub in subs: + if fetched_sub.query.response.code != 404: + check.record_failed( + f"Subscription retrieval succeeded for {sub_id}", + severity=Severity.High, + details=f"Subscription retrieval for {sub_id} returned {fetched_sub.query.response.code} " + f"when we expected 404, as the subscription has been deleted", + query_timestamps=[fetched_sub.query.request.timestamp], + ) + + async def _get_deleted_isas(self): + return await asyncio.gather( + *[self._get_isa(isa_id) for isa_id in self._isa_ids] + ) + + async def _get_deleted_subscriptions(self): + return await asyncio.gather( + *[self._get_subscription(sub_id) for sub_id in self._sub_ids] + ) + + def _search_deleted_isas_step(self): with self.check( "Successful ISAs search", [self._dss_wrapper.participant_id] ) as check: @@ -494,9 +943,31 @@ def _search_deleted_isas(self): query_timestamps=[isas.dss_query.query.request.timestamp], ) + def _search_deleted_subscriptions_step(self): + with self.check( + "Successful subscriptions search", [self._dss_wrapper.participant_id] + ) as check: + subs = self._dss_wrapper.search_subs( + check, + area=self._isa_area, + ) + + with self.check( + "Subscriptions not returned by search", [self._dss_wrapper.participant_id] + ) as check: + for sub_id in self._sub_ids: + if sub_id in subs.subscriptions.keys(): + check.record_failed( + f"Subscriptions search returned deleted subscription {sub_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned subscriptions {subs.subscriptions.keys()} that contained some of the subscriptions we had previously deleted.", + query_timestamps=[subs.dss_query.query.request.timestamp], + ) + def cleanup(self): self.begin_cleanup() + self._delete_subscriptions_if_exists() self._delete_isas_if_exists() self._async_session.close() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py index 17e12fc569..a2c2221de9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -3,10 +3,17 @@ from monitoring.monitorlib import schema_validation from monitoring.monitorlib.fetch import rid as fetch -from monitoring.monitorlib.fetch.rid import ISA, FetchedISA, FetchedISAs +from monitoring.monitorlib.fetch.rid import ( + ISA, + FetchedISA, + FetchedISAs, + Subscription, + FetchedSubscription, + FetchedSubscriptions, +) from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.mutate import rid as mutate -from monitoring.monitorlib.mutate.rid import ChangedISA +from monitoring.monitorlib.mutate.rid import ChangedISA, ChangedSubscription from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.scenarios.scenario import ( @@ -319,3 +326,271 @@ def validate_searched_isas( fetched_isas.query.request.timestamp, expected_version=expected_versions[isa_id], ) + + +class SubscriptionValidator(object): + """Wraps the validation logic for a subscription that was returned by the DSS. + It will compare the returned subscription with the parameters specified at its creation. + """ + + _main_check: PendingCheck + _scenario: GenericTestScenario + _sub_params: Dict[str, any] + _dss_id: [str] + _rid_version: RIDVersion + _owner: str + + def __init__( + self, + main_check: PendingCheck, + scenario: GenericTestScenario, + sub_params: Dict[str, any], + dss_id: str, + rid_version: RIDVersion, + owner: str, + ): + self._main_check = main_check + self._scenario = scenario + self._sub_params = sub_params + self._dss_id = [dss_id] + self._rid_version = rid_version + self._owner = owner + + def _fail_sub_check( + self, _sub_check: PendingCheck, _summary: str, _details: str, t_dss: datetime + ) -> None: + """Fails with Medium severity the sub_check and with High severity the main check.""" + + _sub_check.record_failed( + summary=_summary, + severity=Severity.Medium, + details=_details, + query_timestamps=[t_dss], + ) + + self._main_check.record_failed( + summary=f"Subscription request succeeded, but the DSS response is not valid: {_summary}", + severity=Severity.High, + details=_details, + query_timestamps=[t_dss], + ) + + def _validate_subscription( + self, + expected_sub_id: str, + sub: Subscription, + t_dss: datetime, + previous_version: Optional[ + str + ] = None, # If set, we control that the version changed + expected_version: Optional[ + str + ] = None, # If set, we control that the version has not changed + ) -> None: + + dss_id = self._dss_id + with self._scenario.check("Subscription ID matches", dss_id) as sub_check: + if expected_sub_id != sub.id: + self._fail_sub_check( + sub_check, + "DSS did not return correct subscription", + f"Expected subscription ID {expected_sub_id} but got {sub.id}", + t_dss, + ) + + with self._scenario.check( + "Subscription start time matches", dss_id + ) as sub_check: + expected_start = self._sub_params["start_time"] + if abs((sub.time_start - expected_start).total_seconds()) > MAX_SKEW: + self._fail_sub_check( + sub_check, + f"DSS returned subscription (ID {expected_sub_id}) with incorrect start time", + f"DSS should have returned a subscription with a start time of {expected_start}, but instead the subscription returned had a start time of {sub.time_start}", + t_dss, + ) + + with self._scenario.check("Subscription end time matches", dss_id) as sub_check: + expected_end = self._sub_params["end_time"] + if abs((sub.time_end - expected_end).total_seconds()) > MAX_SKEW: + self._fail_sub_check( + sub_check, + f"DSS returned subscription (ID {expected_sub_id}) with incorrect end time", + f"DSS should have returned a subscription with an end time of {expected_end}, but instead the subscription returned had an end time of {sub.time_end}", + t_dss, + ) + + if previous_version is not None: + with self._scenario.check( + "Subscription version changed", dss_id + ) as sub_check: + if sub.version == previous_version: + self._fail_sub_check( + sub_check, + f"Subscription (ID {expected_sub_id}) version was not updated", + f"Got old version {previous_version} while expecting new version", + t_dss, + ) + + if expected_version is not None: + with self._scenario.check( + "Subscription version matches", dss_id + ) as sub_check: + if sub.version != expected_version: + self._fail_sub_check( + sub_check, + f"Subscription (ID {expected_sub_id}) version is not the previously held one, although no modification was done to the subscription", + f"Got old version {sub.version} while expecting {expected_version}", + t_dss, + ) + + with self._scenario.check( + "Subscription flights url matches", dss_id + ) as sub_check: + # TODO Confirm that the base URL should contain the + # ID in the path for v22a + expected_isa_url = self._rid_version.post_isa_url_of( + self._sub_params["uss_base_url"], expected_sub_id + ) + if sub.isa_url != expected_isa_url: + self._fail_sub_check( + sub_check, + f"DSS returned subscription (ID {expected_sub_id}) with incorrect ISA URL", + f"DSS should have returned a subscription with an ISA URL of {expected_isa_url}, but instead the subscription returned had an ISA URL of {sub.isa_url}", + t_dss, + ) + + with self._scenario.check("Subscription owner matches", dss_id) as sub_check: + if sub.owner != self._owner: + self._fail_sub_check( + sub_check, + f"DSS returned subscription (ID {expected_sub_id}) with incorrect owner", + f"DSS should have returned a subscription with an owner of {self._owner}, but instead the subscription returned had an owner of {sub.owner}", + t_dss, + ) + + def validate_fetched_subscription( + self, + expected_sub_id: str, + sub: FetchedSubscription, + expected_version: str, + ): + # Validate schema of Subscription: + t_dss = sub.query.request.timestamp + with self._scenario.check( + "Subscription response format", self._dss_id + ) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_get_subscription_response_path, + sub.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + self._fail_sub_check( + sub_check, + "GET Subscription response format was invalid", + "Found the following schema validation errors in the DSS response:\n" + + details, + t_dss, + ) + + self._validate_subscription( + expected_sub_id, sub.subscription, t_dss, expected_version=expected_version + ) + + def validate_mutated_subscription( + self, + expected_sub_id: str, + mutated_sub: ChangedSubscription, + previous_version: Optional[str] = None, + ): + # Validate schema of Subscription: + t_dss = mutated_sub.query.request.timestamp + with self._scenario.check( + "Subscription response format", self._dss_id + ) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_put_subscription_response_path, + mutated_sub.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + self._fail_sub_check( + sub_check, + "Mutated Subscription response format was invalid", + "Found the following schema validation errors in the DSS response:\n" + + details, + t_dss, + ) + + self._validate_subscription( + expected_sub_id, + mutated_sub.subscription, + t_dss, + previous_version=previous_version, + expected_version=None, + ) + + def validate_deleted_subscription( + self, + expected_sub_id: str, + sub: ChangedSubscription, + expected_version: str, + ): + # Validate schema of Subscription: + t_dss = sub.query.request.timestamp + with self._scenario.check( + "Delete subscription response format", self._dss_id + ) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_delete_subscription_response_path, + sub.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + self._fail_sub_check( + sub_check, + "DELETE Subscription response format was invalid", + "Found the following schema validation errors in the DSS response:\n" + + details, + t_dss, + ) + + self._validate_subscription( + expected_sub_id, sub.subscription, t_dss, expected_version=expected_version + ) + + def validate_searched_subscriptions( + self, + fetched_subscriptions: FetchedSubscriptions, + current_subs: Dict[str, Subscription], + ): + with self._scenario.check( + "Subscriptions response format", self._dss_id + ) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_search_subscriptions_response_path, + fetched_subscriptions.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + + self._fail_sub_check( + sub_check, + "GET Subscriptions response format was invalid", + "Found the following schema validation errors in the DSS response:\n" + + details, + fetched_subscriptions.query.request.timestamp, + ) + + for sub_id, expected_sub in current_subs.items(): + self._validate_subscription( + sub_id, + fetched_subscriptions.subscriptions[sub_id], + fetched_subscriptions.query.request.timestamp, + expected_version=expected_sub.version, + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index d2c3dacfc9..4bfff718a9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -1018,7 +1018,7 @@ def cleanup_sub( Severity.Medium, ) - if sub.status_code == 404: + if sub.status_code in [404, 500]: return None with self._scenario.check( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md index 7757a0302e..7b32b917b8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -22,7 +22,19 @@ Create, query and delete ISAs on the DSS, concurrently. ### Ensure clean workspace test step -This scenario creates ISA's with known IDs. This step ensures that no ISA with a known ID is present in the DSS before proceeding with the test. +This scenario creates ISA's and subscriptions with known IDs. This step ensures that no ISA or subscription with a known ID is present in the DSS before proceeding with the test. + +#### Search for all subscriptions in ISA area check + +If a correct request for subscriptions in the parametrized ISA's area fails, the **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the GET Subscriptions endpoint is not met. + +#### Subscription can be queried by ID check + +If a subscription created by the client cannot be queried, the **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the GET Subscription endpoint is not met. + +#### Subscription can be deleted check + +If a subscription created by the client cannot be deleted, the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the DELETE Subscription endpoint is not met. #### Successful ISA query check @@ -36,10 +48,12 @@ If an ISA with the intended ID is already present in the DSS, it needs to be rem When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. -## Concurrent Requests test case +## Concurrent requests test case This test case will: +TODO rewrite description once scenario is implemented + 1. Create ISAs concurrently 2. Query each ISA individually, but concurrently 3. Search for all ISAs in the area of the created ISAs (using a single request) @@ -53,7 +67,44 @@ This step attempts to concurrently create multiple ISAs, as specified in this sc #### Concurrent ISAs creation check -If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. +If any of the concurrent ISA creation requests fails or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Concurrent subscriptions creation check + +If any of the concurrent subscription creation requests fails or leads to the creation of an incorrect subscription, the PUT Subscription DSS endpoint in **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Subscription response format check + +The API for **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +When the subscription is created, the DSS returns the ID of the subscription in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. ### Get ISAs concurrently test step @@ -91,7 +142,60 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. -### [Search Available ISAs test step](test_steps/search_isas.md) +### Get subscriptions concurrently test step + +Verify that the created subscriptions exist and are as expected + +#### Successful concurrent subscription queries check + +If any of the concurrently run queries for the previously created subscription fails, the GET Subscription DSS endpoint in **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Subscription response format check + +The API for **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the resource path, **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Notification indices incremented check + +Subscriptions that exist when ISAs are created must have their notification index incremented for each new created or updated ISA that +overlaps with their defined area. + +If after the creation or mutation of an ISA within the subscription's area the DSS does not increment the subscription's notification index, the DSS is in violation of **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)**, +and this check will fail. + +### [Search available ISAs test step](test_steps/search_isas.md) This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. @@ -127,6 +231,55 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. +### Search subscriptions test step + +Checks that created subscriptions are properly returned by the DSS's search endpoint + +#### Successful subscriptions search check + +The subscription search parameters are valid, as such the subscriptions search should be successful. If the request is not successful, this check will fail per **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. + +#### Correct subscriptions returned by search check + +The search request is expected to return all subscriptions created by the test. If it does not, the **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the GET Subscriptions endpoint is not met. + +#### Subscriptions response format check + +The API for **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the resource path, **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + ### [Delete ISAs concurrently test step](test_steps/delete_isa.md) This step attempts to concurrently delete the earlier created ISAs. @@ -151,15 +304,95 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. -### Access Deleted ISAs test step +### Delete subscriptions concurrently test step + +#### Subscriptions deletion query success check + +If any of the concurrent deletion queries fails, the DELETE Subscription DSS endpoint in **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Delete subscription response format check + +The API for **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the deleted resource path, **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** correctly, and this check will fail. + +#### Notification indices incremented check + +Subscriptions that exist when ISAs are deleted must have their notification index incremented for each deleted ISA. + +If the subscription is then deleted, the DSS is expected to return the correct notification index in the response to the DELETE Subscription request. + +If the DSS returns an incorrect notification index, it is not implementing **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** properly, and this check will fail. + +### Access deleted ISAs test step This step attempts to concurrently access the previously deleted ISAs from the DSS. #### ISAs not found check -The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as the DSS violates **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Subscriptions not found check -### [Search Deleted ISAs test step](test_steps/search_isas.md) +The subscription fetch request was about a deleted subscription, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as the DSS violates **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search deleted ISAs test step](test_steps/search_isas.md) This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. @@ -171,6 +404,19 @@ The ISA search parameters are valid, as such the search should be successful. If The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. +### Search deleted subscriptions test step + +This check issues a search for subscriptions in the area of the previously deleted subscriptions. + +#### Successful subscriptions search check + +The subscription search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscriptions not returned by search check + +The subscription search area covers the area for which previous subscriptions were created, but these have been deleted. +If any of the deleted subscriptions is present in the result, the DSS is not implementing **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** properly. + ## Cleanup The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. @@ -186,3 +432,15 @@ If an ISA with the intended ID is still present in the DSS, it needs to be remov ### Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. + +### Search for all subscriptions in ISA area check + +If a correct request for subscriptions in the parametrized ISA's area fails, the **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the GET Subscriptions endpoint is not met. + +### Subscription can be deleted check + +If a subscription created by the client cannot be deleted, the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the DELETE Subscription endpoint is not met. + +### Subscription can be queried by ID check + +If a subscription created by the client cannot be queried, the **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the GET Subscription endpoint is not met. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md index 3c7177ea0a..e53264e7ea 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -22,7 +22,19 @@ Create, query and delete ISAs on the DSS, concurrently. ### Ensure clean workspace test step -This scenario creates ISA's with known IDs. This step ensures that no ISA with a known ID is present in the DSS before proceeding with the test. +This scenario creates ISA's and subscriptions with known IDs. This step ensures that no ISA or subscription with a known ID is present in the DSS before proceeding with the test. + +#### Search for all subscriptions in ISA area check + +If a correct request for subscriptions in the parametrized ISA's area fails, the **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the GET Subscriptions endpoint is not met. + +#### Subscription can be queried by ID check + +If a subscription created by the client cannot be queried, the **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the GET Subscription endpoint is not met. + +#### Subscription can be deleted check + +If a subscription created by the client cannot be deleted, the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the DELETE Subscription endpoint is not met. #### Successful ISA query check @@ -36,10 +48,12 @@ If an ISA with the intended ID is already present in the DSS, it needs to be rem When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. -## Concurrent Requests test case +## Concurrent requests test case This test case will: +TODO rewrite description once scenario is implemented + 1. Create ISAs concurrently 2. Query each ISA individually, but concurrently 3. Search for all ISAs in the area of the created ISAs (using a single request) @@ -53,7 +67,44 @@ This step attempts to concurrently create multiple ISAs, as specified in this sc #### Concurrent ISAs creation check -If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. +If any of the concurrent ISA creation requests fails or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Concurrent subscriptions creation check + +If any of the concurrent subscription creation requests fails or leads to the creation of an incorrect subscription, the PUT Subscription DSS endpoint in **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Subscription response format check + +The API for **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +When the subscription is created, the DSS returns the ID of the subscription in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. ### Get ISAs concurrently test step @@ -91,7 +142,60 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -### [Search Available ISAs test step](test_steps/search_isas.md) +### Get subscriptions concurrently test step + +Verify that the created subscriptions exist and are as expected + +#### Successful concurrent subscription queries check + +If any of the concurrently run queries for the previously created subscription fails, the GET Subscription DSS endpoint in **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Subscription response format check + +The API for **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the resource path, **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Notification indices incremented check + +Subscriptions that exist when ISAs are created must have their notification index incremented for each new created or updated ISA that +overlaps with their defined area. + +If after the creation or mutation of an ISA within the subscription's area the DSS does not increment the subscription's notification index, the DSS is in violation of **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)**, +and this check will fail. + +### [Search available ISAs test step](test_steps/search_isas.md) This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. @@ -127,6 +231,55 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. +### Search subscriptions test step + +Checks that created subscriptions are properly returned by the DSS's search endpoint + +#### Successful subscriptions search check + +The subscription search parameters are valid, as such the subscriptions search should be successful. If the request is not successful, this check will fail per **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Correct subscriptions returned by search check + +The search request is expected to return all subscriptions created by the test. If it does not, the **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the GET Subscriptions endpoint is not met. + +#### Subscriptions response format check + +The API for **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the resource path, **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + ### [Delete ISAs concurrently test step](test_steps/delete_isa.md) This step attempts to concurrently delete the earlier created ISAs. @@ -151,15 +304,95 @@ The ISA creation request specified an exact end time, so the DSS should have cre When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -### Access Deleted ISAs test step +### Delete subscriptions concurrently test step + +#### Subscriptions deletion query success check + +If any of the concurrent deletion queries fails, the DELETE Subscription DSS endpoint in **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Delete subscription response format check + +The API for **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### Subscription ID matches check + +If the returned subscription ID does not match the ID requested in the deleted resource path, **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Created ISAs mention subscriptions known to exist check + +ISAs created *after* a subscription has been successfully created for the same area must contain the subscription ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Created subscriptions mention ISAs known to exist check + +Subscriptions created *after* an ISA has been successfully created for the same area must contain the ISA ID, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription start time matches check + +The subscription creation request specified an exact start time slightly past now, so the DSS should have created a subscription starting at exactly that time. If the DSS response indicates the subscription start time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription end time matches check + +The subscription creation request specified an exact end time, so the DSS should have created a subscription ending at exactly that time. If the DSS response indicates the subscription end time is not this value, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription flights url matches check + +The subscription creation request specified the base URL for the flights endpoint. If the DSS response does not contain the proper URL, **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### Subscription owner matches check + +The DSS indicates in every returned subscription the identify of its owner. If this identity does not correspond to the one that is used by the USS qualifier to authenticate to the DSS, +the DSS is not implementing **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Subscription version matches check + +If the version of the subscription has changed without any update having been done, the DSS might not be implementing **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** correctly, and this check will fail. + +#### Notification indices incremented check + +Subscriptions that exist when ISAs are deleted must have their notification index incremented for each deleted ISA. + +If the subscription is then deleted, the DSS is expected to return the correct notification index in the response to the DELETE Subscription request. + +If the DSS returns an incorrect notification index, it is not implementing **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** properly, and this check will fail. + +### Access deleted ISAs test step This step attempts to concurrently access the previously deleted ISAs from the DSS. #### ISAs not found check -The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as the DSS violates **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Subscriptions not found check -### [Search Deleted ISAs test step](test_steps/search_isas.md) +The subscription fetch request was about a deleted subscription, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as the DSS violates **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search deleted ISAs test step](test_steps/search_isas.md) This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. @@ -171,6 +404,19 @@ The ISA search parameters are valid, as such the search should be successful. If The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. +### Search deleted subscriptions test step + +This check issues a search for subscriptions in the area of the previously deleted subscriptions. + +#### Successful subscriptions search check + +The subscription search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscriptions not returned by search check + +The subscription search area covers the area for which previous subscriptions were created, but these have been deleted. +If any of the deleted subscriptions is present in the result, the DSS is not implementing **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** properly. + ## Cleanup The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. @@ -186,3 +432,15 @@ If an ISA with the intended ID is still present in the DSS, it needs to be remov ### Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. + +### Search for all subscriptions in ISA area check + +If a correct request for subscriptions in the parametrized ISA's area fails, the **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the GET Subscriptions endpoint is not met. + +### Subscription can be deleted check + +If a subscription created by the client cannot be deleted, the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the DELETE Subscription endpoint is not met. + +### Subscription can be queried by ID check + +If a subscription created by the client cannot be queried, the **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the GET Subscription endpoint is not met. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 185f6ae133..b6fd96cf44 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -34,22 +34,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index 485b008ad1..0d60eccbcc 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -4,16 +4,16 @@ ## [Actions](../../../README.md#actions) -1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v19/dss/isa_simple.md) ([`scenarios.astm.netrid.v19.dss.ISASimple`](../../../../scenarios/astm/netrid/v19/dss/isa_simple.py)) -2. Scenario: [ASTM NetRID DSS: Submitted ISA Validations](../../../../scenarios/astm/netrid/v19/dss/isa_validation.md) ([`scenarios.astm.netrid.v19.dss.ISAValidation`](../../../../scenarios/astm/netrid/v19/dss/isa_validation.py)) -3. Scenario: [ASTM NetRID DSS: ISA Expiry](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.md) ([`scenarios.astm.netrid.v19.dss.ISAExpiry`](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.py)) -4. Scenario: [ASTM NetRID DSS: ISA Subscription Interactions](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md) ([`scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions`](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py)) -5. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) -6. Scenario: [ASTM NetRID DSS: Subscription Simple](../../../../scenarios/astm/netrid/v19/dss/subscription_simple.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionSimple`](../../../../scenarios/astm/netrid/v19/dss/subscription_simple.py)) -7. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) -8. Scenario: [ASTM NetRID DSS: Token Validation](../../../../scenarios/astm/netrid/v19/dss/token_validation.md) ([`scenarios.astm.netrid.v19.dss.TokenValidation`](../../../../scenarios/astm/netrid/v19/dss/token_validation.py)) -9. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) -10. Scenario: [ASTM NetRID DSS: Concurrent Requests](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md) ([`scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent`](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py)) +1. Scenario: [ASTM NetRID DSS: Subscription Simple](../../../../scenarios/astm/netrid/v19/dss/subscription_simple.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionSimple`](../../../../scenarios/astm/netrid/v19/dss/subscription_simple.py)) +2. Scenario: [ASTM NetRID DSS: Concurrent Requests](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md) ([`scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent`](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py)) +3. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v19/dss/isa_simple.md) ([`scenarios.astm.netrid.v19.dss.ISASimple`](../../../../scenarios/astm/netrid/v19/dss/isa_simple.py)) +4. Scenario: [ASTM NetRID DSS: Submitted ISA Validations](../../../../scenarios/astm/netrid/v19/dss/isa_validation.md) ([`scenarios.astm.netrid.v19.dss.ISAValidation`](../../../../scenarios/astm/netrid/v19/dss/isa_validation.py)) +5. Scenario: [ASTM NetRID DSS: ISA Expiry](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.md) ([`scenarios.astm.netrid.v19.dss.ISAExpiry`](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.py)) +6. Scenario: [ASTM NetRID DSS: ISA Subscription Interactions](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md) ([`scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions`](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py)) +7. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) +8. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) +9. Scenario: [ASTM NetRID DSS: Token Validation](../../../../scenarios/astm/netrid/v19/dss/token_validation.md) ([`scenarios.astm.netrid.v19.dss.TokenValidation`](../../../../scenarios/astm/netrid/v19/dss/token_validation.py)) +10. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -38,22 +38,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml index 78918859b6..978cd9a2ab 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml @@ -7,44 +7,50 @@ resources: problematically_big_area: resources.VerticesResource actions: - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.ISASimple + scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionSimple resources: dss: dss id_generator: id_generator isa: isa problematically_big_area: problematically_big_area - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.ISAValidation + scenario_type: scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent + resources: + dss: dss + id_generator: id_generator + isa: isa + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.dss.ISASimple resources: dss: dss id_generator: id_generator isa: isa problematically_big_area: problematically_big_area - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.ISAExpiry + scenario_type: scenarios.astm.netrid.v19.dss.ISAValidation resources: dss: dss id_generator: id_generator isa: isa + problematically_big_area: problematically_big_area - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions + scenario_type: scenarios.astm.netrid.v19.dss.ISAExpiry resources: dss: dss id_generator: id_generator isa: isa - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation + scenario_type: scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions resources: dss: dss id_generator: id_generator isa: isa - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionSimple + scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation resources: dss: dss id_generator: id_generator isa: isa - problematically_big_area: problematically_big_area - test_scenario: scenario_type: scenarios.astm.netrid.v19.DSSInteroperability resources: @@ -60,9 +66,3 @@ actions: scenario_type: scenarios.astm.netrid.v19.dss.CRDBAccess resources: {} on_failure: Continue - - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent - resources: - dss: dss - id_generator: id_generator - isa: isa diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 74282404f8..6b3a82fd29 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -55,22 +55,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md index a8a2f25bdd..2de53fe665 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md @@ -59,22 +59,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md index 1cb4e858e6..8cc245e7db 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md @@ -32,22 +32,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -223,22 +223,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 4c6abe3b89..9e16a1df8b 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -50,22 +50,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050 diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 710953fe93..6685446531 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -52,22 +52,22 @@ DSS0030,c Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0030,e Implemented - ASTM NetRID DSS: Subscription Simple + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: Subscription Simple DSS0030,f Implemented - ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation DSS0050