From 24bd9ee50e4acdcba08d224636177974485a8a36 Mon Sep 17 00:00:00 2001 From: Christopher Chiche Date: Fri, 13 Oct 2023 16:06:49 +0200 Subject: [PATCH 1/2] Update schemas --- reports/src/types/TestRunReport.d.ts | 1268 +++++++++-------- schemas/TODO.md | 2 +- schemas/monitorlib/dicts/RemovedElement.json | 18 + schemas/monitorlib/fetch/Query.json | 60 +- .../ActionGeneratorDefinition.json | 34 +- .../configuration/ArtifactsConfiguration.json | 71 +- .../configuration/ReportConfiguration.json | 20 +- .../ReportHTMLConfiguration.json | 16 +- .../SequenceViewConfiguration.json | 19 + .../TemplatedReportConfiguration.json | 35 + .../TemplatedReportInjectedConfiguration.json | 12 + .../configuration/TestConfiguration.json | 22 +- .../TestedRequirementsConfiguration.json | 49 + .../configuration/TestedRole.json | 32 - .../TestedRolesConfiguration.json | 26 +- .../uss_qualifier/reports/graphs/Node.json | 44 - .../reports/report/ActionGeneratorReport.json | 44 +- .../reports/report/PassedCheck.json | 36 +- .../reports/report/SkippedActionReport.json | 29 + .../reports/report/TestCaseReport.json | 38 +- .../reports/report/TestRunReport.json | 49 +- .../reports/report/TestScenarioReport.json | 94 +- .../reports/report/TestSuiteReport.json | 74 +- .../definitions/TestScenarioDeclaration.json | 36 +- .../definitions/TestCaseDocumentation.json | 31 - .../definitions/TestCheckDocumentation.json | 31 - .../TestScenarioDocumentation.json | 50 - .../definitions/TestStepDocumentation.json | 31 - .../ParticipantRequirementPerformance.json | 30 - .../requirements/TestedRequirement.json | 33 - .../TestSuiteActionDeclaration.json | 46 +- 31 files changed, 1225 insertions(+), 1155 deletions(-) create mode 100644 schemas/monitorlib/dicts/RemovedElement.json rename schemas/uss_qualifier/{suites => action_generators}/definitions/ActionGeneratorDefinition.json (64%) create mode 100644 schemas/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json create mode 100644 schemas/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json create mode 100644 schemas/uss_qualifier/configurations/configuration/TemplatedReportInjectedConfiguration.json create mode 100644 schemas/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json delete mode 100644 schemas/uss_qualifier/configurations/configuration/TestedRole.json delete mode 100644 schemas/uss_qualifier/reports/graphs/Node.json create mode 100644 schemas/uss_qualifier/reports/report/SkippedActionReport.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/definitions/TestCaseDocumentation.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/definitions/TestCheckDocumentation.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/definitions/TestScenarioDocumentation.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/definitions/TestStepDocumentation.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/requirements/ParticipantRequirementPerformance.json delete mode 100644 schemas/uss_qualifier/scenarios/documentation/requirements/TestedRequirement.json diff --git a/reports/src/types/TestRunReport.d.ts b/reports/src/types/TestRunReport.d.ts index 5f6ea16..f167942 100644 --- a/reports/src/types/TestRunReport.d.ts +++ b/reports/src/types/TestRunReport.d.ts @@ -17,8 +17,8 @@ export interface ReportsReportTestRunReport { * Signature of the test run including codebase version and all file signatures except excluded environmental files. * * This field can be used to identify that a particular expected test baseline (codebase, all non-environmental inputs) - * was run. The value of this field is computed from codebase_version and all elements of file_signatures that are not - * explicitly excluded as environmental configuration. + * was run. The value of this field is computed from codebase_version and all elements of the configuration that are + * not explicitly excluded as environmental configuration. */ baseline_signature: string; /** @@ -29,354 +29,481 @@ export interface ReportsReportTestRunReport { * Full commit hash of codebase used to run uss_qualifier */ commit_hash: string; - report: ReportsReportTestSuiteActionReport; configuration: ConfigurationsConfigurationTestConfiguration; /** - * Mapping between the names of files loaded during test run and the SHA-1 hashes of those files' content. + * Signature of the environmental inputs of the configuration not included in the baseline signature. */ - file_signatures: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - }; + environment_signature: string; + report: ReportsReportTestSuiteActionReport; [k: string]: unknown; } /** - * Report produced by configured test action + * Configuration used to run uss_qualifier */ -export interface ReportsReportTestSuiteActionReport { +export interface ConfigurationsConfigurationTestConfiguration { /** * Path to content that replaces the $ref */ // $ref?: string; + action: SuitesDefinitionsTestSuiteActionDeclaration; /** - * If this action was a test scenario, this field will hold its report - */ - test_scenario?: null | ReportsReportTestScenarioReport; - /** - * If this action was an action generator, this field will hold its report + * List of portions of the configuration that should not be considered when computing the test baseline signature (e.g., environmental definitions). */ - action_generator?: null | ReportsReportActionGeneratorReport; - /** - * If this action was a test suite, this field will hold its report - */ - test_suite?: null | ReportsReportTestSuiteReport; + non_baseline_inputs?: string[] | null; + resources: ResourcesDefinitionsResourceCollection; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.TestScenarioReport, as defined in monitoring/uss_qualifier/reports/report.py + * The action this test configuration wants to run (usually a test suite) */ -export interface ReportsReportTestScenarioReport { +export interface SuitesDefinitionsTestSuiteActionDeclaration { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * True iff test scenario completed normally with no failed checks - */ - successful?: boolean; - /** - * Time at which the test scenario started + * If this field is populated, declaration of a generator that will produce 0 or more test suite actions */ - start_time: string; + action_generator?: null | ActionGeneratorsDefinitionsActionGeneratorDefinition; /** - * Reports for each of the test cases in this test scenario + * What to do if this action fails */ - cases: ReportsReportTestCaseReport[]; + on_failure?: "Continue" | "Abort"; /** - * URL at which this test scenario is described + * If this field is populated, declaration of the test scenario to run */ - documentation_url: string; + test_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; /** - * If there was an error while executing this test scenario, this field describes the error + * If this field is populated, declaration of the test suite to run */ - execution_error?: null | ReportsReportErrorReport; + test_suite?: null | SuitesDefinitionsTestSuiteDeclaration; + [k: string]: unknown; +} +/** + * monitoring.uss_qualifier.action_generators.definitions.ActionGeneratorDefinition, as defined in monitoring/uss_qualifier/action_generators/definitions.py + */ +export interface ActionGeneratorsDefinitionsActionGeneratorDefinition { /** - * Time at which the test scenario completed or encountered an error + * Path to content that replaces the $ref */ - end_time?: string | null; + // $ref?: string; /** - * If this test scenario performed cleanup, this report captures the relevant information. + * Type of action generator */ - cleanup?: null | ReportsReportTestStepReport; + generator_type: string; /** - * Additional information about this scenario that may be useful + * Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent test suite. + * + * The child action resource ID is supplied by the parent test suite resource ID . + * + * Resources not included in this field will not be available to the child action. + * + * If the parent resource ID is suffixed with ? then the resource will not be required (and will not be populated for the child action when not present in the parent) */ - notes?: { + resources: { /** * Path to content that replaces the $ref */ // $ref?: string; - [k: string]: ReportsReportNote; - } | null; - /** - * Type of this test scenario - */ - scenario_type: string; + [k: string]: string; + }; /** - * Name of this test scenario + * Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type` */ - name: string; + specification?: { + [k: string]: unknown; + }; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.TestCaseReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py */ -export interface ReportsReportTestCaseReport { +export interface ScenariosDefinitionsTestScenarioDeclaration { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Name of this test case + * Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite. + * + * The additional argument to concrete test scenario constructor is supplied by the parent suite resource . */ - name: string; + resources?: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [k: string]: string; + } | null; /** - * URL at which this test case is described + * Type of test scenario. */ - documentation_url: string; + scenario_type: string; + [k: string]: unknown; +} +/** + * monitoring.uss_qualifier.suites.definitions.TestSuiteDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py + */ +export interface SuitesDefinitionsTestSuiteDeclaration { /** - * Time at which the test case completed or encountered an error + * Path to content that replaces the $ref */ - end_time?: string | null; + // $ref?: string; /** - * Time at which the test case started + * Definition of test suite internal to the configuration -- specified instead of `suite_type`. */ - start_time: string; + suite_definition?: null | SuitesDefinitionsTestSuiteDefinition; /** - * Reports for each of the test steps in this test case + * Mapping of the ID a resource will be known by in the child test suite -> the ID a resource is known by in the parent test suite. + * + * The child suite resource is supplied by the parent suite resource . */ - steps: ReportsReportTestStepReport[]; + resources: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [k: string]: string; + }; + /** + * Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder + */ + suite_type?: string | null; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.TestStepReport, as defined in monitoring/uss_qualifier/reports/report.py + * Schema for the definition of a test suite, analogous to the Python TestScenario subclass for scenarios + * + * monitoring.uss_qualifier.suites.definitions.TestSuiteDefinition, as defined in monitoring/uss_qualifier/suites/definitions.py */ -export interface ReportsReportTestStepReport { +export interface SuitesDefinitionsTestSuiteDefinition { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * The checks which failed in this test step - */ - failed_checks: ReportsReportFailedCheck[]; - /** - * Name of this test step + * Name of the test suite */ name: string; /** - * URL at which this test step is described - */ - documentation_url: string; - /** - * Time at which the test step completed or encountered an error + * The scenario executed after all the actions that evaluates the test suite report. Must be a ReportEvaluationScenario. */ - end_time?: string | null; + report_evaluation_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; /** - * Description of HTTP requests relevant to this issue + * Definitions of capabilities verified by this test suite for individual participants. */ - queries?: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchQueryJson[] | null; + participant_verifiable_capabilities?: + | ReportsCapabilityDefinitionsParticipantCapabilityDefinition[] + | null; /** - * Time at which the test step started + * Enumeration of the resources used by this test suite */ - start_time: string; + resources: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [k: string]: string; + }; /** - * The checks which successfully passed in this test step + * The actions to take when running the test suite. Components will be executed in order. */ - passed_checks: ReportsReportPassedCheck[]; + actions: SuitesDefinitionsTestSuiteActionDeclaration1[]; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.FailedCheck, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.capability_definitions.ParticipantCapabilityDefinition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py */ -export interface ReportsReportFailedCheck { +export interface ReportsCapabilityDefinitionsParticipantCapabilityDefinition { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Name of the check that failed - */ - name: string; - /** - * URL at which the check which failed is described - */ - documentation_url: string; - /** - * Human-readable summary of the issue - */ - summary: string; - /** - * Human-readable description of the issue + * Human-readable description of the capability. */ - details: string; + description: string; + verification_condition: ReportsCapabilityDefinitionsCapabilityVerificationCondition; /** - * List of the `report` timestamp field for queries relevant to this failed check + * Human-readable name of the capability. */ - query_report_timestamps?: string[] | null; + name: string; /** - * How severe the issue is + * Identifier of this capability, unique at the level in which this capability is defined. */ - severity: "Critical" | "High" | "Medium" | "Low"; + id: string; + [k: string]: unknown; +} +/** + * Condition required in order to verify the capability. + */ +export interface ReportsCapabilityDefinitionsCapabilityVerificationCondition { /** - * Participants that may not meet the relevant requirements due to this failed check + * Path to content that replaces the $ref */ - participants: string[]; + // $ref?: string; + no_failed_checks?: null | ReportsCapabilityDefinitionsNoFailedChecksCondition; + any_conditions?: null | ReportsCapabilityDefinitionsAnyCondition; + requirements_checked?: null | ReportsCapabilityDefinitionsRequirementsCheckedCondition; + all_conditions?: null | ReportsCapabilityDefinitionsAllConditions; + capability_verified?: null | ReportsCapabilityDefinitionsCapabilityVerifiedCondition; + [k: string]: unknown; +} +/** + * Condition will only be satisfied if there are no applicable failed checks. + * + * For a capability to be verified for a participant, only checks including the participant's ID will be considered. + * + * monitoring.uss_qualifier.reports.capability_definitions.NoFailedChecksCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + */ +export interface ReportsCapabilityDefinitionsNoFailedChecksCondition { /** - * Requirements that are not met due to this failed check + * Path to content that replaces the $ref */ - requirements: string[]; + // $ref?: string; + [k: string]: unknown; +} +/** + * Condition will be satisfied when any of the specified conditions are satisfied. + * + * Note that an empty list of conditions will result in an unsuccessful evaluation. + * + * monitoring.uss_qualifier.reports.capability_definitions.AnyCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + */ +export interface ReportsCapabilityDefinitionsAnyCondition { /** - * Time the issue was discovered + * Path to content that replaces the $ref */ - timestamp: string; + // $ref?: string; + conditions: ReportsCapabilityDefinitionsCapabilityVerificationCondition1[]; + [k: string]: unknown; +} +/** + * Specification of a single condition used to determine whether a capability should be verified. + * + * Exactly one field must be specified. + * + * monitoring.uss_qualifier.reports.capability_definitions.CapabilityVerificationCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + */ +export interface ReportsCapabilityDefinitionsCapabilityVerificationCondition1 { /** - * Additional data, structured according to the checks' needs, that may be relevant for understanding this failed check + * Path to content that replaces the $ref */ - additional_data?: { - [k: string]: unknown; - } | null; + // $ref?: string; + no_failed_checks?: null | ReportsCapabilityDefinitionsNoFailedChecksCondition; + any_conditions?: null | ReportsCapabilityDefinitionsAnyCondition; + requirements_checked?: null | ReportsCapabilityDefinitionsRequirementsCheckedCondition; + all_conditions?: null | ReportsCapabilityDefinitionsAllConditions; + capability_verified?: null | ReportsCapabilityDefinitionsCapabilityVerifiedCondition; [k: string]: unknown; } /** - * monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py + * Condition will only be satisfied if at least one successful check exists for all specified requirements. + * + * Note that an empty collection of requirements will result in an unsuccessful evaluation. + * + * monitoring.uss_qualifier.reports.capability_definitions.RequirementsCheckedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py */ -export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchQueryJson { +export interface ReportsCapabilityDefinitionsRequirementsCheckedCondition { /** * Path to content that replaces the $ref */ // $ref?: string; - response: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchResponseDescriptionJson; - request: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchRequestDescriptionJson; + checked: RequirementsDefinitionsRequirementCollection; [k: string]: unknown; } /** - * monitoring.monitorlib.fetch.ResponseDescription, as defined in monitoring/monitorlib/fetch/__init__.py + * Each requirement contained within this collection must be covered by at least one successful check. */ -export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchResponseDescriptionJson { +export interface RequirementsDefinitionsRequirementCollection { /** * Path to content that replaces the $ref */ // $ref?: string; - code?: number | null; - failure?: string | null; - headers?: { - [k: string]: unknown; - } | null; - json?: { - [k: string]: unknown; - } | null; - elapsed_s: number; - body?: string | null; - reported: string; + /** + * This collection includes all of the requirements in all of these requirement collections. + */ + requirement_collections?: + | RequirementsDefinitionsRequirementCollection1[] + | null; + /** + * This collection includes all of these requirements. + */ + requirements?: string[] | null; + /** + * This collection includes all requirements in all of these requirement sets. + */ + requirement_sets?: string[] | null; + /** + * This collection does not include any of these requirements, despite all previous fields. + */ + exclude?: null | RequirementsDefinitionsRequirementCollection1; [k: string]: unknown; } /** - * monitoring.monitorlib.fetch.RequestDescription, as defined in monitoring/monitorlib/fetch/__init__.py + * monitoring.uss_qualifier.requirements.definitions.RequirementCollection, as defined in monitoring/uss_qualifier/requirements/definitions.py */ -export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchRequestDescriptionJson { +export interface RequirementsDefinitionsRequirementCollection1 { /** * Path to content that replaces the $ref */ // $ref?: string; - initiated_at?: string | null; - received_at?: string | null; - headers?: { - [k: string]: unknown; - } | null; - json?: { - [k: string]: unknown; - } | null; - method: string; - url: string; - body?: string | null; + /** + * This collection includes all of the requirements in all of these requirement collections. + */ + requirement_collections?: + | RequirementsDefinitionsRequirementCollection1[] + | null; + /** + * This collection includes all of these requirements. + */ + requirements?: string[] | null; + /** + * This collection includes all requirements in all of these requirement sets. + */ + requirement_sets?: string[] | null; + /** + * This collection does not include any of these requirements, despite all previous fields. + */ + exclude?: null | RequirementsDefinitionsRequirementCollection1; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py + * Condition will only be satisfied when all specified conditions are satisfied. + * + * Note that an empty list of conditions will result in a successful evaluation. + * + * monitoring.uss_qualifier.reports.capability_definitions.AllConditions, as defined in monitoring/uss_qualifier/reports/capability_definitions.py */ -export interface ReportsReportPassedCheck { +export interface ReportsCapabilityDefinitionsAllConditions { /** * Path to content that replaces the $ref */ // $ref?: string; + conditions: ReportsCapabilityDefinitionsCapabilityVerificationCondition1[]; + [k: string]: unknown; +} +/** + * Condition will be satisfied when the specified capability is verified. + * + * Note that a capability which do not declare any requirement will result in an unsuccessful evaluation. + * + * monitoring.uss_qualifier.reports.capability_definitions.CapabilityVerifiedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + */ +export interface ReportsCapabilityDefinitionsCapabilityVerifiedCondition { /** - * Participants that may not have met the relevant requirements if this check had failed + * Path to content that replaces the $ref */ - participants: string[]; + // $ref?: string; /** - * Name of the check that passed + * Location of report to inspect for the verification of the specified capability, relative to the report in which + * the capability is defined. Implicit default value is "$" (look for verified capability in the report in which the + * dependant capability is defined). + * + * If this location resolves to multiple TestSuiteReports, then the capability must be verified in all resolved reports + * in order for this condition to be satisfied. When this location resolves to artifacts that are not + * TestSuiteReports, those artifacts will be ignored. + * + * Note that capabilities are verified in the order they are defined. So, if capability B defined in a particular + * location depends on whether capability A in that same location is granted, capability A must be defined before + * capability B is defined. Also note that capability verifications are computed as test components are completed. + * Since a parent test component (e.g., test suite) is not complete until all of its child components are complete, a + * descendant test component's capability condition cannot depend on whether an ancestor's (e.g., parent's) capability + * is verified. */ - name: string; + capability_location?: string | null; /** - * Requirements that would not have been met if this check had failed + * List of identifier of capability that must be verified for this condition to be satisfied. */ - requirements: string[]; + capability_ids: string[]; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.ErrorReport, as defined in monitoring/uss_qualifier/reports/report.py + * Defines a step in the sequence of things to do for a test suite. + * + * Exactly one of `test_scenario`, `test_suite`, or `action_generator` must be specified. + * + * monitoring.uss_qualifier.suites.definitions.TestSuiteActionDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py */ -export interface ReportsReportErrorReport { +export interface SuitesDefinitionsTestSuiteActionDeclaration1 { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Error message + * If this field is populated, declaration of a generator that will produce 0 or more test suite actions */ - message: string; + action_generator?: null | ActionGeneratorsDefinitionsActionGeneratorDefinition; /** - * Full stack trace of error + * What to do if this action fails */ - stacktrace: string; + on_failure?: "Continue" | "Abort"; /** - * Type of error + * If this field is populated, declaration of the test scenario to run */ - type: string; + test_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; /** - * Time at which the error was logged + * If this field is populated, declaration of the test suite to run */ - timestamp: string; + test_suite?: null | SuitesDefinitionsTestSuiteDeclaration; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.Note, as defined in monitoring/uss_qualifier/reports/report.py + * Declarations for resources used by the test suite */ -export interface ReportsReportNote { +export interface ResourcesDefinitionsResourceCollection { /** * Path to content that replaces the $ref */ // $ref?: string; - message: string; - timestamp: string; + /** + * Mapping of globally (within resource collection) unique name identifying a resource to the declaration of that resource + */ + resource_declarations: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [ + k: string + ]: ResourcesDefinitionsResourceDeclaration; + }; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.ActionGeneratorReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.resources.definitions.ResourceDeclaration, as defined in monitoring/uss_qualifier/resources/definitions.py */ -export interface ReportsReportActionGeneratorReport { +export interface ResourcesDefinitionsResourceDeclaration { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Type of action generator + * Mapping of dependency parameter (additional argument to concrete resource constructor) to `name` of resource to use */ - generator_type: string; + dependencies?: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [k: string]: string; + }; /** - * Reports from the actions generated by the action generator + * Type of resource, expressed as a Python class name qualified relative to this `resources` module */ - actions: ReportsReportTestSuiteActionReport1[]; + resource_type: string; + /** + * Specification of resource; format is the SpecificationType that corresponds to the `resource_type` + */ + specification?: { + [k: string]: unknown; + }; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.TestSuiteActionReport, as defined in monitoring/uss_qualifier/reports/report.py + * Report produced by configured test action */ -export interface ReportsReportTestSuiteActionReport1 { +export interface ReportsReportTestSuiteActionReport { /** * Path to content that replaces the $ref */ @@ -396,778 +523,753 @@ export interface ReportsReportTestSuiteActionReport1 { [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.TestSuiteReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.TestScenarioReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportTestSuiteReport { +export interface ReportsReportTestScenarioReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * True iff test suite completed normally with no failed checks + * Reports for each of the test cases in this test scenario, in chronological order. */ - successful?: boolean; + cases: ReportsReportTestCaseReport[]; /** - * Time at which the test suite started + * If this test scenario performed cleanup, this report captures the relevant information. */ - start_time: string; + cleanup?: null | ReportsReportTestStepReport; /** - * URL at which this test suite is described + * URL at which this test scenario is described */ documentation_url: string; /** - * List of capabilities defined in this test suite, evaluated for each participant. + * Time at which the test scenario completed or encountered an error */ - capability_evaluations: ReportsReportParticipantCapabilityEvaluationReport[]; + end_time?: string | null; /** - * Time at which the test suite completed + * If there was an error while executing this test scenario, this field describes the error */ - end_time?: string | null; + execution_error?: null | ReportsReportErrorReport; /** - * Reports from test scenarios and test suites comprising the test suite for this report + * Name of this test scenario */ - actions: ReportsReportTestSuiteActionReport1[]; + name: string; /** - * Type/location of this test suite + * Additional information about this scenario that may be useful */ - suite_type: string; + notes?: { + /** + * Path to content that replaces the $ref + */ + // $ref?: string; + [k: string]: ReportsReportNote; + } | null; /** - * Name of this test suite + * Type of this test scenario */ - name: string; + scenario_type: string; + /** + * Time at which the test scenario started + */ + start_time: string; + /** + * True iff test scenario completed normally with no failed checks + */ + successful?: boolean; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.report.ParticipantCapabilityEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.TestCaseReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportParticipantCapabilityEvaluationReport { +export interface ReportsReportTestCaseReport { /** * Path to content that replaces the $ref */ // $ref?: string; - condition_evaluation: ReportsReportParticipantCapabilityConditionEvaluationReport; /** - * Whether the capability was successfully verified. + * URL at which this test case is described */ - verified: boolean; + documentation_url: string; /** - * ID of participant for which capability is being evaluated. + * Time at which the test case completed or encountered an error */ - participant_id: string; + end_time?: string | null; /** - * ID of capability being evaluated. + * Name of this test case */ - capability_id: string; + name: string; + /** + * Time at which the test case started + */ + start_time: string; + /** + * Reports for each of the test steps in this test case, in chronological order. + */ + steps: ReportsReportTestStepReport[]; [k: string]: unknown; } /** - * Report produced by evaluating the condition for verifying this capability. + * monitoring.uss_qualifier.reports.report.TestStepReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportParticipantCapabilityConditionEvaluationReport { +export interface ReportsReportTestStepReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * When specified, the condition evaluated was CapabilityVerifiedCondition. + * The checks which failed in this test step */ - capability_verified?: null | ReportsReportCapabilityVerifiedConditionEvaluationReport; + failed_checks: ReportsReportFailedCheck[]; /** - * When specified, the condition evaluated was NoFailedChecksCondition. + * Name of this test step */ - no_failed_checks?: null | ReportsReportNoFailedChecksConditionEvaluationReport; + name: string; /** - * Whether the condition was satisfied for the relevant participant. + * URL at which this test step is described */ - condition_satisfied: boolean; + documentation_url: string; /** - * When specified, the condition evaluated was AnyCondition. + * Time at which the test step completed or encountered an error */ - any_conditions?: null | ReportsReportAnyConditionEvaluationReport; + end_time?: string | null; /** - * When specified, the condition evaluated was RequirementsCheckedCondition. + * Description of HTTP requests relevant to this issue */ - requirements_checked?: null | ReportsReportRequirementsCheckedConditionEvaluationReport; + queries?: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchQueryJson[] | null; /** - * When specified, the condition evaluated was AllConditions. + * Time at which the test step started */ - all_conditions?: null | ReportsReportAllConditionsEvaluationReport; + start_time: string; + /** + * The checks which successfully passed in this test step + */ + passed_checks: ReportsReportPassedCheck[]; [k: string]: unknown; } /** - * Result of an evaluation of a CapabilityVerifiedCondition dependent on whether other capabilities were verified. - * - * monitoring.uss_qualifier.reports.report.CapabilityVerifiedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.FailedCheck, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportCapabilityVerifiedConditionEvaluationReport { +export interface ReportsReportFailedCheck { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * All capability evaluations checked for this condition. + * Name of the check that failed */ - checked_capabilities: ReportsReportCheckedCapability[]; + name: string; /** - * Report elements matching the condition's `capability_location`, but not of the type TestSuiteReport. + * URL at which the check which failed is described */ - spurious_matches: ReportsReportSpuriousReportMatch[]; + documentation_url: string; /** - * Capabilities specified for this condition but not found in the report. + * Human-readable summary of the issue */ - missing_capabilities: string[]; - [k: string]: unknown; -} -/** - * Existing/previous participant-verifiable capability upon which a CapabilityVerifiedCondition depends. - * - * monitoring.uss_qualifier.reports.report.CheckedCapability, as defined in monitoring/uss_qualifier/reports/report.py - */ -export interface ReportsReportCheckedCapability { + summary: string; /** - * Path to content that replaces the $ref + * Human-readable description of the issue */ - // $ref?: string; + details: string; /** - * Whether this capability was successfully verified + * List of the `report` timestamp field for queries relevant to this failed check */ - capability_verified: boolean; + query_report_timestamps?: string[] | null; /** - * The location of the ParticipantCapabilityConditionEvaluationReport for the capability, relative to the TestSuiteReport in which this checked requirement is located. + * How severe the issue is */ - capability_location: string; + severity: "Critical" | "High" | "Medium" | "Low"; /** - * Location of the ParticipantCapabilityEvaluationReport for the existing/previous capability, relative to the TestSuiteReport in which the CapabilityVerifiedConditionEvaluationReport containing this CheckedCapability is located. + * Participants that may not meet the relevant requirements due to this failed check */ - report_location: string; + participants: string[]; /** - * ID of the existing/previous participant-verifiable capability. + * Requirements that are not met due to this failed check */ - capability_id: string; + requirements: string[]; + /** + * Time the issue was discovered + */ + timestamp: string; + /** + * Additional data, structured according to the checks' needs, that may be relevant for understanding this failed check + */ + additional_data?: { + [k: string]: unknown; + } | null; [k: string]: unknown; } /** - * Participant-verifiable capability evaluations are only present in TestSuiteReports. If a CapabilityVerifiedCondition points to a report element that is not a TestSuiteReport, an instance of this class will be generated. - * - * monitoring.uss_qualifier.reports.report.SpuriousReportMatch, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py */ -export interface ReportsReportSpuriousReportMatch { +export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchQueryJson { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Data type of the report element (not TestSuiteReport). - */ - type: string; + * If specified, the recognized type of this query. + */ + query_type?: + | ( + | "astm.f3411.v22a.sp.flights" + | "astm.f3411.v19.sp.flights" + | "astm.f3411.v22a.sp.flight_details" + | "astm.f3411.v19.sp.flight_details" + | "astm.f3548.v21.dss.queryOperationalIntentReferences" + | "astm.f3548.v21.dss.getOperationalIntentReference" + | "astm.f3548.v21.dss.createOperationalIntentReference" + | "astm.f3548.v21.dss.updateOperationalIntentReference" + | "astm.f3548.v21.dss.deleteOperationalIntentReference" + | "astm.f3548.v21.dss.queryConstraintReferences" + | "astm.f3548.v21.dss.getConstraintReference" + | "astm.f3548.v21.dss.createConstraintReference" + | "astm.f3548.v21.dss.updateConstraintReference" + | "astm.f3548.v21.dss.deleteConstraintReference" + | "astm.f3548.v21.dss.querySubscriptions" + | "astm.f3548.v21.dss.getSubscription" + | "astm.f3548.v21.dss.createSubscription" + | "astm.f3548.v21.dss.updateSubscription" + | "astm.f3548.v21.dss.deleteSubscription" + | "astm.f3548.v21.dss.makeDssReport" + | "astm.f3548.v21.uss.setUssAvailability" + | "astm.f3548.v21.uss.getUssAvailability" + | "astm.f3548.v21.uss.getOperationalIntentDetails" + | "astm.f3548.v21.uss.getOperationalIntentTelemetry" + | "astm.f3548.v21.uss.notifyOperationalIntentDetailsChanged" + | "astm.f3548.v21.uss.getConstraintDetails" + | "astm.f3548.v21.uss.notifyConstraintDetailsChanged" + | "astm.f3548.v21.uss.makeUssReport" + ) + | null; + request: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchRequestDescriptionJson; + response: HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchResponseDescriptionJson; /** - * Location of the non-TestSuiteReport report element matching the CapabilityVerifiedCondition's `capability_location`, relative to the TestSuiteReport in which this condition is located. + * If specified, identifier of the USS/participant hosting the server involved in this query. */ - location: string; + server_id?: string | null; [k: string]: unknown; } /** - * Result of an evaluation of NoFailedChecksCondition dependent on whether any checks failed within the scope of the test suite in which this condition is located. - * - * monitoring.uss_qualifier.reports.report.NoFailedChecksConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.monitorlib.fetch.RequestDescription, as defined in monitoring/monitorlib/fetch/__init__.py */ -export interface ReportsReportNoFailedChecksConditionEvaluationReport { +export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchRequestDescriptionJson { /** * Path to content that replaces the $ref */ // $ref?: string; - /** - * The location of each FailedCheck, relative to the TestSuiteReport in which this report is located. - */ - failed_checks: string[]; + initiated_at?: string | null; + received_at?: string | null; + headers?: { + [k: string]: unknown; + } | null; + json?: { + [k: string]: unknown; + } | null; + method: string; + url: string; + body?: string | null; [k: string]: unknown; } /** - * Result of an evaluation of AnyCondition determined by whether any of the subconditions are satisfied. - * - * monitoring.uss_qualifier.reports.report.AnyConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.monitorlib.fetch.ResponseDescription, as defined in monitoring/monitorlib/fetch/__init__.py */ -export interface ReportsReportAnyConditionEvaluationReport { +export interface HttpsGithubComInterussMonitoringBlobMainSchemasMonitoringMonitorlibFetchResponseDescriptionJson { /** * Path to content that replaces the $ref */ // $ref?: string; - /** - * Which of the specified options were satisfied (if any were satisfied, then this condition should be satisfied). - */ - satisfied_options: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; - /** - * Which of the specified options were not satisfied (these are informational only and do not affect the evaluation). - */ - unsatisfied_options: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; + code?: number | null; + failure?: string | null; + headers?: { + [k: string]: unknown; + } | null; + json?: { + [k: string]: unknown; + } | null; + elapsed_s: number; + body?: string | null; + reported: string; [k: string]: unknown; -} -/** - * Result of an evaluation of a condition related to whether a participant capability should be verified. - * - * Exactly one field other than `condition_satisfied` must be specified. - * - * monitoring.uss_qualifier.reports.report.ParticipantCapabilityConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py +} +/** + * monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportParticipantCapabilityConditionEvaluationReport1 { +export interface ReportsReportPassedCheck { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * When specified, the condition evaluated was CapabilityVerifiedCondition. - */ - capability_verified?: null | ReportsReportCapabilityVerifiedConditionEvaluationReport; - /** - * When specified, the condition evaluated was NoFailedChecksCondition. - */ - no_failed_checks?: null | ReportsReportNoFailedChecksConditionEvaluationReport; - /** - * Whether the condition was satisfied for the relevant participant. + * Name of the check that passed */ - condition_satisfied: boolean; + name: string; /** - * When specified, the condition evaluated was AnyCondition. + * Participants that may not have met the relevant requirements if this check had failed */ - any_conditions?: null | ReportsReportAnyConditionEvaluationReport; + participants: string[]; /** - * When specified, the condition evaluated was RequirementsCheckedCondition. + * Requirements that would not have been met if this check had failed */ - requirements_checked?: null | ReportsReportRequirementsCheckedConditionEvaluationReport; + requirements: string[]; /** - * When specified, the condition evaluated was AllConditions. + * Time the issue was discovered */ - all_conditions?: null | ReportsReportAllConditionsEvaluationReport; + timestamp: string; [k: string]: unknown; } /** - * Result of an evaluation of RequirementsCheckedCondition dependent on whether a set of requirements were successfully checked. - * - * monitoring.uss_qualifier.reports.report.RequirementsCheckedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.ErrorReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportRequirementsCheckedConditionEvaluationReport { +export interface ReportsReportErrorReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Requirements with only PassedChecks. + * Error message */ - passed_requirements: ReportsReportCheckedRequirement[]; + message: string; /** - * Requirements with FailedChecks. + * Full stack trace of error */ - failed_requirements: ReportsReportCheckedRequirement[]; + stacktrace: string; /** - * Requirements that didn't have any PassedChecks or FailedChecks within the scope of the test suite in which this condition is located. + * Type of error */ - untested_requirements: string[]; + type: string; + /** + * Time at which the error was logged + */ + timestamp: string; [k: string]: unknown; } /** - * A single requirement being checked for participant-verifiable capability verification. - * - * monitoring.uss_qualifier.reports.report.CheckedRequirement, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.Note, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportCheckedRequirement { +export interface ReportsReportNote { /** * Path to content that replaces the $ref */ // $ref?: string; - /** - * The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located. - */ - passed_checks: string[]; - /** - * The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located. - */ - failed_checks: string[]; - /** - * The requirement being checked. - */ - requirement_id: string; + message: string; + timestamp: string; [k: string]: unknown; } /** - * Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied. - * - * monitoring.uss_qualifier.reports.report.AllConditionsEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py + * monitoring.uss_qualifier.reports.report.ActionGeneratorReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsReportAllConditionsEvaluationReport { +export interface ReportsReportActionGeneratorReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * All of the conditions that were satisfied (there must be at least one). + * Reports from the actions generated by the action generator, in order of execution. */ - satisfied_conditions: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; + actions: ReportsReportTestSuiteActionReport1[]; /** - * All of the conditions that were unsatisfied (if any, then this condition will not be satisfied). + * Time at which the action generator completed or encountered an error */ - unsatisfied_conditions: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; + end_time?: string | null; + /** + * Type of action generator + */ + generator_type: string; + /** + * Time at which the action generator started + */ + start_time: string; + /** + * True iff all actions completed normally with no failed checks + */ + successful?: boolean; [k: string]: unknown; } /** - * Configuration used to run uss_qualifier + * monitoring.uss_qualifier.reports.report.TestSuiteActionReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ConfigurationsConfigurationTestConfiguration { +export interface ReportsReportTestSuiteActionReport1 { /** * Path to content that replaces the $ref */ // $ref?: string; - action: SuitesDefinitionsTestSuiteActionDeclaration; /** - * List of file inputs that should not be considered when computing the test baseline signature (e.g., environmental definitions). + * If this action was a test scenario, this field will hold its report */ - non_baseline_inputs?: string[] | null; - resources: ResourcesDefinitionsResourceCollection; + test_scenario?: null | ReportsReportTestScenarioReport; + /** + * If this action was an action generator, this field will hold its report + */ + action_generator?: null | ReportsReportActionGeneratorReport; + /** + * If this action was a test suite, this field will hold its report + */ + test_suite?: null | ReportsReportTestSuiteReport; [k: string]: unknown; } /** - * The action this test configuration wants to run (usually a test suite) + * monitoring.uss_qualifier.reports.report.TestSuiteReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface SuitesDefinitionsTestSuiteActionDeclaration { +export interface ReportsReportTestSuiteReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * If this field is populated, declaration of a generator that will produce 0 or more test suite actions + * Reports from test scenarios and test suites comprising the test suite for this report, in order of execution. */ - action_generator?: null | SuitesDefinitionsActionGeneratorDefinition; + actions: ReportsReportTestSuiteActionReport1[]; /** - * If this field is populated, declaration of the test suite to run + * List of capabilities defined in this test suite, evaluated for each participant. */ - test_suite?: null | SuitesDefinitionsTestSuiteDeclaration; + capability_evaluations: ReportsReportParticipantCapabilityEvaluationReport[]; /** - * What to do if this action fails + * URL at which this test suite is described */ - on_failure?: "Continue" | "Abort"; + documentation_url: string; /** - * If this field is populated, declaration of the test scenario to run + * Time at which the test suite completed */ - test_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; - [k: string]: unknown; -} -/** - * monitoring.uss_qualifier.suites.definitions.ActionGeneratorDefinition, as defined in monitoring/uss_qualifier/suites/definitions.py - */ -export interface SuitesDefinitionsActionGeneratorDefinition { + end_time?: string | null; /** - * Path to content that replaces the $ref + * Name of this test suite */ - // $ref?: string; + name: string; /** - * Type of action generator + * Reports for actions configured but not executed. */ - generator_type: string; + skipped_actions: ReportsReportSkippedActionReport[]; /** - * Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent test suite. - * - * The child action resource ID is supplied by the parent test suite resource ID . - * - * Resources not included in this field will not be available to the child action. - * - * If the parent resource ID is suffixed with ? then the resource will not be required (and will not be populated for the child action when not present in the parent) + * Time at which the test suite started */ - resources: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - }; + start_time: string; /** - * Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type` + * True iff test suite completed normally with no failed checks */ - specification?: { - [k: string]: unknown; - }; + successful?: boolean; + /** + * Type/location of this test suite + */ + suite_type: string; [k: string]: unknown; } /** - * monitoring.uss_qualifier.suites.definitions.TestSuiteDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py + * monitoring.uss_qualifier.reports.report.ParticipantCapabilityEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface SuitesDefinitionsTestSuiteDeclaration { +export interface ReportsReportParticipantCapabilityEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; + condition_evaluation: ReportsReportParticipantCapabilityConditionEvaluationReport; /** - * Definition of test suite internal to the configuration -- specified instead of `suite_type`. + * Whether the capability was successfully verified. */ - suite_definition?: null | SuitesDefinitionsTestSuiteDefinition; + verified: boolean; /** - * Mapping of the ID a resource will be known by in the child test suite -> the ID a resource is known by in the parent test suite. - * - * The child suite resource is supplied by the parent suite resource . + * ID of participant for which capability is being evaluated. */ - resources: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - }; + participant_id: string; /** - * Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder + * ID of capability being evaluated. */ - suite_type?: string | null; + capability_id: string; [k: string]: unknown; } /** - * Schema for the definition of a test suite, analogous to the Python TestScenario subclass for scenarios - * - * monitoring.uss_qualifier.suites.definitions.TestSuiteDefinition, as defined in monitoring/uss_qualifier/suites/definitions.py + * Report produced by evaluating the condition for verifying this capability. */ -export interface SuitesDefinitionsTestSuiteDefinition { +export interface ReportsReportParticipantCapabilityConditionEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Name of the test suite + * When specified, the condition evaluated was CapabilityVerifiedCondition. */ - name: string; + capability_verified?: null | ReportsReportCapabilityVerifiedConditionEvaluationReport; /** - * The scenario executed after all the actions that evaluates the test suite report. Must be a ReportEvaluationScenario. + * When specified, the condition evaluated was NoFailedChecksCondition. */ - report_evaluation_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; + no_failed_checks?: null | ReportsReportNoFailedChecksConditionEvaluationReport; /** - * Definitions of capabilities verified by this test suite for individual participants. + * Whether the condition was satisfied for the relevant participant. */ - participant_verifiable_capabilities?: - | ReportsCapabilityDefinitionsParticipantCapabilityDefinition[] - | null; + condition_satisfied: boolean; /** - * Enumeration of the resources used by this test suite + * When specified, the condition evaluated was AnyCondition. */ - resources: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - }; + any_conditions?: null | ReportsReportAnyConditionEvaluationReport; /** - * The actions to take when running the test suite. Components will be executed in order. + * When specified, the condition evaluated was RequirementsCheckedCondition. */ - actions: SuitesDefinitionsTestSuiteActionDeclaration1[]; + requirements_checked?: null | ReportsReportRequirementsCheckedConditionEvaluationReport; + /** + * When specified, the condition evaluated was AllConditions. + */ + all_conditions?: null | ReportsReportAllConditionsEvaluationReport; [k: string]: unknown; } /** - * monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py + * Result of an evaluation of a CapabilityVerifiedCondition dependent on whether other capabilities were verified. + * + * monitoring.uss_qualifier.reports.report.CapabilityVerifiedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ScenariosDefinitionsTestScenarioDeclaration { +export interface ReportsReportCapabilityVerifiedConditionEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite. - * - * The additional argument to concrete test scenario constructor is supplied by the parent suite resource . + * All capability evaluations checked for this condition. */ - resources?: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - } | null; + checked_capabilities: ReportsReportCheckedCapability[]; /** - * Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder + * Report elements matching the condition's `capability_location`, but not of the type TestSuiteReport. */ - scenario_type: string; + spurious_matches: ReportsReportSpuriousReportMatch[]; + /** + * Capabilities specified for this condition but not found in the report. + */ + missing_capabilities: string[]; [k: string]: unknown; } /** - * monitoring.uss_qualifier.reports.capability_definitions.ParticipantCapabilityDefinition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * Existing/previous participant-verifiable capability upon which a CapabilityVerifiedCondition depends. + * + * monitoring.uss_qualifier.reports.report.CheckedCapability, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsParticipantCapabilityDefinition { +export interface ReportsReportCheckedCapability { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Human-readable description of the capability. + * Whether this capability was successfully verified */ - description: string; - verification_condition: ReportsCapabilityDefinitionsCapabilityVerificationCondition; + capability_verified: boolean; /** - * Human-readable name of the capability. + * The location of the ParticipantCapabilityConditionEvaluationReport for the capability, relative to the TestSuiteReport in which this checked requirement is located. */ - name: string; + capability_location: string; /** - * Identifier of this capability, unique at the level in which this capability is defined. + * Location of the ParticipantCapabilityEvaluationReport for the existing/previous capability, relative to the TestSuiteReport in which the CapabilityVerifiedConditionEvaluationReport containing this CheckedCapability is located. */ - id: string; - [k: string]: unknown; -} -/** - * Condition required in order to verify the capability. - */ -export interface ReportsCapabilityDefinitionsCapabilityVerificationCondition { + report_location: string; /** - * Path to content that replaces the $ref + * ID of the existing/previous participant-verifiable capability. */ - // $ref?: string; - no_failed_checks?: null | ReportsCapabilityDefinitionsNoFailedChecksCondition; - any_conditions?: null | ReportsCapabilityDefinitionsAnyCondition; - requirements_checked?: null | ReportsCapabilityDefinitionsRequirementsCheckedCondition; - all_conditions?: null | ReportsCapabilityDefinitionsAllConditions; - capability_verified?: null | ReportsCapabilityDefinitionsCapabilityVerifiedCondition; + capability_id: string; [k: string]: unknown; } /** - * Condition will only be satisfied if there are no applicable failed checks. - * - * For a capability to be verified for a participant, only checks including the participant's ID will be considered. + * Participant-verifiable capability evaluations are only present in TestSuiteReports. If a CapabilityVerifiedCondition points to a report element that is not a TestSuiteReport, an instance of this class will be generated. * - * monitoring.uss_qualifier.reports.capability_definitions.NoFailedChecksCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * monitoring.uss_qualifier.reports.report.SpuriousReportMatch, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsNoFailedChecksCondition { +export interface ReportsReportSpuriousReportMatch { /** * Path to content that replaces the $ref */ // $ref?: string; + /** + * Data type of the report element (not TestSuiteReport). + */ + type: string; + /** + * Location of the non-TestSuiteReport report element matching the CapabilityVerifiedCondition's `capability_location`, relative to the TestSuiteReport in which this condition is located. + */ + location: string; [k: string]: unknown; } /** - * Condition will be satisfied when any of the specified conditions are satisfied. - * - * Note that an empty list of conditions will result in an unsuccessful evaluation. + * Result of an evaluation of NoFailedChecksCondition dependent on whether any checks failed within the scope of the test suite in which this condition is located. * - * monitoring.uss_qualifier.reports.capability_definitions.AnyCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * monitoring.uss_qualifier.reports.report.NoFailedChecksConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsAnyCondition { +export interface ReportsReportNoFailedChecksConditionEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; - conditions: ReportsCapabilityDefinitionsCapabilityVerificationCondition1[]; + /** + * The location of each FailedCheck, relative to the TestSuiteReport in which this report is located. + */ + failed_checks: string[]; [k: string]: unknown; } /** - * Specification of a single condition used to determine whether a capability should be verified. - * - * Exactly one field must be specified. + * Result of an evaluation of AnyCondition determined by whether any of the subconditions are satisfied. * - * monitoring.uss_qualifier.reports.capability_definitions.CapabilityVerificationCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * monitoring.uss_qualifier.reports.report.AnyConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsCapabilityVerificationCondition1 { +export interface ReportsReportAnyConditionEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; - no_failed_checks?: null | ReportsCapabilityDefinitionsNoFailedChecksCondition; - any_conditions?: null | ReportsCapabilityDefinitionsAnyCondition; - requirements_checked?: null | ReportsCapabilityDefinitionsRequirementsCheckedCondition; - all_conditions?: null | ReportsCapabilityDefinitionsAllConditions; - capability_verified?: null | ReportsCapabilityDefinitionsCapabilityVerifiedCondition; + /** + * Which of the specified options were satisfied (if any were satisfied, then this condition should be satisfied). + */ + satisfied_options: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; + /** + * Which of the specified options were not satisfied (these are informational only and do not affect the evaluation). + */ + unsatisfied_options: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; [k: string]: unknown; } /** - * Condition will only be satisfied if at least one successful check exists for all specified requirements. + * Result of an evaluation of a condition related to whether a participant capability should be verified. * - * Note that an empty collection of requirements will result in an unsuccessful evaluation. + * Exactly one field other than `condition_satisfied` must be specified. * - * monitoring.uss_qualifier.reports.capability_definitions.RequirementsCheckedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * monitoring.uss_qualifier.reports.report.ParticipantCapabilityConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsRequirementsCheckedCondition { +export interface ReportsReportParticipantCapabilityConditionEvaluationReport1 { /** * Path to content that replaces the $ref */ // $ref?: string; - checked: RequirementsDefinitionsRequirementCollection; - [k: string]: unknown; -} -/** - * Each requirement contained within this collection must be covered by at least one successful check. - */ -export interface RequirementsDefinitionsRequirementCollection { /** - * Path to content that replaces the $ref + * When specified, the condition evaluated was CapabilityVerifiedCondition. */ - // $ref?: string; + capability_verified?: null | ReportsReportCapabilityVerifiedConditionEvaluationReport; /** - * This collection includes all of the requirements in all of these requirement collections. + * When specified, the condition evaluated was NoFailedChecksCondition. */ - requirement_collections?: - | RequirementsDefinitionsRequirementCollection1[] - | null; + no_failed_checks?: null | ReportsReportNoFailedChecksConditionEvaluationReport; /** - * This collection includes all of these requirements. + * Whether the condition was satisfied for the relevant participant. */ - requirements?: string[] | null; + condition_satisfied: boolean; /** - * This collection includes all requirements in all of these requirement sets. + * When specified, the condition evaluated was AnyCondition. */ - requirement_sets?: string[] | null; + any_conditions?: null | ReportsReportAnyConditionEvaluationReport; /** - * This collection does not include any of these requirements, despite all previous fields. + * When specified, the condition evaluated was RequirementsCheckedCondition. */ - exclude?: null | RequirementsDefinitionsRequirementCollection1; + requirements_checked?: null | ReportsReportRequirementsCheckedConditionEvaluationReport; + /** + * When specified, the condition evaluated was AllConditions. + */ + all_conditions?: null | ReportsReportAllConditionsEvaluationReport; [k: string]: unknown; } /** - * monitoring.uss_qualifier.requirements.definitions.RequirementCollection, as defined in monitoring/uss_qualifier/requirements/definitions.py + * Result of an evaluation of RequirementsCheckedCondition dependent on whether a set of requirements were successfully checked. + * + * monitoring.uss_qualifier.reports.report.RequirementsCheckedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface RequirementsDefinitionsRequirementCollection1 { +export interface ReportsReportRequirementsCheckedConditionEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * This collection includes all of the requirements in all of these requirement collections. - */ - requirement_collections?: - | RequirementsDefinitionsRequirementCollection1[] - | null; - /** - * This collection includes all of these requirements. + * Requirements with only PassedChecks. */ - requirements?: string[] | null; + passed_requirements: ReportsReportCheckedRequirement[]; /** - * This collection includes all requirements in all of these requirement sets. + * Requirements with FailedChecks. */ - requirement_sets?: string[] | null; + failed_requirements: ReportsReportCheckedRequirement[]; /** - * This collection does not include any of these requirements, despite all previous fields. + * Requirements that didn't have any PassedChecks or FailedChecks within the scope of the test suite in which this condition is located. */ - exclude?: null | RequirementsDefinitionsRequirementCollection1; + untested_requirements: string[]; [k: string]: unknown; } /** - * Condition will only be satisfied when all specified conditions are satisfied. - * - * Note that an empty list of conditions will result in a successful evaluation. + * A single requirement being checked for participant-verifiable capability verification. * - * monitoring.uss_qualifier.reports.capability_definitions.AllConditions, as defined in monitoring/uss_qualifier/reports/capability_definitions.py + * monitoring.uss_qualifier.reports.report.CheckedRequirement, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ReportsCapabilityDefinitionsAllConditions { +export interface ReportsReportCheckedRequirement { /** * Path to content that replaces the $ref */ // $ref?: string; - conditions: ReportsCapabilityDefinitionsCapabilityVerificationCondition1[]; - [k: string]: unknown; -} -/** - * Condition will be satisfied when the specified capability is verified. - * - * Note that a capability which do not declare any requirement will result in an unsuccessful evaluation. - * - * monitoring.uss_qualifier.reports.capability_definitions.CapabilityVerifiedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py - */ -export interface ReportsCapabilityDefinitionsCapabilityVerifiedCondition { /** - * Path to content that replaces the $ref + * The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located. */ - // $ref?: string; + passed_checks: string[]; /** - * Location of report to inspect for the verification of the specified capability, relative to the report in which - * the capability is defined. Implicit default value is "$" (look for verified capability in the report in which the - * dependant capability is defined). - * - * If this location resolves to multiple TestSuiteReports, then the capability must be verified in all resolved reports - * in order for this condition to be satisfied. When this location resolves to artifacts that are not - * TestSuiteReports, those artifacts will be ignored. - * - * Note that capabilities are verified in the order they are defined. So, if capability B defined in a particular - * location depends on whether capability A in that same location is granted, capability A must be defined before - * capability B is defined. Also note that capability verifications are computed as test components are completed. - * Since a parent test component (e.g., test suite) is not complete until all of its child components are complete, a - * descendant test component's capability condition cannot depend on whether an ancestor's (e.g., parent's) capability - * is verified. + * The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located. */ - capability_location?: string | null; + failed_checks: string[]; /** - * List of identifier of capability that must be verified for this condition to be satisfied. + * The requirement being checked. */ - capability_ids: string[]; + requirement_id: string; [k: string]: unknown; } /** - * Defines a step in the sequence of things to do for a test suite. - * - * Exactly one of `test_scenario`, `test_suite`, or `action_generator` must be specified. + * Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied. * - * monitoring.uss_qualifier.suites.definitions.TestSuiteActionDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py + * monitoring.uss_qualifier.reports.report.AllConditionsEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface SuitesDefinitionsTestSuiteActionDeclaration1 { +export interface ReportsReportAllConditionsEvaluationReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * If this field is populated, declaration of a generator that will produce 0 or more test suite actions - */ - action_generator?: null | SuitesDefinitionsActionGeneratorDefinition; - /** - * If this field is populated, declaration of the test suite to run - */ - test_suite?: null | SuitesDefinitionsTestSuiteDeclaration; - /** - * What to do if this action fails + * All of the conditions that were satisfied (there must be at least one). */ - on_failure?: "Continue" | "Abort"; + satisfied_conditions: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; /** - * If this field is populated, declaration of the test scenario to run + * All of the conditions that were unsatisfied (if any, then this condition will not be satisfied). */ - test_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; + unsatisfied_conditions: ReportsReportParticipantCapabilityConditionEvaluationReport1[]; [k: string]: unknown; } /** - * Declarations for resources used by the test suite + * monitoring.uss_qualifier.reports.report.SkippedActionReport, as defined in monitoring/uss_qualifier/reports/report.py */ -export interface ResourcesDefinitionsResourceCollection { +export interface ReportsReportSkippedActionReport { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Mapping of globally (within resource collection) unique name identifying a resource to the declaration of that resource + * Index of the skipped action in the configured declaration. */ - resource_declarations: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [ - k: string - ]: ResourcesDefinitionsResourceDeclaration; - }; + action_declaration_index: number; + declaration: SuitesDefinitionsTestSuiteActionDeclaration2; + /** + * The reason the action was skipped. + */ + reason: string; [k: string]: unknown; } /** - * monitoring.uss_qualifier.resources.definitions.ResourceDeclaration, as defined in monitoring/uss_qualifier/resources/definitions.py + * Defines a step in the sequence of things to do for a test suite. + * + * Exactly one of `test_scenario`, `test_suite`, or `action_generator` must be specified. + * + * monitoring.uss_qualifier.suites.definitions.TestSuiteActionDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py */ -export interface ResourcesDefinitionsResourceDeclaration { +export interface SuitesDefinitionsTestSuiteActionDeclaration2 { /** * Path to content that replaces the $ref */ // $ref?: string; /** - * Mapping of dependency parameter (additional argument to concrete resource constructor) to `name` of resource to use + * If this field is populated, declaration of a generator that will produce 0 or more test suite actions */ - dependencies?: { - /** - * Path to content that replaces the $ref - */ - // $ref?: string; - [k: string]: string; - }; + action_generator?: null | ActionGeneratorsDefinitionsActionGeneratorDefinition; /** - * Type of resource, expressed as a Python class name qualified relative to this `resources` module + * What to do if this action fails */ - resource_type: string; + on_failure?: "Continue" | "Abort"; /** - * Specification of resource; format is the SpecificationType that corresponds to the `resource_type` + * If this field is populated, declaration of the test scenario to run */ - specification?: { - [k: string]: unknown; - }; + test_scenario?: null | ScenariosDefinitionsTestScenarioDeclaration; + /** + * If this field is populated, declaration of the test suite to run + */ + test_suite?: null | SuitesDefinitionsTestSuiteDeclaration; [k: string]: unknown; } diff --git a/schemas/TODO.md b/schemas/TODO.md index c7f1f53..4ad9ffa 100644 --- a/schemas/TODO.md +++ b/schemas/TODO.md @@ -1 +1 @@ -TODO: Find an elegant way to retrieve interuss/monitoring/schemas/* files. \ No newline at end of file +TODO: Find an elegant way to retrieve interuss/monitoring/schemas/\* files. diff --git a/schemas/monitorlib/dicts/RemovedElement.json b/schemas/monitorlib/dicts/RemovedElement.json new file mode 100644 index 0000000..baacca5 --- /dev/null +++ b/schemas/monitorlib/dicts/RemovedElement.json @@ -0,0 +1,18 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/dicts/RemovedElement.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.monitorlib.dicts.RemovedElement, as defined in monitoring/monitorlib/dicts.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "address": { + "type": "string" + } + }, + "required": [ + "address" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitorlib/fetch/Query.json b/schemas/monitorlib/fetch/Query.json index 26188f8..6c33dc6 100644 --- a/schemas/monitorlib/fetch/Query.json +++ b/schemas/monitorlib/fetch/Query.json @@ -1,22 +1,66 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/fetch/Query.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, - "response": { - "$ref": "ResponseDescription.json" + "query_type": { + "description": "If specified, the recognized type of this query.", + "enum": [ + "astm.f3411.v22a.sp.flights", + "astm.f3411.v19.sp.flights", + "astm.f3411.v22a.sp.flight_details", + "astm.f3411.v19.sp.flight_details", + "astm.f3548.v21.dss.queryOperationalIntentReferences", + "astm.f3548.v21.dss.getOperationalIntentReference", + "astm.f3548.v21.dss.createOperationalIntentReference", + "astm.f3548.v21.dss.updateOperationalIntentReference", + "astm.f3548.v21.dss.deleteOperationalIntentReference", + "astm.f3548.v21.dss.queryConstraintReferences", + "astm.f3548.v21.dss.getConstraintReference", + "astm.f3548.v21.dss.createConstraintReference", + "astm.f3548.v21.dss.updateConstraintReference", + "astm.f3548.v21.dss.deleteConstraintReference", + "astm.f3548.v21.dss.querySubscriptions", + "astm.f3548.v21.dss.getSubscription", + "astm.f3548.v21.dss.createSubscription", + "astm.f3548.v21.dss.updateSubscription", + "astm.f3548.v21.dss.deleteSubscription", + "astm.f3548.v21.dss.makeDssReport", + "astm.f3548.v21.uss.setUssAvailability", + "astm.f3548.v21.uss.getUssAvailability", + "astm.f3548.v21.uss.getOperationalIntentDetails", + "astm.f3548.v21.uss.getOperationalIntentTelemetry", + "astm.f3548.v21.uss.notifyOperationalIntentDetailsChanged", + "astm.f3548.v21.uss.getConstraintDetails", + "astm.f3548.v21.uss.notifyConstraintDetailsChanged", + "astm.f3548.v21.uss.makeUssReport" + ], + "type": [ + "string", + "null" + ] }, "request": { "$ref": "RequestDescription.json" + }, + "response": { + "$ref": "ResponseDescription.json" + }, + "server_id": { + "description": "If specified, identifier of the USS/participant hosting the server involved in this query.", + "type": [ + "string", + "null" + ] } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/fetch/Query.json", - "description": "monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py", "required": [ "request", "response" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/suites/definitions/ActionGeneratorDefinition.json b/schemas/uss_qualifier/action_generators/definitions/ActionGeneratorDefinition.json similarity index 64% rename from schemas/uss_qualifier/suites/definitions/ActionGeneratorDefinition.json rename to schemas/uss_qualifier/action_generators/definitions/ActionGeneratorDefinition.json index 27a4877..076034a 100644 --- a/schemas/uss_qualifier/suites/definitions/ActionGeneratorDefinition.json +++ b/schemas/uss_qualifier/action_generators/definitions/ActionGeneratorDefinition.json @@ -1,37 +1,37 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/action_generators/definitions/ActionGeneratorDefinition.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.action_generators.definitions.ActionGeneratorDefinition, as defined in monitoring/uss_qualifier/action_generators/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "generator_type": { - "type": "string", - "description": "Type of action generator" + "description": "Type of action generator", + "type": "string" }, "resources": { - "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent test suite.\n\nThe child action resource ID is supplied by the parent test suite resource ID .\n\nResources not included in this field will not be available to the child action.\n\nIf the parent resource ID is suffixed with ? then the resource will not be required (and will not be populated for the child action when not present in the parent)", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent test suite.\n\nThe child action resource ID is supplied by the parent test suite resource ID .\n\nResources not included in this field will not be available to the child action.\n\nIf the parent resource ID is suffixed with ? then the resource will not be required (and will not be populated for the child action when not present in the parent)" + "type": "object" }, "specification": { - "type": "object", - "description": "Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`" + "description": "Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`", + "type": "object" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/suites/definitions/ActionGeneratorDefinition.json", - "description": "monitoring.uss_qualifier.suites.definitions.ActionGeneratorDefinition, as defined in monitoring/uss_qualifier/suites/definitions.py", "required": [ "generator_type", "resources" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json b/schemas/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json index 3eb89fc..65a548b 100644 --- a/schemas/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json +++ b/schemas/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json @@ -1,23 +1,29 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.ArtifactsConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, - "report_html": { + "graph": { + "description": "If specified, configuration describing a desired graph visualization summarizing the test run", "oneOf": [ { "type": "null" }, { - "$ref": "ReportHTMLConfiguration.json" + "$ref": "GraphConfiguration.json" } - ], - "description": "If specified, configuration describing how an HTML version of the report should be generated" + ] + }, + "redact_access_tokens": { + "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens", + "type": "boolean" }, "report": { + "description": "Configuration for report generation", "oneOf": [ { "type": "null" @@ -25,32 +31,59 @@ { "$ref": "ReportConfiguration.json" } - ], - "description": "Configuration for report generation" + ] }, - "tested_roles": { + "report_html": { + "description": "If specified, configuration describing how an HTML version of the report should be generated", "oneOf": [ { "type": "null" }, { - "$ref": "TestedRolesConfiguration.json" + "$ref": "ReportHTMLConfiguration.json" } - ], - "description": "If specified, configuration describing a desired report summarizing tested requirements for each specified participant and role" + ] }, - "graph": { + "sequence_view": { + "description": "If specified, configuration describing a desired report describing the sequence of events that occurred during the test", "oneOf": [ { "type": "null" }, { - "$ref": "GraphConfiguration.json" + "$ref": "SequenceViewConfiguration.json" + } + ] + }, + "templated_reports": { + "description": "List of report templates to be rendered", + "items": { + "$ref": "TemplatedReportConfiguration.json" + }, + "type": "array" + }, + "tested_requirements": { + "description": "If specified, configuration describing a desired report summarizing all tested requirements for each participant", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestedRequirementsConfiguration.json" } - ], - "description": "If specified, configuration describing a desired graph visualization summarizing the test run" + ] + }, + "tested_roles": { + "description": "If specified, configuration describing a desired report summarizing tested requirements for each specified participant and role", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestedRolesConfiguration.json" + } + ] } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.ArtifactsConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py" + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/ReportConfiguration.json b/schemas/uss_qualifier/configurations/configuration/ReportConfiguration.json index 4550754..ea0cff1 100644 --- a/schemas/uss_qualifier/configurations/configuration/ReportConfiguration.json +++ b/schemas/uss_qualifier/configurations/configuration/ReportConfiguration.json @@ -1,23 +1,19 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.ReportConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "report_path": { - "type": "string", - "description": "File name of the report to write (if test_config provided) or read (if test_config not provided)" - }, - "redact_access_tokens": { - "type": "boolean", - "description": "When True, look for instances of \"Authorization\" keys in the report with values starting \"Bearer \" and redact the signature from those access tokens" + "description": "File name of the report to write (if test_config provided) or read (if test_config not provided)", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.ReportConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "required": [ "report_path" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json b/schemas/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json index 4ad3d1f..1877035 100644 --- a/schemas/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json +++ b/schemas/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json @@ -1,19 +1,19 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.ReportHTMLConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "html_path": { - "type": "string", - "description": "Path of HTML file to contain an HTML rendering of the test report" + "description": "Path of HTML file to contain an HTML rendering of the raw test report object", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.ReportHTMLConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "required": [ "html_path" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json b/schemas/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json new file mode 100644 index 0000000..f1428d0 --- /dev/null +++ b/schemas/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json @@ -0,0 +1,19 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.SequenceViewConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "output_path": { + "description": "Path of a folder into which report HTML files should be written", + "type": "string" + } + }, + "required": [ + "output_path" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json b/schemas/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json new file mode 100644 index 0000000..6e26232 --- /dev/null +++ b/schemas/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json @@ -0,0 +1,35 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.TemplatedReportConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "configuration": { + "description": "Configuration to be injected in the templated report", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TemplatedReportInjectedConfiguration.json" + } + ] + }, + "output_path": { + "description": "Path of HTML file to contain the rendered templated report", + "type": "string" + }, + "template_url": { + "description": "Url of the template to download from", + "type": "string" + } + }, + "required": [ + "output_path", + "template_url" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TemplatedReportInjectedConfiguration.json b/schemas/uss_qualifier/configurations/configuration/TemplatedReportInjectedConfiguration.json new file mode 100644 index 0000000..8e1da06 --- /dev/null +++ b/schemas/uss_qualifier/configurations/configuration/TemplatedReportInjectedConfiguration.json @@ -0,0 +1,12 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TemplatedReportInjectedConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.TemplatedReportInjectedConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TestConfiguration.json b/schemas/uss_qualifier/configurations/configuration/TestConfiguration.json index c90781d..800958b 100644 --- a/schemas/uss_qualifier/configurations/configuration/TestConfiguration.json +++ b/schemas/uss_qualifier/configurations/configuration/TestConfiguration.json @@ -1,34 +1,34 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.TestConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "action": { "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", "description": "The action this test configuration wants to run (usually a test suite)" }, "non_baseline_inputs": { - "type": [ - "array", - "null" - ], + "description": "List of portions of the configuration that should not be considered when computing the test baseline signature (e.g., environmental definitions).", "items": { "type": "string" }, - "description": "List of file inputs that should not be considered when computing the test baseline signature (e.g., environmental definitions)." + "type": [ + "array", + "null" + ] }, "resources": { "$ref": "../../resources/definitions/ResourceCollection.json", "description": "Declarations for resources used by the test suite" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.TestConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "required": [ "action", "resources" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json b/schemas/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json new file mode 100644 index 0000000..35ffb9d --- /dev/null +++ b/schemas/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json @@ -0,0 +1,49 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRequirementsConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.TestedRequirementsConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "output_path": { + "description": "Path of a folder into which report HTML files should be written", + "type": "string" + }, + "participant_requirements": { + "additionalProperties": { + "type": "string" + }, + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + } + }, + "type": [ + "object", + "null" + ] + }, + "requirement_collections": { + "additionalProperties": { + "$ref": "../../requirements/definitions/RequirementCollection.json" + }, + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + } + }, + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "output_path" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TestedRole.json b/schemas/uss_qualifier/configurations/configuration/TestedRole.json deleted file mode 100644 index 6674cbe..0000000 --- a/schemas/uss_qualifier/configurations/configuration/TestedRole.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "participants": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Participants fulfilling the role" - }, - "name": { - "type": "string", - "description": "Name of role" - }, - "requirement_set": { - "type": "string", - "description": "Set of requirements a participant must satisfy to fulfill the role" - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRole.json", - "description": "monitoring.uss_qualifier.configurations.configuration.TestedRole, as defined in monitoring/uss_qualifier/configurations/configuration.py", - "required": [ - "name", - "participants", - "requirement_set" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json b/schemas/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json index 0b24e5a..af8cf7a 100644 --- a/schemas/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json +++ b/schemas/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json @@ -1,27 +1,19 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.TestedRolesConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "report_path": { - "type": "string", - "description": "Path of HTML file to contain a requirements-based summary of the test report" - }, - "roles": { - "type": "array", - "items": { - "$ref": "TestedRole.json" - }, - "description": "Roles (and participants filling those roles) tested by the test run" + "description": "Path of folder to write HTML files containing a fulfilled-requirements-based view of the test report", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestedRolesConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.TestedRolesConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "required": [ - "report_path", - "roles" - ] + "report_path" + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/graphs/Node.json b/schemas/uss_qualifier/reports/graphs/Node.json deleted file mode 100644 index fb0e789..0000000 --- a/schemas/uss_qualifier/reports/graphs/Node.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "label": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": "string" - }, - "children": { - "type": "array", - "items": { - "type": "string" - } - }, - "attributes": { - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - } - }, - "additionalProperties": { - "type": "string" - } - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/graphs/Node.json", - "description": "Represents a node to be used in a GraphViz graph.\n\nmonitoring.uss_qualifier.reports.graphs.Node, as defined in monitoring/uss_qualifier/reports/graphs.py", - "required": [ - "attributes", - "children", - "name" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/ActionGeneratorReport.json b/schemas/uss_qualifier/reports/report/ActionGeneratorReport.json index c35a3db..83de506 100644 --- a/schemas/uss_qualifier/reports/report/ActionGeneratorReport.json +++ b/schemas/uss_qualifier/reports/report/ActionGeneratorReport.json @@ -1,27 +1,45 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.ActionGeneratorReport, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "generator_type": { - "type": "string", - "description": "Type of action generator" + "description": "Path to content that replaces the $ref", + "type": "string" }, "actions": { - "type": "array", + "description": "Reports from the actions generated by the action generator, in order of execution.", "items": { "$ref": "TestSuiteActionReport.json" }, - "description": "Reports from the actions generated by the action generator" + "type": "array" + }, + "end_time": { + "description": "Time at which the action generator completed or encountered an error", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "generator_type": { + "description": "Type of action generator", + "type": "string" + }, + "start_time": { + "description": "Time at which the action generator started", + "format": "date-time", + "type": "string" + }, + "successful": { + "description": "True iff all actions completed normally with no failed checks", + "type": "boolean" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json", - "description": "monitoring.uss_qualifier.reports.report.ActionGeneratorReport, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "actions", - "generator_type" - ] + "generator_type", + "start_time" + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/PassedCheck.json b/schemas/uss_qualifier/reports/report/PassedCheck.json index 30ec39e..efa9217 100644 --- a/schemas/uss_qualifier/reports/report/PassedCheck.json +++ b/schemas/uss_qualifier/reports/report/PassedCheck.json @@ -1,35 +1,41 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "name": { + "description": "Name of the check that passed", + "type": "string" }, "participants": { - "type": "array", + "description": "Participants that may not have met the relevant requirements if this check had failed", "items": { "type": "string" }, - "description": "Participants that may not have met the relevant requirements if this check had failed" - }, - "name": { - "type": "string", - "description": "Name of the check that passed" + "type": "array" }, "requirements": { - "type": "array", + "description": "Requirements that would not have been met if this check had failed", "items": { "type": "string" }, - "description": "Requirements that would not have been met if this check had failed" + "type": "array" + }, + "timestamp": { + "description": "Time the issue was discovered", + "format": "date-time", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json", - "description": "monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "name", "participants", - "requirements" - ] + "requirements", + "timestamp" + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/SkippedActionReport.json b/schemas/uss_qualifier/reports/report/SkippedActionReport.json new file mode 100644 index 0000000..2477757 --- /dev/null +++ b/schemas/uss_qualifier/reports/report/SkippedActionReport.json @@ -0,0 +1,29 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.reports.report.SkippedActionReport, as defined in monitoring/uss_qualifier/reports/report.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "action_declaration_index": { + "description": "Index of the skipped action in the configured declaration.", + "type": "integer" + }, + "declaration": { + "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", + "description": "Full declaration of the action that was skipped." + }, + "reason": { + "description": "The reason the action was skipped.", + "type": "string" + } + }, + "required": [ + "action_declaration_index", + "declaration", + "reason" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/TestCaseReport.json b/schemas/uss_qualifier/reports/report/TestCaseReport.json index 3b6cc25..53c666d 100644 --- a/schemas/uss_qualifier/reports/report/TestCaseReport.json +++ b/schemas/uss_qualifier/reports/report/TestCaseReport.json @@ -1,46 +1,46 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestCaseReport.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.TestCaseReport, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "name": { - "type": "string", - "description": "Name of this test case" + "description": "Path to content that replaces the $ref", + "type": "string" }, "documentation_url": { - "type": "string", - "description": "URL at which this test case is described" + "description": "URL at which this test case is described", + "type": "string" }, "end_time": { + "description": "Time at which the test case completed or encountered an error", + "format": "date-time", "type": [ "string", "null" - ], - "format": "date-time", - "description": "Time at which the test case completed or encountered an error" + ] + }, + "name": { + "description": "Name of this test case", + "type": "string" }, "start_time": { - "type": "string", + "description": "Time at which the test case started", "format": "date-time", - "description": "Time at which the test case started" + "type": "string" }, "steps": { - "type": "array", + "description": "Reports for each of the test steps in this test case, in chronological order.", "items": { "$ref": "TestStepReport.json" }, - "description": "Reports for each of the test steps in this test case" + "type": "array" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestCaseReport.json", - "description": "monitoring.uss_qualifier.reports.report.TestCaseReport, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "documentation_url", "name", "start_time", "steps" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/TestRunReport.json b/schemas/uss_qualifier/reports/report/TestRunReport.json index 23e5e9f..24d1069 100644 --- a/schemas/uss_qualifier/reports/report/TestRunReport.json +++ b/schemas/uss_qualifier/reports/report/TestRunReport.json @@ -1,53 +1,44 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestRunReport.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.TestRunReport, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "baseline_signature": { - "type": "string", - "description": "Signature of the test run including codebase version and all file signatures except excluded environmental files.\n\nThis field can be used to identify that a particular expected test baseline (codebase, all non-environmental inputs)\nwas run. The value of this field is computed from codebase_version and all elements of file_signatures that are not\nexplicitly excluded as environmental configuration." + "description": "Signature of the test run including codebase version and all file signatures except excluded environmental files.\n\nThis field can be used to identify that a particular expected test baseline (codebase, all non-environmental inputs)\nwas run. The value of this field is computed from codebase_version and all elements of the configuration that are\nnot explicitly excluded as environmental configuration.", + "type": "string" }, "codebase_version": { - "type": "string", - "description": "Version of codebase used to run uss_qualifier" + "description": "Version of codebase used to run uss_qualifier", + "type": "string" }, "commit_hash": { - "type": "string", - "description": "Full commit hash of codebase used to run uss_qualifier" - }, - "report": { - "$ref": "TestSuiteActionReport.json", - "description": "Report produced by configured test action" + "description": "Full commit hash of codebase used to run uss_qualifier", + "type": "string" }, "configuration": { "$ref": "../../configurations/configuration/TestConfiguration.json", "description": "Configuration used to run uss_qualifier" }, - "file_signatures": { - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - } - }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping between the names of files loaded during test run and the SHA-1 hashes of those files' content." + "environment_signature": { + "description": "Signature of the environmental inputs of the configuration not included in the baseline signature.", + "type": "string" + }, + "report": { + "$ref": "TestSuiteActionReport.json", + "description": "Report produced by configured test action" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestRunReport.json", - "description": "monitoring.uss_qualifier.reports.report.TestRunReport, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "baseline_signature", "codebase_version", "commit_hash", "configuration", - "file_signatures", + "environment_signature", "report" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/TestScenarioReport.json b/schemas/uss_qualifier/reports/report/TestScenarioReport.json index 88c09ae..f3af7ae 100644 --- a/schemas/uss_qualifier/reports/report/TestScenarioReport.json +++ b/schemas/uss_qualifier/reports/report/TestScenarioReport.json @@ -1,93 +1,93 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.TestScenarioReport, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "successful": { - "type": "boolean", - "description": "True iff test scenario completed normally with no failed checks" - }, - "start_time": { - "type": "string", - "format": "date-time", - "description": "Time at which the test scenario started" + "description": "Path to content that replaces the $ref", + "type": "string" }, "cases": { - "type": "array", + "description": "Reports for each of the test cases in this test scenario, in chronological order.", "items": { "$ref": "TestCaseReport.json" }, - "description": "Reports for each of the test cases in this test scenario" - }, - "documentation_url": { - "type": "string", - "description": "URL at which this test scenario is described" + "type": "array" }, - "execution_error": { + "cleanup": { + "description": "If this test scenario performed cleanup, this report captures the relevant information.", "oneOf": [ { "type": "null" }, { - "$ref": "ErrorReport.json" + "$ref": "TestStepReport.json" } - ], - "description": "If there was an error while executing this test scenario, this field describes the error" + ] + }, + "documentation_url": { + "description": "URL at which this test scenario is described", + "type": "string" }, "end_time": { + "description": "Time at which the test scenario completed or encountered an error", + "format": "date-time", "type": [ "string", "null" - ], - "format": "date-time", - "description": "Time at which the test scenario completed or encountered an error" + ] }, - "cleanup": { + "execution_error": { + "description": "If there was an error while executing this test scenario, this field describes the error", "oneOf": [ { "type": "null" }, { - "$ref": "TestStepReport.json" + "$ref": "ErrorReport.json" } - ], - "description": "If this test scenario performed cleanup, this report captures the relevant information." + ] + }, + "name": { + "description": "Name of this test scenario", + "type": "string" }, "notes": { - "type": [ - "object", - "null" - ], + "additionalProperties": { + "$ref": "Note.json" + }, + "description": "Additional information about this scenario that may be useful", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "$ref": "Note.json" - }, - "description": "Additional information about this scenario that may be useful" + "type": [ + "object", + "null" + ] }, "scenario_type": { - "type": "string", - "description": "Type of this test scenario" + "description": "Type of this test scenario", + "type": "string" }, - "name": { - "type": "string", - "description": "Name of this test scenario" + "start_time": { + "description": "Time at which the test scenario started", + "format": "date-time", + "type": "string" + }, + "successful": { + "description": "True iff test scenario completed normally with no failed checks", + "type": "boolean" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json", - "description": "monitoring.uss_qualifier.reports.report.TestScenarioReport, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "cases", "documentation_url", "name", "scenario_type", "start_time" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/reports/report/TestSuiteReport.json b/schemas/uss_qualifier/reports/report/TestSuiteReport.json index 33581bd..b068d9b 100644 --- a/schemas/uss_qualifier/reports/report/TestSuiteReport.json +++ b/schemas/uss_qualifier/reports/report/TestSuiteReport.json @@ -1,63 +1,71 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.TestSuiteReport, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "successful": { - "type": "boolean", - "description": "True iff test suite completed normally with no failed checks" + "description": "Path to content that replaces the $ref", + "type": "string" }, - "start_time": { - "type": "string", - "format": "date-time", - "description": "Time at which the test suite started" - }, - "documentation_url": { - "type": "string", - "description": "URL at which this test suite is described" + "actions": { + "description": "Reports from test scenarios and test suites comprising the test suite for this report, in order of execution.", + "items": { + "$ref": "TestSuiteActionReport.json" + }, + "type": "array" }, "capability_evaluations": { - "type": "array", + "description": "List of capabilities defined in this test suite, evaluated for each participant.", "items": { "$ref": "ParticipantCapabilityEvaluationReport.json" }, - "description": "List of capabilities defined in this test suite, evaluated for each participant." + "type": "array" + }, + "documentation_url": { + "description": "URL at which this test suite is described", + "type": "string" }, "end_time": { + "description": "Time at which the test suite completed", + "format": "date-time", "type": [ "string", "null" - ], - "format": "date-time", - "description": "Time at which the test suite completed" + ] }, - "actions": { - "type": "array", + "name": { + "description": "Name of this test suite", + "type": "string" + }, + "skipped_actions": { + "description": "Reports for actions configured but not executed.", "items": { - "$ref": "TestSuiteActionReport.json" + "$ref": "SkippedActionReport.json" }, - "description": "Reports from test scenarios and test suites comprising the test suite for this report" + "type": "array" }, - "suite_type": { - "type": "string", - "description": "Type/location of this test suite" + "start_time": { + "description": "Time at which the test suite started", + "format": "date-time", + "type": "string" }, - "name": { - "type": "string", - "description": "Name of this test suite" + "successful": { + "description": "True iff test suite completed normally with no failed checks", + "type": "boolean" + }, + "suite_type": { + "description": "Type/location of this test suite", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json", - "description": "monitoring.uss_qualifier.reports.report.TestSuiteReport, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "actions", "capability_evaluations", "documentation_url", "name", + "skipped_actions", "start_time", "suite_type" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json b/schemas/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json index 60d2424..26e32cb 100644 --- a/schemas/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json +++ b/schemas/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json @@ -1,35 +1,35 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "resources": { - "type": [ - "object", - "null" - ], + "additionalProperties": { + "type": "string" + }, + "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource .", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource ." + "type": [ + "object", + "null" + ] }, "scenario_type": { - "type": "string", - "description": "Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder" + "description": "Type of test scenario.", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", - "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "required": [ "scenario_type" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/definitions/TestCaseDocumentation.json b/schemas/uss_qualifier/scenarios/documentation/definitions/TestCaseDocumentation.json deleted file mode 100644 index 106de7d..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/definitions/TestCaseDocumentation.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "url": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": "string" - }, - "steps": { - "type": "array", - "items": { - "$ref": "TestStepDocumentation.json" - } - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/definitions/TestCaseDocumentation.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.definitions.TestCaseDocumentation, as defined in monitoring/uss_qualifier/scenarios/documentation/definitions.py", - "required": [ - "name", - "steps" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/definitions/TestCheckDocumentation.json b/schemas/uss_qualifier/scenarios/documentation/definitions/TestCheckDocumentation.json deleted file mode 100644 index d7453cc..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/definitions/TestCheckDocumentation.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "url": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": "string" - }, - "applicable_requirements": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/definitions/TestCheckDocumentation.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.definitions.TestCheckDocumentation, as defined in monitoring/uss_qualifier/scenarios/documentation/definitions.py", - "required": [ - "applicable_requirements", - "name" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/definitions/TestScenarioDocumentation.json b/schemas/uss_qualifier/scenarios/documentation/definitions/TestScenarioDocumentation.json deleted file mode 100644 index 3205fea..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/definitions/TestScenarioDocumentation.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "name": { - "type": "string" - }, - "cases": { - "type": "array", - "items": { - "$ref": "TestCaseDocumentation.json" - } - }, - "url": { - "type": [ - "string", - "null" - ] - }, - "cleanup": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "TestStepDocumentation.json" - } - ] - }, - "resources": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/definitions/TestScenarioDocumentation.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.definitions.TestScenarioDocumentation, as defined in monitoring/uss_qualifier/scenarios/documentation/definitions.py", - "required": [ - "cases", - "name" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/definitions/TestStepDocumentation.json b/schemas/uss_qualifier/scenarios/documentation/definitions/TestStepDocumentation.json deleted file mode 100644 index 4f63660..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/definitions/TestStepDocumentation.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "url": { - "type": [ - "string", - "null" - ] - }, - "name": { - "type": "string" - }, - "checks": { - "type": "array", - "items": { - "$ref": "TestCheckDocumentation.json" - } - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/definitions/TestStepDocumentation.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.definitions.TestStepDocumentation, as defined in monitoring/uss_qualifier/scenarios/documentation/definitions.py", - "required": [ - "checks", - "name" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/requirements/ParticipantRequirementPerformance.json b/schemas/uss_qualifier/scenarios/documentation/requirements/ParticipantRequirementPerformance.json deleted file mode 100644 index 0a8075a..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/requirements/ParticipantRequirementPerformance.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "successes": { - "type": "array", - "items": { - "type": "string" - }, - "description": "List of passed checks involving the requirement" - }, - "failures": { - "type": "array", - "items": { - "type": "string" - }, - "description": "List of failed checks involving the requirement" - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/requirements/ParticipantRequirementPerformance.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.requirements.ParticipantRequirementPerformance, as defined in monitoring/uss_qualifier/scenarios/documentation/requirements.py", - "required": [ - "failures", - "successes" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/scenarios/documentation/requirements/TestedRequirement.json b/schemas/uss_qualifier/scenarios/documentation/requirements/TestedRequirement.json deleted file mode 100644 index 60179d2..0000000 --- a/schemas/uss_qualifier/scenarios/documentation/requirements/TestedRequirement.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "requirement_id": { - "type": "string", - "description": "Identity of the requirement" - }, - "participant_performance": { - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - } - }, - "additionalProperties": { - "$ref": "ParticipantRequirementPerformance.json" - }, - "description": "The performance of each involved participant on the requirement" - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/documentation/requirements/TestedRequirement.json", - "description": "monitoring.uss_qualifier.scenarios.documentation.requirements.TestedRequirement, as defined in monitoring/uss_qualifier/scenarios/documentation/requirements.py", - "required": [ - "participant_performance", - "requirement_id" - ] -} \ No newline at end of file diff --git a/schemas/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json b/schemas/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json index 28b77ba..e2527d7 100644 --- a/schemas/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json +++ b/schemas/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json @@ -1,42 +1,33 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "Defines a step in the sequence of things to do for a test suite.\n\nExactly one of `test_scenario`, `test_suite`, or `action_generator` must be specified.\n\nmonitoring.uss_qualifier.suites.definitions.TestSuiteActionDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "action_generator": { + "description": "If this field is populated, declaration of a generator that will produce 0 or more test suite actions", "oneOf": [ { "type": "null" }, { - "$ref": "ActionGeneratorDefinition.json" - } - ], - "description": "If this field is populated, declaration of a generator that will produce 0 or more test suite actions" - }, - "test_suite": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "TestSuiteDeclaration.json" + "$ref": "../../action_generators/definitions/ActionGeneratorDefinition.json" } - ], - "description": "If this field is populated, declaration of the test suite to run" + ] }, "on_failure": { - "type": "string", + "description": "What to do if this action fails", "enum": [ "Continue", "Abort" ], - "description": "What to do if this action fails" + "type": "string" }, "test_scenario": { + "description": "If this field is populated, declaration of the test scenario to run", "oneOf": [ { "type": "null" @@ -44,10 +35,19 @@ { "$ref": "../../scenarios/definitions/TestScenarioDeclaration.json" } - ], - "description": "If this field is populated, declaration of the test scenario to run" + ] + }, + "test_suite": { + "description": "If this field is populated, declaration of the test suite to run", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestSuiteDeclaration.json" + } + ] } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteActionDeclaration.json", - "description": "Defines a step in the sequence of things to do for a test suite.\n\nExactly one of `test_scenario`, `test_suite`, or `action_generator` must be specified.\n\nmonitoring.uss_qualifier.suites.definitions.TestSuiteActionDeclaration, as defined in monitoring/uss_qualifier/suites/definitions.py" + "type": "object" } \ No newline at end of file From 58e968e2038d30cc053761fec085475243e3bbf7 Mon Sep 17 00:00:00 2001 From: Christopher Chiche Date: Fri, 13 Oct 2023 16:07:34 +0200 Subject: [PATCH 2/2] Handle reports starting with an action_generator --- reports/src/capability/reportParser.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/reports/src/capability/reportParser.ts b/reports/src/capability/reportParser.ts index a0df25b..3a7a46f 100644 --- a/reports/src/capability/reportParser.ts +++ b/reports/src/capability/reportParser.ts @@ -240,12 +240,15 @@ const parseActions = ( }; const _parseReport = (report: ReportsReportTestSuiteActionReport): Report => { - if (report.test_suite) { + const testSuite: ReportsReportTestSuiteReport | undefined | null = + report.test_suite || + report.action_generator?.actions.find((a) => !!a.test_suite)?.test_suite; + if (testSuite) { return { - name: report.test_suite.name, - participants: report.test_suite.capability_evaluations.map( - (c) => c.participant_id - ), + name: testSuite.name, + participants: testSuite.capability_evaluations + .map((c) => c.participant_id) + .sort(), capability: { name: "root", childCapabilities: parseActions(report),