From 7ed120ca4bdcaeca6451b543e3180a42b1f1b399 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Sat, 27 Jul 2024 15:07:40 +0200 Subject: [PATCH] [ci] Add test step to dss-deploy workflow (#1046) This PR augments the dss-deploy github actions workflow to test the deployment using the USS qualifier. Note that in addition to adding test, some dependencies checks have been set to increase the reliability of starting and destroying the cluster. A future PR will export the test results. Successful run: https://github.com/Orbitalize/dss/actions/runs/9613273501/job/26515682884 --- .../terraform-aws-kubernetes/network_lb.tf | 4 +- .../operations/ci/aws-1/test-resources.yaml | 289 ++++++++++++++++++ deploy/operations/ci/aws-1/test.sh | 13 +- 3 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 deploy/operations/ci/aws-1/test-resources.yaml diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf index 0c8df4b7e..e48ecb5dc 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf @@ -15,7 +15,9 @@ resource "helm_release" "aws-load-balancer-controller" { } depends_on = [ - aws_eks_cluster.kubernetes_cluster + aws_eks_cluster.kubernetes_cluster, + aws_iam_role_policy_attachment.AWSLoadBalancerControllerPolicy, + aws_eks_node_group.eks_node_group ] } diff --git a/deploy/operations/ci/aws-1/test-resources.yaml b/deploy/operations/ci/aws-1/test-resources.yaml new file mode 100644 index 000000000..72fd8c8ca --- /dev/null +++ b/deploy/operations/ci/aws-1/test-resources.yaml @@ -0,0 +1,289 @@ +# +# This manifest creates a namespace and the resources required to run the uss_qualifier. +# It will create the following resources: +# - Dedicated namespace +# - Config map with the uss qualifier configuration +# - Dummy oauth deployment with related service to provide tokens +# - The USS qualifier job +# +# Note that it expects the private key in a secret which can be created with the following command: +# kubectl create secret generic -n tests dummy-oauth-certs --from-file=../../../../build/test-certs/auth2.key + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tests + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: uss-qualifier-configurations + namespace: tests +data: + ci_environment.yaml: |+ + # The resources in this file describe the system/environment under test and should not change the test being run. + # This file defines the environment deployed by the github actions workflow `dss-deploy`. + + # ===== Auth ===== + utm_auth: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.communications.AuthAdapterResource + specification: + environment_variable_containing_auth_spec: AUTH_SPEC + scopes_authorized: + # ASTM F3411-22a USS emulation roles + - rid.service_provider + - rid.display_provider + # ASTM F3411-19 USS emulation roles + - dss.write.identification_service_areas + - dss.read.identification_service_areas + # ASTM F3548-21 USS emulation roles + - utm.strategic_coordination + - utm.conformance_monitoring_sa + - utm.availability_arbitration + - utm.constraint_management + + + second_utm_auth: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.communications.AuthAdapterResource + specification: + environment_variable_containing_auth_spec: AUTH_SPEC_2 + scopes_authorized: + - utm.strategic_coordination + + utm_client_identity: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.communications.ClientIdentityResource + dependencies: + auth_adapter: utm_auth + specification: + whoami_audience: localhost + whoami_scope: rid.display_provider + + # ===== NetRID ===== + + netrid_dss_instances_v19: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.astm.f3411.DSSInstancesResource + dependencies: + auth_adapter: utm_auth + specification: + dss_instances: + - participant_id: uss_aws + rid_version: F3411-19 + base_url: https://dss.ci.aws-interuss.uspace.dev + has_private_address: false + + netrid_dss_instances_v22a: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.astm.f3411.DSSInstancesResource + dependencies: + auth_adapter: utm_auth + specification: + dss_instances: + - participant_id: uss_aws + rid_version: F3411-22a + base_url: https://dss.ci.aws-interuss.uspace.dev/rid/v2 + has_private_address: false + + # ===== F3548 ===== + + scd_dss: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.astm.f3548.v21.DSSInstanceResource + dependencies: + auth_adapter: utm_auth + specification: + participant_id: uss_aws + base_url: https://dss.ci.aws-interuss.uspace.dev + has_private_address: false + + scd_dss_instances: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.astm.f3548.v21.DSSInstancesResource + dependencies: + auth_adapter: utm_auth + specification: + dss_instances: + - participant_id: uss_aws + base_url: https://dss.ci.aws-interuss.uspace.dev + has_private_address: false + + dss_crdb_cluster: + $content_schema: monitoring/uss_qualifier/resources/interuss/crdb/crdb/CockroachDBClusterResource.json + resource_type: resources.interuss.crdb.crdb.CockroachDBClusterResource + specification: + nodes: + - participant_id: uss_aws + host: 0.db.ci.aws-interuss.uspace.dev + port: 26257 + - participant_id: uss_aws + host: 1.db.ci.aws-interuss.uspace.dev + port: 26257 + - participant_id: uss_aws + host: 2.db.ci.aws-interuss.uspace.dev + port: 26257 + + aws_dss_probing.yaml: | + $content_schema: monitoring/uss_qualifier/configurations/configuration/USSQualifierConfiguration.json + v1: + test_run: + resources: + resource_declarations: + kentland_service_area: { $ref: '../dev/library/resources.yaml#/kentland_service_area' } + kentland_planning_area: { $ref: '../dev/library/resources.yaml#/kentland_planning_area' } + kentland_problematically_big_area: { $ref: '../dev/library/resources.yaml#/kentland_problematically_big_area' } + utm_auth: { $ref: './ci_environment.yaml#/utm_auth' } + second_utm_auth: {$ref: './ci_environment.yaml#/second_utm_auth'} + utm_client_identity: { $ref: '../dev/library/resources.yaml#/utm_client_identity' } + id_generator: { $ref: '../dev/library/resources.yaml#/id_generator' } + dss_crdb_cluster: { $ref: './ci_environment.yaml#/dss_crdb_cluster' } + scd_dss_instances: { $ref: './ci_environment.yaml#/scd_dss_instances' } + netrid_dss_instances_v22a: { $ref: './ci_environment.yaml#/netrid_dss_instances_v22a' } + netrid_dss_instances_v19: { $ref: './ci_environment.yaml#/netrid_dss_instances_v19' } + che_non_conflicting_flights: {$ref: '../dev/library/resources.yaml#/che_non_conflicting_flights'} + non_baseline_inputs: + - v1.test_run.resources.resource_declarations.utm_auth + - v1.test_run.resources.resource_declarations.second_utm_auth + - v1.test_run.resources.resource_declarations.dss_crdb_cluster + - v1.test_run.resources.resource_declarations.scd_dss_instances + - v1.test_run.resources.resource_declarations.netrid_dss_instances_v22a + - v1.test_run.resources.resource_declarations.netrid_dss_instances_v19 + action: + test_suite: + suite_type: suites.interuss.dss.all_tests + resources: + f3411v19_dss_instances: netrid_dss_instances_v19 + f3411v22a_dss_instances: netrid_dss_instances_v22a + f3548v21_dss_instances: scd_dss_instances + dss_crdb_cluster: dss_crdb_cluster + utm_client_identity: utm_client_identity + id_generator: id_generator + service_area: kentland_service_area + planning_area: kentland_planning_area + problematically_big_area: kentland_problematically_big_area + second_utm_auth: second_utm_auth + flight_intents: che_non_conflicting_flights + execution: + stop_fast: false + artifacts: + output_path: output/pooled_dss_probing + raw_report: { } + sequence_view: { } + tested_requirements: + - report_name: requirements + requirement_collections: + all_astm_dss_requirements: + requirement_collections: + - requirement_sets: + - astm.f3411.v22a.dss_provider + - astm.f3411.v19.dss_provider + - astm.f3548.v21.dss_provider + participant_requirements: + uss1: all_astm_dss_requirements + uss2: all_astm_dss_requirements + validation: + criteria: + - $ref: ../dev/library/validation.yaml#/execution_error_none + - $ref: ../dev/library/validation.yaml#/failed_check_severity_max_low + - applicability: + skipped_actions: {} + pass_condition: + elements: + count: + equal_to: 0 + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: uss-qualifier + namespace: tests +spec: + template: + metadata: {} + spec: + volumes: + - name: uss-qualifier-configuration + configMap: + name: uss-qualifier-configurations + - name: cache + emptyDir: {} + - name: output + emptyDir: {} + initContainers: + - name: wait-for-dss-public + image: alpine:3.17.3 + command: [ 'sh', '-c', "until wget -nv https://dss.ci.aws-interuss.uspace.dev/healthy; do echo waiting for dss to be available from the public internet; sleep 2; done" ] + containers: + - image: interuss/monitoring:v0.7.0 + name: uss-qualifier + workingDir: /app/monitoring/uss_qualifier + volumeMounts: + - name: uss-qualifier-configuration + mountPath: /app/monitoring/uss_qualifier/configurations/ci/ + - name: output + mountPath: /app/monitoring/uss_qualifier/output + - name: cache + mountPath: /app/monitoring/uss_qualifier/.templates_cache + env: + - name: PYTHONBUFFERED + value: "1" + - name: AUTH_SPEC + value: DummyOAuth(http://dummy-oauth.tests.svc.cluster.local:8085/token,uss_qualifier) + - name: AUTH_SPEC_2 + value: DummyOAuth(http://dummy-oauth.tests.svc.cluster.local:8085/token,uss_qualifier_2) + command: + - python + - main.py + args: + - --config + - configurations.ci.aws_dss_probing + restartPolicy: Never + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dummy-oauth + namespace: tests +spec: + replicas: 1 + selector: + matchLabels: + run: dummy-oauth + template: + metadata: + labels: + run: dummy-oauth + spec: + volumes: + - name: dummy-oauth-certs + secret: + secretName: dummy-oauth-certs + containers: + - image: interuss/dummy-oauth:latest + name: dummy-oauth + volumeMounts: + - mountPath: /build/test-certs/ + name: dummy-oauth-certs + ports: + - containerPort: 8085 + +--- +apiVersion: v1 +kind: Service +metadata: + name: dummy-oauth + namespace: tests + labels: + run: dummy-oauth +spec: + ports: + - port: 8085 + targetPort: 8085 + selector: + run: dummy-oauth diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index a59d833c1..eb4b2bc1b 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -38,9 +38,15 @@ cd "$BASEDIR/../../../services/helm-charts/dss" RELEASE_NAME="dss" helm dep update --kube-context="$KUBE_CONTEXT" helm upgrade --install --debug --kube-context="$KUBE_CONTEXT" -f "${WORKSPACE_LOCATION}/helm_values.yml" "$RELEASE_NAME" . -kubectl wait --for=condition=complete --timeout=3m job/rid-schema-manager-1 +kubectl wait --for=condition=complete --timeout=3m job --all + +# Test the deployment of the DSS +kubectl apply -f "$BASEDIR/test-resources.yaml" +kubectl create secret generic -n tests dummy-oauth-certs --from-file="$BASEDIR/../../../../build/test-certs/auth2.key" +kubectl wait -n tests --for=condition=complete --timeout=10m job.batch/uss-qualifier +# dummy-oauth-certs secret is deleted with the namespace using the command below +kubectl delete -f "$BASEDIR/test-resources.yaml" -# TODO: Test the deployment of the DSS if [ -n "$DO_NOT_DESTROY" ]; then echo "Destroy disabled. Exit." @@ -56,6 +62,9 @@ kubectl delete pvc --wait --all=true kubectl delete pv --wait --all=true # TODO: Check completeness +# Debug: show all resources +kubectl get all + # Delete cluster cd "$BASEDIR" terraform destroy -auto-approve