From 19814baedf1b833c496b79c4a299beccb68e12df Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jan 2025 09:31:08 -0900 Subject: [PATCH 01/39] add velocity option for OPERA_DISP_TMS jobs --- CHANGELOG.md | 5 +++++ job_spec/OPERA_DISP_TMS.yml | 1 + 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b19ae904d..4d56681e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.2.1] + +### Added +- `velocity` option for the `tile_type` parameter of `OPERA_DISP_TMS` jobs + ## [9.2.0] ### Added diff --git a/job_spec/OPERA_DISP_TMS.yml b/job_spec/OPERA_DISP_TMS.yml index 40294504c..9a5f953a8 100644 --- a/job_spec/OPERA_DISP_TMS.yml +++ b/job_spec/OPERA_DISP_TMS.yml @@ -13,6 +13,7 @@ OPERA_DISP_TMS: enum: - displacement - secant_velocity + - velocity example: displacement bounds: api_schema: From 038db24b49f36709fafbdc9ae8378c0d973bc913 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 13 Jan 2025 13:37:57 -0500 Subject: [PATCH 02/39] ruff fixes --- apps/disable-private-dns/src/disable_private_dns.py | 4 ++-- tests/test_api/test_list_jobs.py | 2 +- tests/test_upload_log.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/disable-private-dns/src/disable_private_dns.py b/apps/disable-private-dns/src/disable_private_dns.py index 53c205744..4f4e302aa 100644 --- a/apps/disable-private-dns/src/disable_private_dns.py +++ b/apps/disable-private-dns/src/disable_private_dns.py @@ -34,10 +34,10 @@ def set_private_dns_disabled(endpoint_id): def disable_private_dns(vpc_id, endpoint_name): endpoint = get_endpoint(vpc_id, endpoint_name) if endpoint['PrivateDnsEnabled']: - print(f"Private DNS enabled for VPC Endpoint: {endpoint['VpcEndpointId']}, changing...") + print(f'Private DNS enabled for VPC Endpoint: {endpoint["VpcEndpointId"]}, changing...') set_private_dns_disabled(endpoint['VpcEndpointId']) else: - print(f"Private DNS already disabled for VPC Endpoint: {endpoint['VpcEndpointId']}, doing nothing.") + print(f'Private DNS already disabled for VPC Endpoint: {endpoint["VpcEndpointId"]}, doing nothing.') def lambda_handler(event, context): diff --git a/tests/test_api/test_list_jobs.py b/tests/test_api/test_list_jobs.py index 021d1bce9..e63346d43 100644 --- a/tests/test_api/test_list_jobs.py +++ b/tests/test_api/test_list_jobs.py @@ -188,7 +188,7 @@ def test_bad_date_formats(client): '2020-13-01T00:00:00Z', '01-JAN-2020', '01/01/2020', - '2020-01-01' '2020-01-01T00:00Z', + '2020-01-012020-01-01T00:00Z', '2020-01-01T00:00:00', '2020-01-01T00:00:00+01', '2020-01-01T00:00:00+0100', diff --git a/tests/test_upload_log.py b/tests/test_upload_log.py index c5a154bf0..8d1d14a9c 100644 --- a/tests/test_upload_log.py +++ b/tests/test_upload_log.py @@ -133,7 +133,7 @@ def test_lambda_handler_no_log_stream(mock_write_log_to_s3: MagicMock): 'processing_results': { 'step_0': { 'Error': '', - 'Cause': '{"Container": {},' '"Status": "FAILED",' '"StatusReason": "foo reason",' '"Attempts": []}', + 'Cause': '{"Container": {},"Status": "FAILED","StatusReason": "foo reason","Attempts": []}', } }, } From cd47ae613300aba979a3c81d1f259dab48dc005a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 13 Jan 2025 13:48:31 -0500 Subject: [PATCH 03/39] Update tests/test_api/test_list_jobs.py Co-authored-by: Andrew Johnston --- tests/test_api/test_list_jobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_api/test_list_jobs.py b/tests/test_api/test_list_jobs.py index e63346d43..3761af27f 100644 --- a/tests/test_api/test_list_jobs.py +++ b/tests/test_api/test_list_jobs.py @@ -188,7 +188,8 @@ def test_bad_date_formats(client): '2020-13-01T00:00:00Z', '01-JAN-2020', '01/01/2020', - '2020-01-012020-01-01T00:00Z', + '2020-01-01', + '2020-01-01T00:00Z', '2020-01-01T00:00:00', '2020-01-01T00:00:00+01', '2020-01-01T00:00:00+0100', From bafe2de5fed1758415dba6eb00e1e68e9e7bd608 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:40:52 +0000 Subject: [PATCH 04/39] Bump ASFHyP3/actions from 0.13.2 to 0.14.0 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.13.2 to 0.14.0. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.13.2...v0.14.0) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/deploy-daac.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-template-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 232d1498a..071ebb41e 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,4 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.14.0 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 7646baa5f..c4e970a8d 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.14.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index 1b9206ca0..ca3416af8 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -112,6 +112,6 @@ jobs: call-bump-version-workflow: if: github.ref == 'refs/heads/main' needs: deploy - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.14.0 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 465aaa825..c3c050d8f 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.14.0 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index 1dcb24a30..7b6ddffde 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,7 +7,7 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.14.0 permissions: pull-requests: write with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4852be3f..4f6914e2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.14.0 with: release_prefix: HyP3 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 6a80718b4..625b263ec 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -8,7 +8,7 @@ on: push jobs: call-ruff-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.14.0 call-mypy-workflow: uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.14.0 @@ -84,4 +84,4 @@ jobs: snyk iac test --severity-threshold=high call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.13.2 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.14.0 From 46026ed91d1a58ca0589e1cb2ba9ba96e6ac3845 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jan 2025 15:26:21 -0900 Subject: [PATCH 05/39] restore automated deployment to hyp3-opera-disp-sandbox deployment --- .../workflows/deploy-opera-disp-sandbox.yml | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/deploy-opera-disp-sandbox.yml diff --git a/.github/workflows/deploy-opera-disp-sandbox.yml b/.github/workflows/deploy-opera-disp-sandbox.yml new file mode 100644 index 000000000..79bd24eac --- /dev/null +++ b/.github/workflows/deploy-opera-disp-sandbox.yml @@ -0,0 +1,80 @@ +name: Deploy OPERA-DISP Sandbox Stack to AWS + +on: + push: + branches: + - develop + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-opera-disp-sandbox + domain: hyp3-opera-disp-sandbox.asf.alaska.edu + template_bucket: cf-templates-1hz9ldhhl4ahu-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: DEFAULT + deploy_ref: refs/heads/develop + job_files: >- + job_spec/OPERA_DISP_TMS.yml + instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge + default_max_vcpus: 640 + expanded_max_vcpus: 640 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id + distribution_url: '' + + environment: + name: ${{ matrix.environment }} + url: https://${{ matrix.domain }} + + steps: + - uses: actions/checkout@v4.2.2 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + DOMAIN_NAME: ${{ matrix.domain }} + API_NAME: ${{ matrix.environment }} + CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} + IMAGE_TAG: ${{ matrix.image_tag }} + PRODUCT_LIFETIME: ${{ matrix.product_lifetime_in_days }} + VPC_ID: ${{ secrets.VPC_ID }} + SUBNET_IDS: ${{ secrets.SUBNET_IDS }} + SECRET_ARN: ${{ secrets.SECRET_ARN }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + DEFAULT_CREDITS_PER_USER: ${{ matrix.default_credits_per_user }} + DEFAULT_APPLICATION_STATUS: ${{ matrix.default_application_status }} + COST_PROFILE: ${{ matrix.cost_profile }} + JOB_FILES: ${{ matrix.job_files }} + DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} + EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} + MONTHLY_BUDGET: ${{ secrets.MONTHLY_BUDGET }} + REQUIRED_SURPLUS: ${{ matrix.required_surplus }} + ORIGIN_ACCESS_IDENTITY_ID: ${{ secrets.ORIGIN_ACCESS_IDENTITY_ID }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + AMI_ID: ${{ matrix.ami_id }} + INSTANCE_TYPES: ${{ matrix.instance_types }} + DISTRIBUTION_URL: ${{ matrix.distribution_url }} + AUTH_PUBLIC_KEY: ${{ secrets.AUTH_PUBLIC_KEY }} From 8c0ad409a9c609d25276c30884c214d8777da9e1 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jan 2025 15:28:15 -0900 Subject: [PATCH 06/39] fix python version --- .github/workflows/deploy-opera-disp-sandbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-opera-disp-sandbox.yml b/.github/workflows/deploy-opera-disp-sandbox.yml index 79bd24eac..7c590798a 100644 --- a/.github/workflows/deploy-opera-disp-sandbox.yml +++ b/.github/workflows/deploy-opera-disp-sandbox.yml @@ -49,7 +49,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - uses: ./.github/actions/deploy-hyp3 with: From 56d49f040ae1696ed7190db12c6c5ac832816e72 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 17 Jan 2025 15:49:06 -0900 Subject: [PATCH 07/39] upgrade to mypy action v0.15.0 --- .github/workflows/static-analysis.yml | 2 +- pyproject.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 625b263ec..2e7427253 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -11,7 +11,7 @@ jobs: uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.14.0 call-mypy-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.15.0 cfn-lint: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 17429969b..f4235043a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,10 @@ warn_unused_ignores = true warn_unreachable = true strict_equality = true check_untyped_defs = true +install_types = true +non_interactive = true +pretty = true +disable_error_code = ["import-untyped"] exclude = [ "/build/", "/setup\\.py$", From 5256ea3f1daacaa6b73c35a118d2c206b6100076 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:01:27 +0000 Subject: [PATCH 08/39] Bump cfn-lint from 1.22.3 to 1.22.5 Bumps [cfn-lint](https://github.com/aws-cloudformation/cfn-lint) from 1.22.3 to 1.22.5. - [Release notes](https://github.com/aws-cloudformation/cfn-lint/releases) - [Changelog](https://github.com/aws-cloudformation/cfn-lint/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-cloudformation/cfn-lint/compare/v1.22.3...v1.22.5) --- updated-dependencies: - dependency-name: cfn-lint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 5cfeef66c..d6dfbf029 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -15,4 +15,4 @@ ruff mypy setuptools==75.7.0 openapi-spec-validator==0.7.1 -cfn-lint==1.22.3 +cfn-lint==1.22.5 From 57494008f98502bf3ca85606d78bdefdf7a659cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:01:38 +0000 Subject: [PATCH 09/39] Bump boto3 from 1.35.93 to 1.36.2 Bumps [boto3](https://github.com/boto/boto3) from 1.35.93 to 1.36.2. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.35.93...1.36.2) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- requirements-apps-disable-private-dns.txt | 2 +- requirements-apps-start-execution-manager.txt | 2 +- requirements-apps-start-execution-worker.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-all.txt b/requirements-all.txt index 5cfeef66c..2e72fff77 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,7 +5,7 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.35.93 +boto3==1.36.2 jinja2==3.1.5 moto[dynamodb]==5.0.26 pytest==8.3.4 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index 288abcb01..a174c07ee 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.35.93 +boto3==1.36.2 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 6aed09652..01b4b5af6 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.35.93 +boto3==1.36.2 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index 59eab2e78..39b079b07 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.35.93 +boto3==1.36.2 ./lib/lambda_logging/ From 42712ffe8e6948e72d00beb0a22707ea13444ad5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:29:55 +0000 Subject: [PATCH 10/39] Bump ASFHyP3/actions from 0.14.0 to 0.15.0 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.14.0 to 0.15.0. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.14.0...v0.15.0) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/deploy-daac.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-template-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 071ebb41e..030906523 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,4 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.15.0 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index c4e970a8d..b7ffba87b 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.15.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index ca3416af8..160055119 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -112,6 +112,6 @@ jobs: call-bump-version-workflow: if: github.ref == 'refs/heads/main' needs: deploy - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.15.0 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index c3c050d8f..2d4eb5bf3 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.15.0 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index 7b6ddffde..ba1a9c4ad 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,7 +7,7 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.15.0 permissions: pull-requests: write with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f6914e2c..e01360300 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.15.0 with: release_prefix: HyP3 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 625b263ec..3a4b6bfb3 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -8,10 +8,10 @@ on: push jobs: call-ruff-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.15.0 call-mypy-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.15.0 cfn-lint: runs-on: ubuntu-latest @@ -84,4 +84,4 @@ jobs: snyk iac test --severity-threshold=high call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.14.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.15.0 From 17dedf943f57c22f7142e722b87e205db394818c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 22 Jan 2025 09:21:59 -0900 Subject: [PATCH 11/39] update code of conduct per guidance from NASA --- CODE_OF_CONDUCT.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 4547fa8c7..091d0be46 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -3,14 +3,10 @@ ## Our Pledge We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +community a harassment-free experience for everyone. We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. +and healthy community. ## Our Standards From 5aded4a3ad51b9f53cb5df8d98ef81acbf9eb429 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 19:26:47 +0000 Subject: [PATCH 12/39] Bump moto[dynamodb] from 5.0.26 to 5.0.27 Bumps [moto[dynamodb]](https://github.com/getmoto/moto) from 5.0.26 to 5.0.27. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.0.26...5.0.27) --- updated-dependencies: - dependency-name: moto[dynamodb] dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 2e72fff77..84d9869f2 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -7,7 +7,7 @@ -r requirements-apps-update-db.txt boto3==1.36.2 jinja2==3.1.5 -moto[dynamodb]==5.0.26 +moto[dynamodb]==5.0.27 pytest==8.3.4 PyYAML==6.0.2 responses==0.25.3 From 8bebfd470f69f49315a84de53cc5eede740ed5fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 19:28:54 +0000 Subject: [PATCH 13/39] Bump responses from 0.25.3 to 0.25.6 Bumps [responses](https://github.com/getsentry/responses) from 0.25.3 to 0.25.6. - [Release notes](https://github.com/getsentry/responses/releases) - [Changelog](https://github.com/getsentry/responses/blob/master/CHANGES) - [Commits](https://github.com/getsentry/responses/compare/0.25.3...0.25.6) --- updated-dependencies: - dependency-name: responses dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index c5ccd3e5f..330d7588b 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -10,7 +10,7 @@ jinja2==3.1.5 moto[dynamodb]==5.0.27 pytest==8.3.4 PyYAML==6.0.2 -responses==0.25.3 +responses==0.25.6 ruff mypy setuptools==75.7.0 From 713442c1abf277e8ae5d25347c840fe7b5e6dcc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 19:32:49 +0000 Subject: [PATCH 14/39] Bump setuptools from 75.7.0 to 75.8.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 75.7.0 to 75.8.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v75.7.0...v75.8.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 330d7588b..bb1b9e051 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -13,6 +13,6 @@ PyYAML==6.0.2 responses==0.25.6 ruff mypy -setuptools==75.7.0 +setuptools==75.8.0 openapi-spec-validator==0.7.1 cfn-lint==1.22.5 From 70f0a77a61dc3e75a1c4b650713af1c1dadee188 Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 23 Jan 2025 11:48:19 -0900 Subject: [PATCH 15/39] Add validator for SRG jobs to check bounds size + remove default bounds --- apps/api/src/hyp3_api/validation.py | 9 +++++++++ job_spec/SRG_GSLC.yml | 7 ++++--- job_spec/SRG_TIME_SERIES.yml | 7 ++++--- tests/test_api/test_validation.py | 10 ++++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 1358b1feb..eb0f005e3 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -204,6 +204,15 @@ def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: return jobs +def check_bounding_box_size(job: dict, max_bounds_area: float = 4.5): + bounds = job['job_parameters']['bounds'] + + bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0]) + + if bounds_area > max_bounds_area: + raise BoundsValidationError(f'Bounds must be smaller than {max_bounds_area} degrees squared') + + def validate_jobs(jobs: list[dict]) -> None: jobs = convert_single_burst_jobs(jobs) diff --git a/job_spec/SRG_GSLC.yml b/job_spec/SRG_GSLC.yml index 92421982e..082b17b49 100644 --- a/job_spec/SRG_GSLC.yml +++ b/job_spec/SRG_GSLC.yml @@ -1,6 +1,7 @@ SRG_GSLC: required_parameters: - granules + - bounds parameters: granules: api_schema: @@ -20,8 +21,7 @@ SRG_GSLC: bounds: api_schema: type: array - description: Bounds for extent of processing, formatted like [min lon, min lat, max lon, max lat] in EPSG:4326. Setting to [0, 0, 0, 0] will use the extent of the first granule. - default: [0.0, 0.0, 0.0, 0.0] + description: Bounds for extent of processing, formatted like [min lon, min lat, max lon, max lat] in EPSG:4326. minItems: 4 maxItems: 4 example: @@ -35,7 +35,8 @@ SRG_GSLC: validators: [ check_bounds_formatting, check_granules_intersecting_bounds, - check_same_relative_orbits + check_same_relative_orbits, + check_bounding_box_size ] cost_profiles: DEFAULT: diff --git a/job_spec/SRG_TIME_SERIES.yml b/job_spec/SRG_TIME_SERIES.yml index f073a338c..5ef802b10 100644 --- a/job_spec/SRG_TIME_SERIES.yml +++ b/job_spec/SRG_TIME_SERIES.yml @@ -1,6 +1,7 @@ SRG_TIME_SERIES: required_parameters: - granules + - bounds parameters: granules: api_schema: @@ -21,8 +22,7 @@ SRG_TIME_SERIES: bounds: api_schema: type: array - description: Bounds for extent of processing, formatted like [min lon, min lat, max lon, max lat] in EPSG:4326. Setting to [0, 0, 0, 0] will use the extent of the first granule. - default: [0.0, 0.0, 0.0, 0.0] + description: Bounds for extent of processing, formatted like [min lon, min lat, max lon, max lat] in EPSG:4326. minItems: 4 maxItems: 4 example: @@ -36,7 +36,8 @@ SRG_TIME_SERIES: validators: [ check_bounds_formatting, check_granules_intersecting_bounds, - check_same_relative_orbits + check_same_relative_orbits, + check_bounding_box_size ] cost_profiles: DEFAULT: diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 556c4d707..dd6039136 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -536,3 +536,13 @@ def test_check_same_relative_orbits(): error_pattern = r'.*69 is not 87.*' with raises(validation.GranuleValidationError, match=error_pattern): validation.check_same_relative_orbits({}, invalid_granule_metadata) + + +def test_check_bounding_box_size(): + job = {'job_parameters': {'bounds': [0, 0, 10, 10]}} + + validation.check_bounding_box_size(job, max_bounds_area=100) + + error_pattern = r'.*Bounds must be smaller.*' + with raises(validation.BoundsValidationError, match=error_pattern): + validation.check_bounding_box_size(job, max_bounds_area=99.9) From d76cc581b9ea16ddbb251478dbaf90a332c8963a Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 23 Jan 2025 11:55:38 -0900 Subject: [PATCH 16/39] update changelog --- CHANGELOG.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d56681e6..b5615bf74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.3.0] + +### Added +- Added validator to check that bounds provided are smaller than maximum size for SRG jobs + +### Removed +- Removed default bounds option for SRG jobs + ## [9.2.1] ### Added @@ -350,7 +358,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ### Changed - Update `INSAR_ISCE` and `INSAR_ISCE_TEST` job spec for GUNW version 3+ standard and custom products - `frame_id` is now a required parameter and has no default - - `compute_solid_earth_tide` and `estimate_ionosphere_delay` now default to `true` + - `compute_solid_earth_tide` and `estimate_ionosphere_delay` now default to `true` - `INSAR_ISCE_TEST` exposes custom `goldstein_filter_power`, `output_resolution`, `dense_offsets`, and `unfiltered_coherence` parameters ## [4.4.1] @@ -432,7 +440,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ## [3.10.8] ### Changed - HyP3 deployments at JPL now use On Demand instances instead of Spot instances to prevent `INSAR_ISCE` jobs from being interrupted. - This *should* be a temporary change. + This *should* be a temporary change. ## [3.10.7] ### Changed @@ -576,7 +584,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ## [3.2.0] ### Added - [`job_spec`s](job_spec/) can now specify a required set of secrets and an AWS Secrets Manage Secret ARN to pull the - secret values from. Notably, secrets are now externally managed and not part of the HyP3 stack. + secret values from. Notably, secrets are now externally managed and not part of the HyP3 stack. ## [3.1.2] ### Added @@ -601,13 +609,13 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H - The `flood_depth_estimator` parameter for `WATER_MAP` jobs is now restricted to a set of possible values. - Changed the default value for the `flood_depth_estimator` parameter for `WATER_MAP` jobs from `iterative` to `None`. A value of `None` indicates that a flood map will not be included. -- Reduced `ITS_LIVE` product lifetime cycle from 180 days to 45 days. +- Reduced `ITS_LIVE` product lifetime cycle from 180 days to 45 days. ### Removed - Removed the `include_flood_depth` parameter for `WATER_MAP` jobs. ## [2.25.0] ### Added -- `INSAR_ISCE` and `INSAR_ISCE_TEST` jobs now accept a `weather_model` parameter to specify which weather model to use +- `INSAR_ISCE` and `INSAR_ISCE_TEST` jobs now accept a `weather_model` parameter to specify which weather model to use when estimating trophospheric delay data. - Increases the memory available to `AUTORIFT` jobs for Landsat pairs @@ -647,7 +655,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ## [2.21.8] ### Changed -- AUTORIFT jobs for Sentinel-2 scenes can now only be submitted using ESA naming convention. +- AUTORIFT jobs for Sentinel-2 scenes can now only be submitted using ESA naming convention. ## [2.21.7] ### Changed @@ -706,7 +714,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ## [2.19.4] ### Changed -- `scale-cluster` now adjusts the compute environment size based on total month-to-date spending, rather than only EC2 +- `scale-cluster` now adjusts the compute environment size based on total month-to-date spending, rather than only EC2 spending. ## [2.19.3] @@ -870,7 +878,7 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H - `ASF` (default) -- AWS accounts managed by the Alaska Satellite Facility - `EDC` -- AWS accounts managed by the NASA Earthdata CLoud - `JPL` -- AWS accounts managed by the NASA Jet Propulsion Laboratory -- A `security_environment` Make variable used by the `render` target (and any target that depends on `render`). +- A `security_environment` Make variable used by the `render` target (and any target that depends on `render`). Use like `make security_environment=ASF build` ### Changed @@ -941,10 +949,10 @@ HyP3's monthly quota system has been replaced by a credits system. Previously, H ## [2.6.2](https://github.com/ASFHyP3/hyp3/compare/v2.6.1...v2.6.2) ### Added -- New `AmiId` stack parameter to specify a specific AMI for the AWS Batch compute environment +- New `AmiId` stack parameter to specify a specific AMI for the AWS Batch compute environment ### Changed -- `job_spec/*.yml` files are now explicitly selected allowing per-deployment job customization +- `job_spec/*.yml` files are now explicitly selected allowing per-deployment job customization ### Removed - `AutoriftImage`, `AutoriftNamingScheme`, and `AutoriftParameterFile` CloudFormation stack parameters @@ -999,7 +1007,7 @@ to the database but still validate it. - `name` gets only subscriptions with the given name - `job_type` gets only subscriptions with the given job type - `enabled` gets only subscriptions where `enabled` matches -- subscriptions now include `creation_date` which indicates date and time of subscription creation, responses from +- subscriptions now include `creation_date` which indicates date and time of subscription creation, responses from `GET /subscriptions` are sorted by `creation_date` decending @@ -1038,7 +1046,7 @@ to the database but still validate it. - `lib/dynamo` library to allow sharing common code among different apps. ## Changed -- `POST /jobs` responses no longer include the `job_id`, `request_time`, `status_code`, or `user_id` fields when `validate_only=true` +- `POST /jobs` responses no longer include the `job_id`, `request_time`, `status_code`, or `user_id` fields when `validate_only=true` - moved dynamodb functionality from `hyp3_api/dynamo` to `lib/dynamo` - moved job creation buisness logic from `hyp3_api/handlers` to `lib/dynamo` From 6be8b28d8d066f64ba6f8b00319693d5c928e353 Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 23 Jan 2025 12:02:41 -0900 Subject: [PATCH 17/39] for srg jobs bounds check add provided box size to error message --- apps/api/src/hyp3_api/validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index eb0f005e3..70c06916b 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -210,7 +210,9 @@ def check_bounding_box_size(job: dict, max_bounds_area: float = 4.5): bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0]) if bounds_area > max_bounds_area: - raise BoundsValidationError(f'Bounds must be smaller than {max_bounds_area} degrees squared') + raise BoundsValidationError( + f'Bounds must be smaller than {max_bounds_area} degrees squared. Box provided was {bounds_area}' + ) def validate_jobs(jobs: list[dict]) -> None: From 2e4d4f83100ca1932114adad655015d7b521717a Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 23 Jan 2025 12:06:27 -0900 Subject: [PATCH 18/39] SRG bounds check rounds to 2 decimal places in error message --- apps/api/src/hyp3_api/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 70c06916b..15a2db9d3 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -211,7 +211,7 @@ def check_bounding_box_size(job: dict, max_bounds_area: float = 4.5): if bounds_area > max_bounds_area: raise BoundsValidationError( - f'Bounds must be smaller than {max_bounds_area} degrees squared. Box provided was {bounds_area}' + f'Bounds must be smaller than {max_bounds_area} degrees squared. Box provided was {bounds_area:.2f}' ) From 4db27170fd60b4e06cf0fd6fe32dd564a6244cdb Mon Sep 17 00:00:00 2001 From: William Horn Date: Thu, 23 Jan 2025 12:31:20 -0900 Subject: [PATCH 19/39] update example SRG job bounds --- job_spec/SRG_GSLC.yml | 10 +++++----- job_spec/SRG_TIME_SERIES.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/job_spec/SRG_GSLC.yml b/job_spec/SRG_GSLC.yml index 082b17b49..d420dc6fd 100644 --- a/job_spec/SRG_GSLC.yml +++ b/job_spec/SRG_GSLC.yml @@ -25,13 +25,13 @@ SRG_GSLC: minItems: 4 maxItems: 4 example: - - -116.583 - - 35.714 - - -113.209 - - 38.138 + - -114.87 + - 36.00 + - -114.66 + - 36.16 items: type: number - example: -116.583 + example: -114.87 validators: [ check_bounds_formatting, check_granules_intersecting_bounds, diff --git a/job_spec/SRG_TIME_SERIES.yml b/job_spec/SRG_TIME_SERIES.yml index 5ef802b10..4a8929080 100644 --- a/job_spec/SRG_TIME_SERIES.yml +++ b/job_spec/SRG_TIME_SERIES.yml @@ -26,13 +26,13 @@ SRG_TIME_SERIES: minItems: 4 maxItems: 4 example: - - -124.41473278572731 - - 37.098700238673814 - - -120.9825007499895 - - 39.52359974376425 + - -122.53 + - 37.78 + - -122.44 + - 37.85 items: type: number - example: -124.41473278572731 + example: -122.53 validators: [ check_bounds_formatting, check_granules_intersecting_bounds, From 06a595b2dbe1f26e4fe3bc139fb06e7da6a3ac5c Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:15:07 -0600 Subject: [PATCH 20/39] fix arguments for bounds area check --- apps/api/src/hyp3_api/validation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 15a2db9d3..01535344a 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,10 +6,9 @@ import requests import yaml -from shapely.geometry import MultiPolygon, Polygon, shape - from hyp3_api import CMR_URL from hyp3_api.util import get_granules +from shapely.geometry import MultiPolygon, Polygon, shape DEM_COVERAGE = None @@ -204,7 +203,7 @@ def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: return jobs -def check_bounding_box_size(job: dict, max_bounds_area: float = 4.5): +def check_bounding_box_size(job: dict, _, max_bounds_area: float = 4.5): bounds = job['job_parameters']['bounds'] bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0]) From da54dc548c09a7ef99a3f6e70510d8a85400b2ae Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:16:09 -0600 Subject: [PATCH 21/39] fix import order --- apps/api/src/hyp3_api/validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 01535344a..030d2c359 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,9 +6,10 @@ import requests import yaml +from shapely.geometry import MultiPolygon, Polygon, shape + from hyp3_api import CMR_URL from hyp3_api.util import get_granules -from shapely.geometry import MultiPolygon, Polygon, shape DEM_COVERAGE = None From d678046284c23cb05e0ade92a8ef6613aa742421 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:19:17 -0600 Subject: [PATCH 22/39] fix tests --- tests/test_api/test_validation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index dd6039136..abf4e1d50 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,8 +1,8 @@ import responses from pytest import raises -from shapely.geometry import Polygon from hyp3_api import CMR_URL, validation +from shapely.geometry import Polygon from test_api.conftest import setup_requests_mock_with_given_polygons @@ -541,8 +541,8 @@ def test_check_same_relative_orbits(): def test_check_bounding_box_size(): job = {'job_parameters': {'bounds': [0, 0, 10, 10]}} - validation.check_bounding_box_size(job, max_bounds_area=100) + validation.check_bounding_box_size(job, None, max_bounds_area=100) error_pattern = r'.*Bounds must be smaller.*' with raises(validation.BoundsValidationError, match=error_pattern): - validation.check_bounding_box_size(job, max_bounds_area=99.9) + validation.check_bounding_box_size(job, None, max_bounds_area=99.9) From c49db5ffe7f5b90d7ccb3f274fa4610a9f2b6fdf Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:20:29 -0600 Subject: [PATCH 23/39] ruff formatting --- apps/api/src/hyp3_api/validation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 030d2c359..3179c8c6b 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,10 +6,9 @@ import requests import yaml -from shapely.geometry import MultiPolygon, Polygon, shape - from hyp3_api import CMR_URL from hyp3_api.util import get_granules +from shapely.geometry import MultiPolygon, Polygon, shape DEM_COVERAGE = None @@ -204,7 +203,7 @@ def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: return jobs -def check_bounding_box_size(job: dict, _, max_bounds_area: float = 4.5): +def check_bounding_box_size(job: dict, granule_metadata, max_bounds_area: float = 4.5): bounds = job['job_parameters']['bounds'] bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0]) From be5b4e148be178dd5dc7230bcec5a7a2b01f2cdc Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:23:11 -0600 Subject: [PATCH 24/39] fix ruff 2 --- apps/api/src/hyp3_api/validation.py | 3 ++- tests/test_api/test_validation.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 3179c8c6b..162e19189 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,9 +6,10 @@ import requests import yaml +from shapely.geometry import MultiPolygon, Polygon, shape + from hyp3_api import CMR_URL from hyp3_api.util import get_granules -from shapely.geometry import MultiPolygon, Polygon, shape DEM_COVERAGE = None diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index abf4e1d50..927462d8f 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,10 +1,11 @@ import responses from pytest import raises -from hyp3_api import CMR_URL, validation from shapely.geometry import Polygon from test_api.conftest import setup_requests_mock_with_given_polygons +from hyp3_api import CMR_URL, validation + def rectangle(north, south, east, west): return Polygon([[west, north], [east, north], [east, south], [west, south]]) From 703ebfa90c7d8ba4b32163ad38164948da78ffae Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:25:38 -0600 Subject: [PATCH 25/39] ruff fix 3 --- tests/test_api/test_validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 927462d8f..abf4e1d50 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,11 +1,10 @@ import responses from pytest import raises +from hyp3_api import CMR_URL, validation from shapely.geometry import Polygon from test_api.conftest import setup_requests_mock_with_given_polygons -from hyp3_api import CMR_URL, validation - def rectangle(north, south, east, west): return Polygon([[west, north], [east, north], [east, south], [west, south]]) From f2952b56519c844a93e2969e10f1977dfa96e4ad Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:34:04 -0600 Subject: [PATCH 26/39] fix ruff 4 --- tests/test_api/test_validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index abf4e1d50..96d333ba3 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,9 +1,10 @@ import responses from pytest import raises -from hyp3_api import CMR_URL, validation from shapely.geometry import Polygon + from test_api.conftest import setup_requests_mock_with_given_polygons +from hyp3_api import CMR_URL, validation def rectangle(north, south, east, west): From 3587143e5b2bab17b58696ada93ddae61305e3f3 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:34:40 -0600 Subject: [PATCH 27/39] fix ruff 5 --- tests/test_api/test_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 96d333ba3..e5728f95c 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,6 +1,5 @@ import responses from pytest import raises - from shapely.geometry import Polygon from test_api.conftest import setup_requests_mock_with_given_polygons From 68a8ea2ab6212bda8f5f19d55ab5a71bfd9d94b2 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 24 Jan 2025 12:37:18 -0600 Subject: [PATCH 28/39] fix ruff 6 --- tests/test_api/test_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index e5728f95c..c378833e6 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -2,8 +2,8 @@ from pytest import raises from shapely.geometry import Polygon -from test_api.conftest import setup_requests_mock_with_given_polygons from hyp3_api import CMR_URL, validation +from test_api.conftest import setup_requests_mock_with_given_polygons def rectangle(north, south, east, west): From 9213ccdcd3fa022428b3d0118e24d27fd4acf8eb Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 24 Jan 2025 10:39:34 -0900 Subject: [PATCH 29/39] Add unit test to check validator function signiture --- apps/api/src/hyp3_api/validation.py | 14 +++++++------- tests/test_api/test_validation.py | 22 +++++++++++++++++----- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 162e19189..0e5f5785e 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -69,7 +69,7 @@ def is_third_party_granule(granule): return granule.startswith('S2') or granule.startswith('L') -def check_granules_exist(granules, granule_metadata): +def make_sure_granules_exist(granules, granule_metadata): found_granules = [granule['name'] for granule in granule_metadata] not_found_granules = set(granules) - set(found_granules) not_found_granules = {granule for granule in not_found_granules if not is_third_party_granule(granule)} @@ -77,13 +77,13 @@ def check_granules_exist(granules, granule_metadata): raise GranuleValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}') -def check_dem_coverage(_, granule_metadata): +def check_dem_coverage(job, granule_metadata): bad_granules = [g['name'] for g in granule_metadata if not has_sufficient_coverage(g['polygon'])] if bad_granules: raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}') -def check_same_burst_ids(job, _): +def check_same_burst_ids(job, granule_metadata): refs = job['job_parameters']['reference'] secs = job['job_parameters']['secondary'] ref_ids = ['_'.join(ref.split('_')[1:3]) for ref in refs] @@ -103,7 +103,7 @@ def check_same_burst_ids(job, _): ) -def check_valid_polarizations(job, _): +def check_valid_polarizations(job, granule_metadata): polarizations = set(granule.split('_')[4] for granule in get_granules([job])) if len(polarizations) > 1: raise GranuleValidationError( @@ -115,7 +115,7 @@ def check_valid_polarizations(job, _): ) -def check_not_antimeridian(_, granule_metadata): +def check_not_antimeridian(job, granule_metadata): for granule in granule_metadata: bbox = granule['polygon'].bounds if abs(bbox[0] - bbox[2]) > 180.0 and bbox[0] * bbox[2] < 0.0: @@ -140,7 +140,7 @@ def get_multipolygon_from_geojson(input_file): return MultiPolygon(polygons) -def check_bounds_formatting(job, _): +def check_bounds_formatting(job, granule_metadata): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: return @@ -221,7 +221,7 @@ def validate_jobs(jobs: list[dict]) -> None: granules = get_granules(jobs) granule_metadata = get_cmr_metadata(granules) - check_granules_exist(granules, granule_metadata) + make_sure_granules_exist(granules, granule_metadata) for job in jobs: for validator_name in JOB_VALIDATION_MAP[job['job_type']]: job_granule_metadata = [granule for granule in granule_metadata if granule['name'] in get_granules([job])] diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index c378833e6..c8ed6561f 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,3 +1,5 @@ +import inspect + import responses from pytest import raises from shapely.geometry import Polygon @@ -295,7 +297,7 @@ def test_check_valid_polarizations(): validation.check_valid_polarizations(invalid_job, {}) -def test_check_granules_exist(): +def test_make_sure_granules_exist(): granule_metadata = [ { 'name': 'scene1', @@ -305,12 +307,12 @@ def test_check_granules_exist(): }, ] - validation.check_granules_exist([], granule_metadata) - validation.check_granules_exist(['scene1'], granule_metadata) - validation.check_granules_exist(['scene1', 'scene2'], granule_metadata) + validation.make_sure_granules_exist([], granule_metadata) + validation.make_sure_granules_exist(['scene1'], granule_metadata) + validation.make_sure_granules_exist(['scene1', 'scene2'], granule_metadata) with raises(validation.GranuleValidationError) as e: - validation.check_granules_exist( + validation.make_sure_granules_exist( ['scene1', 'scene2', 'scene3', 'scene4', 'S2_foo', 'LC08_bar', 'LC09_bar'], granule_metadata ) assert 'S2_foo' not in str(e) @@ -470,6 +472,16 @@ def test_validate_jobs(): validation.validate_jobs(jobs) +def test_all_check_validators_have_correct_signature(): + validators = [getattr(validation, attr) for attr in dir(validation) if 'check' in attr] + + for validator in validators: + function_sig = inspect.signature(validator).parameters + + assert 'job' in function_sig + assert 'granule_metadata' in function_sig + + def test_check_bounds_formatting(): valid_jobs = [ {'job_parameters': {'bounds': [-10, 0, 10, 10]}}, From 611eea98e12969e4c2b4bae8a75329cebca0dc9f Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 24 Jan 2025 11:04:51 -0900 Subject: [PATCH 30/39] Allow for _ in validator function signatures --- apps/api/src/hyp3_api/validation.py | 14 +++++++------- tests/test_api/test_validation.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 0e5f5785e..5a384938a 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -77,13 +77,13 @@ def make_sure_granules_exist(granules, granule_metadata): raise GranuleValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}') -def check_dem_coverage(job, granule_metadata): +def check_dem_coverage(_, granule_metadata): bad_granules = [g['name'] for g in granule_metadata if not has_sufficient_coverage(g['polygon'])] if bad_granules: raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}') -def check_same_burst_ids(job, granule_metadata): +def check_same_burst_ids(job, _): refs = job['job_parameters']['reference'] secs = job['job_parameters']['secondary'] ref_ids = ['_'.join(ref.split('_')[1:3]) for ref in refs] @@ -103,7 +103,7 @@ def check_same_burst_ids(job, granule_metadata): ) -def check_valid_polarizations(job, granule_metadata): +def check_valid_polarizations(job, _): polarizations = set(granule.split('_')[4] for granule in get_granules([job])) if len(polarizations) > 1: raise GranuleValidationError( @@ -115,7 +115,7 @@ def check_valid_polarizations(job, granule_metadata): ) -def check_not_antimeridian(job, granule_metadata): +def check_not_antimeridian(_, granule_metadata): for granule in granule_metadata: bbox = granule['polygon'].bounds if abs(bbox[0] - bbox[2]) > 180.0 and bbox[0] * bbox[2] < 0.0: @@ -140,7 +140,7 @@ def get_multipolygon_from_geojson(input_file): return MultiPolygon(polygons) -def check_bounds_formatting(job, granule_metadata): +def check_bounds_formatting(job, _): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: return @@ -177,7 +177,7 @@ def check_granules_intersecting_bounds(job, granule_metadata): raise GranuleValidationError(f'The following granules do not intersect the provided bounds: {bad_granules}.') -def check_same_relative_orbits(job, granule_metadata): +def check_same_relative_orbits(_, granule_metadata): previous_relative_orbit = None for granule in granule_metadata: name_split = granule['name'].split('_') @@ -204,7 +204,7 @@ def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: return jobs -def check_bounding_box_size(job: dict, granule_metadata, max_bounds_area: float = 4.5): +def check_bounding_box_size(job: dict, _, max_bounds_area: float = 4.5): bounds = job['job_parameters']['bounds'] bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0]) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index c8ed6561f..3f4f52746 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -476,10 +476,10 @@ def test_all_check_validators_have_correct_signature(): validators = [getattr(validation, attr) for attr in dir(validation) if 'check' in attr] for validator in validators: - function_sig = inspect.signature(validator).parameters + function_params = list(inspect.signature(validator).parameters) - assert 'job' in function_sig - assert 'granule_metadata' in function_sig + assert 'job' or '_' in function_params[0] + assert 'granule_metadata' or '_' in function_params[1] def test_check_bounds_formatting(): From f08b50fb01bc578b38b3de15052f95496ab86188 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 24 Jan 2025 11:25:46 -0900 Subject: [PATCH 31/39] rename file variable --- apps/api/src/hyp3_api/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 5a384938a..c0e27a49a 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -23,8 +23,8 @@ class BoundsValidationError(Exception): pass -with open(Path(__file__).parent / 'job_validation_map.yml') as f: - JOB_VALIDATION_MAP = yaml.safe_load(f.read()) +with open(Path(__file__).parent / 'job_validation_map.yml') as job_validation_map_file: + JOB_VALIDATION_MAP = yaml.safe_load(job_validation_map_file.read()) def has_sufficient_coverage(granule: Polygon): From e2d7c368bd4df960fc18b76fa562e666e42580e0 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 24 Jan 2025 11:26:34 -0900 Subject: [PATCH 32/39] prefix private validation.py functions with underscore --- apps/api/src/hyp3_api/validation.py | 28 +++++++-------- tests/test_api/test_validation.py | 54 ++++++++++++++--------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index c0e27a49a..80fbff1ee 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -27,15 +27,15 @@ class BoundsValidationError(Exception): JOB_VALIDATION_MAP = yaml.safe_load(job_validation_map_file.read()) -def has_sufficient_coverage(granule: Polygon): +def _has_sufficient_coverage(granule: Polygon): global DEM_COVERAGE if DEM_COVERAGE is None: - DEM_COVERAGE = get_multipolygon_from_geojson('dem_coverage_map_cop30.geojson') + DEM_COVERAGE = _get_multipolygon_from_geojson('dem_coverage_map_cop30.geojson') return granule.intersects(DEM_COVERAGE) -def get_cmr_metadata(granules): +def _get_cmr_metadata(granules): cmr_parameters = { 'granule_ur': [f'{granule}*' for granule in granules], 'options[granule_ur][pattern]': 'true', @@ -58,27 +58,27 @@ def get_cmr_metadata(granules): granules = [ { 'name': entry.get('producer_granule_id', entry.get('title')), - 'polygon': Polygon(format_points(entry['polygons'][0][0])), + 'polygon': Polygon(_format_points(entry['polygons'][0][0])), } for entry in response.json()['feed']['entry'] ] return granules -def is_third_party_granule(granule): +def _is_third_party_granule(granule): return granule.startswith('S2') or granule.startswith('L') -def make_sure_granules_exist(granules, granule_metadata): +def _make_sure_granules_exist(granules, granule_metadata): found_granules = [granule['name'] for granule in granule_metadata] not_found_granules = set(granules) - set(found_granules) - not_found_granules = {granule for granule in not_found_granules if not is_third_party_granule(granule)} + not_found_granules = {granule for granule in not_found_granules if not _is_third_party_granule(granule)} if not_found_granules: raise GranuleValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}') def check_dem_coverage(_, granule_metadata): - bad_granules = [g['name'] for g in granule_metadata if not has_sufficient_coverage(g['polygon'])] + bad_granules = [g['name'] for g in granule_metadata if not _has_sufficient_coverage(g['polygon'])] if bad_granules: raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}') @@ -126,13 +126,13 @@ def check_not_antimeridian(_, granule_metadata): raise GranuleValidationError(msg) -def format_points(point_string): +def _format_points(point_string): converted_to_float = [float(x) for x in point_string.split(' ')] points = [list(t) for t in zip(converted_to_float[1::2], converted_to_float[::2])] return points -def get_multipolygon_from_geojson(input_file): +def _get_multipolygon_from_geojson(input_file): dem_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), input_file) with open(dem_file) as f: shp = json.load(f)['features'][0]['geometry'] @@ -194,7 +194,7 @@ def check_same_relative_orbits(_, granule_metadata): ) -def convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: +def _convert_single_burst_jobs(jobs: list[dict]) -> list[dict]: jobs = deepcopy(jobs) for job in jobs: if job['job_type'] == 'INSAR_ISCE_BURST': @@ -216,12 +216,12 @@ def check_bounding_box_size(job: dict, _, max_bounds_area: float = 4.5): def validate_jobs(jobs: list[dict]) -> None: - jobs = convert_single_burst_jobs(jobs) + jobs = _convert_single_burst_jobs(jobs) granules = get_granules(jobs) - granule_metadata = get_cmr_metadata(granules) + granule_metadata = _get_cmr_metadata(granules) - make_sure_granules_exist(granules, granule_metadata) + _make_sure_granules_exist(granules, granule_metadata) for job in jobs: for validator_name in JOB_VALIDATION_MAP[job['job_type']]: job_granule_metadata = [granule for granule in granule_metadata if granule['name'] in get_granules([job])] diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 3f4f52746..155e3bf85 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -30,49 +30,49 @@ def test_not_antimeridian(): def test_has_sufficient_coverage(): # Wyoming poly = rectangle(45, 41, -104, -111) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # completely covered Aleutian Islands over antimeridian; should pass with fixed antimeridian poly = rectangle(51.7, 51.3, 179.7, -179.3) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # not enough coverage of Aleutian Islands over antimeridian # NOTE: Passes today but should FAIL legacy with antimeridian feature fix poly = rectangle(51.7, 41.3, 179.7, -179.3) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # completely encloses tile over Ascension Island in the Atlantic poly = rectangle(-6, -9, -15, -14) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # # minimum sufficient coverage off the coast of Eureka, CA poly = rectangle(40.1, 40, -126, -125.000138) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # almost minimum sufficient coverage off the coast of Eureka, CA poly = rectangle(40.1, 40, -126, -125.000140) - assert not validation.has_sufficient_coverage(poly) + assert not validation._has_sufficient_coverage(poly) # polygon in missing tile over Gulf of California poly = rectangle(26.9, 26.1, -110.1, -110.9) - assert not validation.has_sufficient_coverage(poly) + assert not validation._has_sufficient_coverage(poly) # southern Greenland poly = rectangle(62, 61, -44, -45) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # Antarctica poly = rectangle(-62, -90, 180, -180) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) # ocean over antimeridian; this case incorrectly passes, see https://github.com/ASFHyP3/hyp3/issues/1989 poly = rectangle(-40, -41, 179.7, -179.3) - assert validation.has_sufficient_coverage(poly) + assert validation._has_sufficient_coverage(poly) def test_format_points(): point_string = '-31.43 25.04 -29.76 25.54 -29.56 24.66 -31.23 24.15 -31.43 25.04' - assert validation.format_points(point_string) == [ + assert validation._format_points(point_string) == [ [25.04, -31.43], [25.54, -29.76], [24.66, -29.56], @@ -307,12 +307,12 @@ def test_make_sure_granules_exist(): }, ] - validation.make_sure_granules_exist([], granule_metadata) - validation.make_sure_granules_exist(['scene1'], granule_metadata) - validation.make_sure_granules_exist(['scene1', 'scene2'], granule_metadata) + validation._make_sure_granules_exist([], granule_metadata) + validation._make_sure_granules_exist(['scene1'], granule_metadata) + validation._make_sure_granules_exist(['scene1', 'scene2'], granule_metadata) with raises(validation.GranuleValidationError) as e: - validation.make_sure_granules_exist( + validation._make_sure_granules_exist( ['scene1', 'scene2', 'scene3', 'scene4', 'S2_foo', 'LC08_bar', 'LC09_bar'], granule_metadata ) assert 'S2_foo' not in str(e) @@ -325,17 +325,17 @@ def test_make_sure_granules_exist(): def test_is_third_party_granule(): - assert validation.is_third_party_granule('S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912') - assert validation.is_third_party_granule('S2B_22WEB_20200612_0_L1C') - assert validation.is_third_party_granule('LC08_L1TP_009011_20200820_20200905_02_T1') - assert validation.is_third_party_granule('LO08_L1GT_043001_20201106_20201110_02_T2') - assert validation.is_third_party_granule('LT08_L1GT_041001_20200125_20200925_02_T2') - assert validation.is_third_party_granule('LC09_L1GT_215109_20220125_20220125_02_T2') - assert validation.is_third_party_granule('LO09_L1GT_215109_20220210_20220210_02_T2') - assert validation.is_third_party_granule('LT09_L1GT_215109_20220210_20220210_02_T2') - assert not validation.is_third_party_granule('S1_249434_IW1_20230523T170733_VV_8850-BURST') - assert not validation.is_third_party_granule('S1A_IW_SLC__1SSH_20150608T205059_20150608T205126_006287_0083E8_C4F0') - assert not validation.is_third_party_granule('foo') + assert validation._is_third_party_granule('S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912') + assert validation._is_third_party_granule('S2B_22WEB_20200612_0_L1C') + assert validation._is_third_party_granule('LC08_L1TP_009011_20200820_20200905_02_T1') + assert validation._is_third_party_granule('LO08_L1GT_043001_20201106_20201110_02_T2') + assert validation._is_third_party_granule('LT08_L1GT_041001_20200125_20200925_02_T2') + assert validation._is_third_party_granule('LC09_L1GT_215109_20220125_20220125_02_T2') + assert validation._is_third_party_granule('LO09_L1GT_215109_20220210_20220210_02_T2') + assert validation._is_third_party_granule('LT09_L1GT_215109_20220210_20220210_02_T2') + assert not validation._is_third_party_granule('S1_249434_IW1_20230523T170733_VV_8850-BURST') + assert not validation._is_third_party_granule('S1A_IW_SLC__1SSH_20150608T205059_20150608T205126_006287_0083E8_C4F0') + assert not validation._is_third_party_granule('foo') @responses.activate @@ -356,7 +356,7 @@ def test_get_cmr_metadata(): } responses.post(CMR_URL, json=response_payload) - assert validation.get_cmr_metadata(['foo', 'bar', 'hello']) == [ + assert validation._get_cmr_metadata(['foo', 'bar', 'hello']) == [ { 'name': 'foo', 'polygon': Polygon([[25.0, -31.4], [25.5, -29.7], [24.6, -29.5], [24.1, -31.2]]), From 34a92d1d793b90f60606afcfa40e04557ed2a7d1 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 24 Jan 2025 11:50:00 -0900 Subject: [PATCH 33/39] updates to validator function signature test --- tests/test_api/test_validation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 155e3bf85..1c4ef0510 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -472,14 +472,14 @@ def test_validate_jobs(): validation.validate_jobs(jobs) -def test_all_check_validators_have_correct_signature(): - validators = [getattr(validation, attr) for attr in dir(validation) if 'check' in attr] +def test_all_validators_have_correct_signature(): + validators = [getattr(validation, attr) for attr in dir(validation) if attr.startswith('check_')] for validator in validators: function_params = list(inspect.signature(validator).parameters) - assert 'job' or '_' in function_params[0] - assert 'granule_metadata' or '_' in function_params[1] + assert function_params[0] in ('job', '_') + assert function_params[1] in ('granule_metadata', '_') def test_check_bounds_formatting(): From 4b0ca169429ba0cfcf53490e08ff5f2b4393c3f2 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 24 Jan 2025 12:05:20 -0900 Subject: [PATCH 34/39] Add assertion to validators unit test --- tests/test_api/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 1c4ef0510..29bc622b6 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -478,6 +478,7 @@ def test_all_validators_have_correct_signature(): for validator in validators: function_params = list(inspect.signature(validator).parameters) + assert len(function_params) >= 2 assert function_params[0] in ('job', '_') assert function_params[1] in ('granule_metadata', '_') From e23fdcd26a117c7d44a38706f1c1bacd0de29039 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 24 Jan 2025 14:17:13 -0900 Subject: [PATCH 35/39] fix changelog --- CHANGELOG.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5615bf74..1dc611399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.3.0] ### Added -- Added validator to check that bounds provided are smaller than maximum size for SRG jobs +- Added `velocity` option for the `tile_type` parameter of `OPERA_DISP_TMS` jobs +- Restored previously deleted `hyp3-opera-disp-sandbox` deployment +- Added validator to check that bounds provided do not exceed maximum size for SRG jobs ### Removed - Removed default bounds option for SRG jobs -## [9.2.1] - -### Added -- `velocity` option for the `tile_type` parameter of `OPERA_DISP_TMS` jobs - ## [9.2.0] ### Added From 555ca60b63243eae676a83d3df5be06aac47e5c7 Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 27 Jan 2025 10:03:36 -0900 Subject: [PATCH 36/39] remove ability to pass [0,0,0,0] as valid bounds --- apps/api/src/hyp3_api/validation.py | 12 ++++++++---- tests/test_api/test_validation.py | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 80fbff1ee..3ecabc1d9 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -143,7 +143,9 @@ def _get_multipolygon_from_geojson(input_file): def check_bounds_formatting(job, _): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: - return + raise BoundsValidationError( + 'Invalid bounds. Bounds cannot be [0, 0, 0, 0].' + ) if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]: raise BoundsValidationError( @@ -165,9 +167,11 @@ def bad_lon(lon): def check_granules_intersecting_bounds(job, granule_metadata): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: - bounds = granule_metadata[0]['polygon'] - else: - bounds = Polygon.from_bounds(*bounds) + raise BoundsValidationError( + 'Invalid bounds. Bounds cannot be [0, 0, 0, 0].' + ) + + bounds = Polygon.from_bounds(*bounds) bad_granules = [] for granule in granule_metadata: bbox = granule['polygon'] diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 29bc622b6..8ed9d1d23 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -488,7 +488,6 @@ def test_check_bounds_formatting(): {'job_parameters': {'bounds': [-10, 0, 10, 10]}}, {'job_parameters': {'bounds': [-180, -90, -170, -80]}}, {'job_parameters': {'bounds': [170, 75, 180, 90]}}, - {'job_parameters': {'bounds': [0, 0, 0, 0]}}, ] invalid_jobs_bad_order = [ {'job_parameters': {'bounds': [10, 0, -10, 10]}}, @@ -502,6 +501,9 @@ def test_check_bounds_formatting(): {'job_parameters': {'bounds': [-10, -100, 10, 80]}}, {'job_parameters': {'bounds': [-100, 0, 200, 10]}}, ] + + default_job = {'job_parameters': {'bounds': [0, 0, 0, 0]}} + for valid_job in valid_jobs: validation.check_bounds_formatting(valid_job, {}) for invalid_job in invalid_jobs_bad_order: @@ -511,6 +513,10 @@ def test_check_bounds_formatting(): with raises(validation.BoundsValidationError, match=r'.*Invalid lon/lat value(s)*'): validation.check_bounds_formatting(invalid_job, {}) + with raises(validation.BoundsValidationError, match=r'.*Bounds cannot be.*'): + validation.check_bounds_formatting(default_job, {}) + + def test_check_granules_intersecting_bounds(): job_with_specified_bounds = {'job_parameters': {'bounds': [-10, 0, 10, 10]}} @@ -528,12 +534,19 @@ def test_check_granules_intersecting_bounds(): {'name': 'does_not_intersect3', 'polygon': Polygon.from_bounds(100.0, -50.0, 120.0, -0.1)}, ] validation.check_granules_intersecting_bounds(job_with_specified_bounds, valid_granule_metadata) - validation.check_granules_intersecting_bounds(job_with_default_bounds, valid_granule_metadata) + + error_pattern = r'.*Bounds cannot be.*' + with raises(validation.BoundsValidationError, match=error_pattern): + validation.check_granules_intersecting_bounds(job_with_default_bounds, valid_granule_metadata) + + with raises(validation.BoundsValidationError, match=error_pattern): + validation.check_granules_intersecting_bounds(job_with_default_bounds, invalid_granule_metadata) + error_pattern = r".*bounds: \['does_not_intersect1', 'does_not_intersect2', 'does_not_intersect3'\]*" + with raises(validation.GranuleValidationError, match=error_pattern): validation.check_granules_intersecting_bounds(job_with_specified_bounds, invalid_granule_metadata) - with raises(validation.GranuleValidationError, match=error_pattern): - validation.check_granules_intersecting_bounds(job_with_default_bounds, invalid_granule_metadata) + def test_check_same_relative_orbits(): From 93144903035d631377145eedcc202ccbcbd03a26 Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 27 Jan 2025 10:06:22 -0900 Subject: [PATCH 37/39] update code formatting --- apps/api/src/hyp3_api/validation.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 3ecabc1d9..7a882bea4 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -143,9 +143,7 @@ def _get_multipolygon_from_geojson(input_file): def check_bounds_formatting(job, _): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: - raise BoundsValidationError( - 'Invalid bounds. Bounds cannot be [0, 0, 0, 0].' - ) + raise BoundsValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].') if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]: raise BoundsValidationError( @@ -167,9 +165,7 @@ def bad_lon(lon): def check_granules_intersecting_bounds(job, granule_metadata): bounds = job['job_parameters']['bounds'] if bounds == [0.0, 0.0, 0.0, 0.0]: - raise BoundsValidationError( - 'Invalid bounds. Bounds cannot be [0, 0, 0, 0].' - ) + raise BoundsValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].') bounds = Polygon.from_bounds(*bounds) bad_granules = [] From 86bd840d01a5c7fbde20bc33b666d1c333053bf1 Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 27 Jan 2025 10:07:45 -0900 Subject: [PATCH 38/39] updates for ruff --- tests/test_api/test_validation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 8ed9d1d23..565ca973e 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -517,7 +517,6 @@ def test_check_bounds_formatting(): validation.check_bounds_formatting(default_job, {}) - def test_check_granules_intersecting_bounds(): job_with_specified_bounds = {'job_parameters': {'bounds': [-10, 0, 10, 10]}} job_with_default_bounds = {'job_parameters': {'bounds': [0, 0, 0, 0]}} @@ -548,7 +547,6 @@ def test_check_granules_intersecting_bounds(): validation.check_granules_intersecting_bounds(job_with_specified_bounds, invalid_granule_metadata) - def test_check_same_relative_orbits(): valid_granule_metadata = [ {'name': 'S1A_IW_RAW__0SDV_20201015T161622_20201015T161654_034809_040E95_AF3C'}, From a96a075686c4d420457535024321fe947ee9e05f Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 27 Jan 2025 10:12:02 -0900 Subject: [PATCH 39/39] update variable name in validator test --- tests/test_api/test_validation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 565ca973e..424699645 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -502,7 +502,7 @@ def test_check_bounds_formatting(): {'job_parameters': {'bounds': [-100, 0, 200, 10]}}, ] - default_job = {'job_parameters': {'bounds': [0, 0, 0, 0]}} + job_with_bad_bounds = {'job_parameters': {'bounds': [0, 0, 0, 0]}} for valid_job in valid_jobs: validation.check_bounds_formatting(valid_job, {}) @@ -514,12 +514,12 @@ def test_check_bounds_formatting(): validation.check_bounds_formatting(invalid_job, {}) with raises(validation.BoundsValidationError, match=r'.*Bounds cannot be.*'): - validation.check_bounds_formatting(default_job, {}) + validation.check_bounds_formatting(job_with_bad_bounds, {}) def test_check_granules_intersecting_bounds(): job_with_specified_bounds = {'job_parameters': {'bounds': [-10, 0, 10, 10]}} - job_with_default_bounds = {'job_parameters': {'bounds': [0, 0, 0, 0]}} + job_with_bad_bounds = {'job_parameters': {'bounds': [0, 0, 0, 0]}} valid_granule_metadata = [ {'name': 'intersects1', 'polygon': Polygon.from_bounds(-10.0, 0.0, 10.0, 10.0)}, {'name': 'intersects2', 'polygon': Polygon.from_bounds(-9.0, -1.0, 20.0, 11.0)}, @@ -536,10 +536,10 @@ def test_check_granules_intersecting_bounds(): error_pattern = r'.*Bounds cannot be.*' with raises(validation.BoundsValidationError, match=error_pattern): - validation.check_granules_intersecting_bounds(job_with_default_bounds, valid_granule_metadata) + validation.check_granules_intersecting_bounds(job_with_bad_bounds, valid_granule_metadata) with raises(validation.BoundsValidationError, match=error_pattern): - validation.check_granules_intersecting_bounds(job_with_default_bounds, invalid_granule_metadata) + validation.check_granules_intersecting_bounds(job_with_bad_bounds, invalid_granule_metadata) error_pattern = r".*bounds: \['does_not_intersect1', 'does_not_intersect2', 'does_not_intersect3'\]*"