Merge branch 'develop' into dependabot/pip/moto-dynamodb--5.0.28

.github/workflows/deploy-opera-disp-sandbox.yml

@@ -22,7 +22,6 @@ jobs:
             default_credits_per_user: 0
             default_application_status: APPROVED
             cost_profile: DEFAULT
-            deploy_ref: refs/heads/develop
             job_files: >-
               job_spec/OPERA_DISP_TMS.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.4.0]
+
+### Changed
+- The `OPERA_DISP_TMS` job type is now a fan-out/fan-in workflow.
+
+### Fixed
+- Previously there was a bug in which fan-out job steps, defined using the `map: for item in items` syntax, would fail if `items` was an array of non-string values, because AWS Batch SubmitJob expects string parameters. This bug has been fixed by converting each value to a string before passing it to SubmitJob.
+
 ## [9.3.0]
 
 ### Added

apps/render_cf.py

@@ -126,7 +126,7 @@ def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None =
     batch_params = {}
     for param in step_params:
         if param == map_item:
-            batch_params[f'{map_item}.$'] = '$$.Map.Item.Value'
+            batch_params[f'{map_item}.$'] = "States.Format('{}', $$.Map.Item.Value)"
         else:
             if param not in job_params:
                 raise ValueError(f"job parameter '{param}' has not been defined")

job_spec/OPERA_DISP_TMS.yml

@@ -1,7 +1,7 @@
 OPERA_DISP_TMS:
   required_parameters:
     - measurement_type
-    - frames
+    - frame_ids
     - start_date
     - end_date
   parameters:
@@ -14,7 +14,7 @@ OPERA_DISP_TMS:
           - secant_velocity
           - velocity
         example: displacement
-    frames:
+    frame_ids:
       api_schema:
         description: List of frame IDs to include in mosaic
         type: array
@@ -52,21 +52,38 @@ OPERA_DISP_TMS:
       cost: 1.0
   validators: []
   steps:
-    - name: ''
+    - name: CREATE_MEASUREMENT_GEOTIFF
+      map: for frame_id in frame_ids
       image: ghcr.io/asfhyp3/opera-disp-tms
       command:
+        - ++process
+        - create_measurement_geotiff
+        - Ref::frame_id
         - Ref::measurement_type
-        - Ref::frames
         - Ref::start_date
         - Ref::end_date
         - --bucket
         - '!Ref Bucket'
         - --bucket-prefix
         - Ref::bucket_prefix
-      timeout: 21600 # 6 hr
+      timeout: 1800 # 30 min
       compute_environment: Default
       vcpu: 1
-      memory: 31500
+      memory: 7600
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
+    - name: CREATE_TILE_MAP
+      image: ghcr.io/asfhyp3/opera-disp-tms
+      command:
+        - ++process
+        - create_tile_map
+        - Ref::measurement_type
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::bucket_prefix
+      timeout: 21600 # 6 hr
+      compute_environment: Default
+      vcpu: 1
+      memory: 31500

requirements-all.txt

@@ -5,7 +5,7 @@
 -r requirements-apps-start-execution-worker.txt
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
-boto3==1.36.2
+boto3==1.36.11
 jinja2==3.1.5
 moto[dynamodb]==5.0.28
 pytest==8.3.4
@@ -15,4 +15,4 @@ ruff
 mypy
 setuptools==75.8.0
 openapi-spec-validator==0.7.1
-cfn-lint==1.22.5
+cfn-lint==1.23.1

requirements-apps-disable-private-dns.txt

@@ -1 +1 @@
-boto3==1.36.2
+boto3==1.36.11

requirements-apps-start-execution-manager.txt

@@ -1,3 +1,3 @@
-boto3==1.36.2
+boto3==1.36.11
 ./lib/dynamo/
 ./lib/lambda_logging/

requirements-apps-start-execution-worker.txt

@@ -1,2 +1,2 @@
-boto3==1.36.2
+boto3==1.36.11
 ./lib/lambda_logging/

tests/test_render_cf.py

@@ -35,7 +35,7 @@ def test_get_batch_job_parameters():
     assert render_cf.get_batch_job_parameters(job_spec, step, map_item='param5') == {
         'param2.$': '$.batch_job_parameters.param2',
         'param3.$': '$.batch_job_parameters.param3',
-        'param5.$': '$$.Map.Item.Value',
+        'param5.$': "States.Format('{}', $$.Map.Item.Value)",
     }
 
     step = {'command': ['foo', 'Ref::param2', 'Ref::param3', 'bar', 'Ref::param5']}