diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md b/lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md new file mode 100644 index 000000000..15fdc54d4 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/Readme.md @@ -0,0 +1,170 @@ +# ctTSO v2 pipeline manager + + +* [ctTSO v2 pipeline manager](#cttso-v2-pipeline-manager) + * [Inputs](#inputs) + * [Example Input](#example-input) + * [Outputs](#outputs) + * [Lambdas in this directory](#lambdas-in-this-directory) + * [Get Cache and Output paths](#get-cache-and-output-paths) + * [Generate a trimmed samplesheet](#generate-a-trimmed-samplesheet) + * [Upload the samplesheet to the cache path](#upload-the-samplesheet-to-the-cache-path) + * [Generate a copy manifest](#generate-a-copy-manifest) + * [Launch the nextflow pipeline](#launch-the-nextflow-pipeline) + * [SSM Parameters](#ssm-parameters-) + * [External Parameters required by CDK](#external-parameters-required-by-cdk) + + +This AWS step functions performs the following actions + +* Copies a set of fastq files to a structured location (via the icav2 copy batch utility) +* Uploads a samplesheet to the same structured location +* Invokes the ctTSO v2 pipeline calling the structured location as the run folder + +![](./images/step_functions_image.png) + +## Inputs + +The AWS Step functions takes in the following parameters: + +* project_id +* sample_id (should match the sample_id in the samplesheet, ideally the lab metadata library id) +* portal_run_id (unique identifier for this step functions run and determines the data output location) +* user_reference (an identifier for the run, can also be the same as the portal run id) +* user_tags: (as a dict) + * subject_id + * library_id + * instrument_run_id + * project_owner + * project_name +* technical_tags: + * These are generated by the AWS Step functions and comprise the step_functions_execution_arn and the portal_run_id + * So should be left blank for now +* samplesheet_b64gz: (a base64 encoded and gzipped samplesheet as a dictionary, this can be found as an output of the bssh_icav2_fastq_copy_manager step functions) +* fastq_list_rows_b64gz: (a base64 encoded and gzipped list of fastq rows as a dictionary, this can be found as an output of the bssh_icav2_fastq_copy_manager step functions) + * This will use only the fastq files whose RGSM attribute match the sample id + +### Example Input + +```json +{ + "project_id": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", + "sample_id": "L2301368", + "portal_run_id": "20240308abcd1234", + "user_reference": "PTC-ctTSO-v2-launch-test", + "user_tags": { + "subject_id": "SBJ04405", + "library_id": "L2301368", + "instrument_run_id": "231116_A01052_0172_BHVLM5DSX7", + "project_owner": "UMCCR", + "project_name": "testing" + }, + "samplesheet_b64gz": "H4sIAAAAAAAAA91b227bOBB9z1cU3tek0M2WlD5pVUAo0M0uGvahWCwGsq20Qm3ZlZV0i0X/fTmUGcvlULQVxmiNIAEcUuKZIefqw/8uXrwYfSryeVGPrl/8xz/xz3flooC7Vb3MG3go6k25qvigd9mO1vcVVPmy4P8asc2X6uqmmL67STzfdV3/6o+3RXHL3oE7uVk9eP5o+1BZbZr6fllUDTTf1uJZPp7fFl9GfMJ3nDWqOYzNDgV+BBdm32aLAv/tjt3LzohHjZTVvPh37yFnb6T7kPO48HS2mK0qLmkDm6JpyupjB8ayrMrl/RKaulwuizmIxRdF9bH5xCf548v9afk8XzdFDSv+ukW+xilyRr75DJtPK76KlPTx6c3qrvma10VH3aPgpfcyHFEg53mT8xl/i2dbmHzCIq9Qr1tV4Evz5ZpvZDnHt731fMf1J9HocVhoBIeyJGVZkrEk+2HQw9GUjyYsTZJ0N4rC1eW82Glz9D68cT+4gf/qjeuIX/mP3VNSNe3GinezjKX8hy+Q8HX4Mkwz3dNNF7O/Xw5QRUyogmshTbg+soRSBS6fcRAZOy9VhA6hijTlJ4KvwlJKFXzJjIkJZ6YKlzQQPBbcPkgDSTJ+KHDhMzOQ0KNOBWMoLF+B9BV8mB+bLDk3A/HJU4E7n+3te/dU8PPCJ6S/noF4faoIJvzNPAUgnWfrPemzgUGEsf0ocxYKCbUKScUyfBGm8RtMxNbkzBQS6U8Idw+oD02mwZfm54db1JkpJNYqRKQamc6dcl2gstgvmG/0KmTs6E+IeDdGVE3WgWlH9guG2l6FhAGhCmEqbD+CdNwpxlmccG6qGJOOlOehCQZb0m/gwgjhzMwkolSBOQUaCaWITCxoMI4PvFp+9Sa68do/+PEJEEPShtGj0SUUWi/miFYh+r0QJ9SBQjeS0JaVibTOEJXtQiSrcoY/jEwdMjwBpsTBLkSqWmYi/81IiPwYYmg74UbHZBUrAiidoqfoVLgqTwiRri6Z6LBockSRQZ4QIlX1YaqKnQ/6LHJ7Zpldp9MPkarG+ElMsI+li5Vo8CeESLluEcw1XTbWJjZ2Nzrog+hRZzFNhe8jNxqh7x+CZ4dInUURW7lz0Wx0sm9Kzw6R7AxgR5XR3SJRE2aWA2A/RCq5xOPG80tNpd5WrSeESJlLggA1Fs1DNxaLpzyLVBqRiY6GJhlTuqfPDpFKxrBi0jkdtGZjdWkXIpXpYEHHdC13UQ1aTmn7IWq+F8DWlia64D5bznR6IfpUppOhPWS0RaM971v7s0OkoosojRJdGpGI4HNCiHR/GzMdXTKWMWO7zi5EKrq0CS1t0VjV7Pcbnx0iFV3asoC26ES0+Czni/0Q6eiSiZyLNJdE9O5PqMWAjtE8AmYaLWJkNPXSrUL8OSrAXog/R3nVD5Hs9CXtVyearJtr0dAS10Pkf//BJ0ezxep+TtAbPhZVUedNMe9SD5yX/Ge74vbJr6v6891i9RWHy1n++BmkzrsEhXW5Lhal0Mbovq6uy8WS/5nl13LgenoX+9Px7O5qNvXCq2B+l19FwaS4ymfzSTF1Ijef5b/9Pluk2zc+BOBBhw3RgqKJEIZiZ1FO67z+9shhkTNAFj+gVjrymXVdrOFz2XQJMLfsHX0eDBWNBocHssIBtZyxhsM34vBBljGg1izWcARGHAHIWgXUwsQajrERxxhkQQJq9WENx8SIYwKy6gC1xLCGIzTiCEGWFqDWEdZwREYcEcj6AdRiwRqO2IgjBlkkgFoR2MLRzfxpHL4DshIANe23hsPoT30XZLoPam5vDYfRn/oeyJwe1ATeGg6jP/V9kIk7qFm6NRxGf+oHILNzUFNxaziM/tRHf9qm4KDm27ZwBEZ9BBhf2jwb1KTaGg6FNaNBs50HOy4NUMQZLa5Zw27/fPAOB6awVzTAtvNgx2kBisBiD5jCItEAi6TGHrklQBFJ7AFT2BwaYNt5sON4AEXosAZMZVXQwOQ82HEtgCJWWAM2MQbxSQQ7EjJQjGN7YIyRfCIi+ZYGDBTn1xqY0BjOQwd2RFygWLf2wBhjeujCjgoLFO/VHhhjYA8xsEsyKlDMU3tgjNE99GFHBwWK+2kPjDGkhQHsuENAEYXsgTHG+XAMO/YOUFQda2AiI5hoDJI/AypZRgukveByOA5jERdNQDJQQKWbWMNhLOKiECRZB1RmjjUcRv8fRSDpLqByW6zhMLr+KAbZ0QS1fdmXDN7k1epgILHR7ccOSOYKqDQVe0CMLj9Gl982UEHtltoDYnT3sQeShQIq5cQeEKOrj32QXBNQiSX2gBjdfByAbByD2iW2B8ToVeMxSGoLqDyWA4FcyEZ1s1mNHWdBtKqPYWceyM38YfK0+JQ/lKt7vAs5wgt/csKhlwB7rvgdxUi9kH1uqY2DO91qpbmdJK9dvr7pnpIfOZ5PuMxRhtBesmzhvH/9l+M43e9Yx+T4wXWXUqkOE+z4SxmkYL5BMP8Jle7AHTv6cgUpWGAQLHhCpTxMsOMvSZCCjQ2CjZ9QaQ/csaMvO5CCTQyCTQZV6geI5JJ7deRdYlKk2CBSPKjeHybS8XeCKZFcp18kPn6oSN2uwTCRjr/bS4rkGkTSfE9u6D0MPXjH3tElRTLEK3dYB2PgLh1915YUyRCp3MMjVbcPMnSXjr0zS4pkiFHu4TGq200Z5sSPv6ZFimSITu7h0anbkxmaIh173YoUyRCX3Ekn9b/4fvE/hhiak4tEAAA=", + "fastq_list_rows_b64gz": "H4sIALln6WUC/+Wd32/cNgzH/5Ugz7XPkvyzb66LaQXSl9odBgyD4fO5Q7Dk0l3TDt2w/32kstOtKatLlL7wiD6UTg64fsl+LIqm6F/+Pn9jX708f352bttusK0dWpt2YLZD17Zdqs6fncFH+tf4kQttMmXK+u5nFy/wZ2+3v29v/txeXK530+4z/uZi2i7wG4UfWqaN+uHyann75lW/m/Hzl/P0ST9frdbavFsXqkw2daGTvKmLZNrMc1IbNSu91PNG69Xl1fU2mbbT1ecPy4eVNkqpcmwzlRV6zFSlxxc//nTxunjZ/1yNG52XhRrzuck2c/Jivuputp+W3e3Zp3zUY5WsK9XM9aZJiqXcwBcuZTLV6yWZJ1Wti6XZ5Nl6dfPx9v3H21U/Xb+/gm9EKaNa7XV7Y+zVeJFlanyjRvgrfTd9uP0j/e2v8/8065PWrEnN+zj/X3BVNMVSv9OJ0RuT5HMF3290nszlVBebaanhX+EEj+93l9fw/2elM23ColemLFVZNPjRPDNZNa3nTVU32XeLGgcF92Lwz7OzA8kAcdcCzrZNBzAtcG0HkuRGKMmNN8ZeCyGZ0syL5MdHjYOCEMldB+sxMDx0aQtID+6KIrnKZJJcZd4YeyODZFIzK5IjosZBQYhk6xZlSK5t2lpYkiHFprPrSgkl+WCMfS6EZEozL5IfHzUOCoJr8jAgvsPQph3YsELblsyuKy2UZO2NsS+EkExp5kXy46PGQUF4Tcal2OJC7OpecNXR2bURSrLxxtiXQkimNPMi+fFR46AgXPG6K3nBmoyF62FwZWz9Ncl5Oe6W3cftg3jWp8Lz/sa4V3/vcuwr9LE+WbYfrJ8F598pmtzUBHNywN9trwfcXQ/u8VVL8l+J5r/60sfV3se1EP6P6efF/9OiyU1NcP2HfTjij8+uIZ2HVACyepL/WjT/9Zc+rvc+boTwf0w/L/6fFk1uasI7+c5i+g+LPqCPNwLYAZD8N6L5b770cbP3scqE3ACOOoDXHeCJ8WQnJ5gDtJABdPhsDZ+V48NyuCtQ94Aik3wP2Ku/dwlOVjLuAccdwOoe8NR4spMTuge4XcDgyvjuwRxekfeAKpdJf5V7A1yqZRBPi2ZFeUzcWEgIV/UsZPT4hC51BT0s8pFZfVUIpbnwBrjUCKGZFM2L5oi4sZBwpAMOT5S0Ka7KWKOnSK6FklwX3gB35jJIpkWzIjkmbiwkhHfaWGqzQDIeLIFEmyS5Ekpy5Q1wZyGEZFI0L5Ij4sZCQjDDxjIZZNipdZ1wQ5saguTyQSSbUyF533xUl94Ad7o2JHOyJIdFsyD5KXFjISGYXQ/4B7bHFnPs9hsk10JJrr0B7qyEkEyK5kVyRNxYSAjWsF1Puh1S2CLjU21LktwIJbnxBrizFkIyKZoXyRFxYyEhnF1jI0rbpR3WsWFhpkhuMpkkN5k3wJ2NDJJp0axIjokbCwnh2vXgJiJhd7lrNCdJVkJJVt4Ye50JIZkUzYvkiLixkBA+Kda5iUgplrA7yLVJkrVQkrU3wJ1KCMmkaF4kR8SNhYQj3SEtjix0nV5YxyZJNkJJPhjgTi2EZFI0L5Ij4sZCQrDihR2beGB7uGvbprPrQijJhTfAnUYIyaRoXiRHxI2FhCPzCl3TJvZrWpdm51+TrB+2T85PheR936tW3gB3ukab/GRJDotmQfJT4sZCQnhNxuJ112F23brSF0WyFkqy9ga4sxBCMimaF8kRcWMhIVi7xsH+kFWnbkgKrMokyUYoycYb4M5SCMmkaF4kR8SNhYQjfdd4wBFnnN1NOyJJzoWSfDDAnZUQkknRvEiOiBsLCeF5RTimaLBp2+H40W/skwuhJBfeAHfWQkgmRfMiOSJuLCQEs2s3bBRPNe5f1EGRXAolufQGuLMRQjIpmhfJEXFjISHcrelGh+I+uXMskyRXQkmuvDH2JhNCMimaF8kRcWMhIXyCAvs17d0scDfenyK5Fkpy7Q1wpxJCMimaF8kRcWMh4cibLHGuP/Z4YXJt6dp1I5TkxhvgTi2EZFI0L5Ij4sZCwpEJvfh66TbFyrUrYhMkm0wmySbzBrjTyCCZFs2K5Ji4sZAQnrHppnfdTRrovvE82Qjt8TLKG+BOIT1etGheJEfEjYWE8Jss8SyUO9VoXb8XSbLQHi+jvQHuFNLjRYvmRXJE3FhICK7Jbs5Aa3EiX+teg0WRLLTHyxhvgDuF9HjRonmRHBE3FhKOnGp0b5TGd1hY9zILimShPV7mYIA7hfR40aJ5kRwRNxYSwj1e1p1nTLFT07b082QjtMfLFN4Adwrp8aJF8yI5Im4sJIT7rocOpw2kbt4AtogQJOdC1+T8YIA7hfR40aJZkRwTNxYSHjdbM4+erXlyJBMDDk+fZFo0K5Jj4sZCwuMm8uXRE/lOjmRiLNrpk0yLZkVyTNxYSDhSu7buBAU+Vbb4giiKZKHZdXMwxj6XQjIpmhfJEXFjIeE+yb/+C4TOMeRCoAAA" +} +``` + +## Outputs + +The AWS Step functions returns the following parameters: + +The analysis ID of the nextflow pipeline run launched by the step functions lambda + +The CreateNextflowAnalysis submission as a JSON object for reproducibility + +```json +{ + "analysis_id": "3e17b6ff-ff63-43ab-ac15-e939e88932b6", + "analysis_launch_payload": { + "userReference": "PTC-ctTSO-v2-launch-test", + "pipelineId": "fdef5902-3f50-4ee7-ae17-15d38d4b489c", + "tags": { + "technicalTags": [ + "portal_run_id=20240308abcd1234", + "step_functions_execution_arn=93f7c69b-66eb-4890-8eb4-298ec3b36d37" + ], + "userTags": [ + "subject_id=SBJ04405", + "library_id=L2301368", + "instrument_run_id=231116_A01052_0172_BHVLM5DSX7", + "project_owner=UMCCR", + "project_name=testing" + ], + "referenceTags": [] + }, + "analysisInput": { + "inputs": [ + { + "parameterCode": "run_folder", + "dataIds": [ + "fol.58422302edd141213e0f08dc3cace45e" + ] + }, + { + "parameterCode": "sample_sheet", + "dataIds": [ + "fil.94813f45b9e94977b0a308dc388cf24f" + ] + } + ], + "parameters": [ + { + "code": "StartsFromFastq", + "value": "true" + }, + { + "code": "sample_pair_ids", + "multiValue": [ + "L2301368" + ] + } + ] + }, + "activationCodeDetailId": "7f03a57e-2cfc-4b35-9cbb-d19e6ce9984b", + "analysisStorageId": "3fab13dd-46e7-4b54-bb34-b80a01a99379", + "outputParentFolderId": null, + "analysisOutput": [ + { + "sourcePath": "out/", + "targetProjectId": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", + "targetPath": "/ilmn_cttso_fastq_cache/20240308abcd1234/", + "type": "FOLDER" + } + ] + } +} +``` + +## Lambdas in this directory + +### Get Cache and Output paths + +Combine the root cache and root output ssm parameters with the portal run id submitted in the inputs + +### Generate a trimmed samplesheet + +Generate a samplesheet that only has the TSO500L sections and only for the sample_id submitted in the inputs + +### Upload the samplesheet to the cache path + +Upload the samplesheet dict as a samplesheet csv to the cache path + +### Generate a copy manifest + +Generate a copy manifest for submissions to the icav2 copy batch utility + +### Launch the nextflow pipeline + +Take the fastq output locations, samplesheet location and metadata generated in the inputs and run the launch nextflow object. + +## SSM Parameters + +### External Parameters required by CDK + +``` +/icav2/umccr-prod/service-user-trial-jwt-token-secret-arn +/icav2_copy_batch_utility/state_machine_arn +/icav2/umccr-prod/cache_project_cttso_fastq_path +/icav2/umccr-prod/output_project_cttso_fastq_path +/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id +``` \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore new file mode 100644 index 000000000..f60797b6a --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.gitignore @@ -0,0 +1,8 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore new file mode 100644 index 000000000..c1d6d45dc --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md new file mode 100644 index 000000000..9315fe5b9 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/README.md @@ -0,0 +1,14 @@ +# Welcome to your CDK TypeScript project + +This is a blank project for CDK development with TypeScript. + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +## Useful commands + +* `npm run build` compile typescript to js +* `npm run watch` watch for changes and compile +* `npm run test` perform the jest unit tests +* `npx cdk deploy` deploy this stack to your default AWS account/region +* `npx cdk diff` compare deployed stack with current state +* `npx cdk synth` emits the synthesized CloudFormation template diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts new file mode 100644 index 000000000..dbcb0990e --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/bin/deploy.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env node +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { ctTSOV2LaunchStateMachineStack } from '../lib/stacks/cttso_v2_launch_stack'; +import { + ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_ARN_SSM_PARAMETER_PATH, + ICAV2_JWT_SECRET_ARN_SSM_PARAMETER_PATH, + SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS, +} from '../constants'; + +const app = new cdk.App(); +new ctTSOV2LaunchStateMachineStack(app, 'ctTSOv2LaunchStatemachineStack', { + icav2_jwt_ssm_parameter_path: ICAV2_JWT_SECRET_ARN_SSM_PARAMETER_PATH, + ssm_parameter_list: SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS, + icav2_copy_batch_utility_state_machine_ssm_parameter_path: ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_ARN_SSM_PARAMETER_PATH, + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION + }, +}); \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json new file mode 100644 index 000000000..d8d9da252 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/cdk.json @@ -0,0 +1,65 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/deploy.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-iam:standardizedServicePrincipals": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true + } +} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts new file mode 100644 index 000000000..4dee4b17b --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/constants.ts @@ -0,0 +1,17 @@ + +export const DYNAMODB_ARN_SSM_PARAMETER_PATH = "/umccr/orcabus/stateful/cttso_v2_pipeline_dynamo_db/analysis_table_arn" + +export const CTTSO_CACHE_ROOT_SSM_PARAMETER_PATH = "/icav2/umccr-prod/cache_project_cttso_fastq_path" +export const CTTSO_OUTPUT_ROOT_SSM_PARAMETER_PATH = "/icav2/umccr-prod/output_project_cttso_fastq_path" + +export const ICAV2_JWT_SECRET_ARN_SSM_PARAMETER_PATH = "/icav2/umccr-prod/service-user-trial-jwt-token-secret-arn" +export const ICAV2_COPY_BATCH_UTILITY_STATE_MACHINE_ARN_SSM_PARAMETER_PATH = "/icav2_copy_batch_utility/state_machine_arn" +export const PIPELINE_ID_SSM_PARAMETER_PATH = "/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id" + + +export const SSM_PARAMETER_LIST_FOR_CTTSO_LAUNCH_LAMBDAS = [ + CTTSO_CACHE_ROOT_SSM_PARAMETER_PATH, + CTTSO_OUTPUT_ROOT_SSM_PARAMETER_PATH, + ICAV2_JWT_SECRET_ARN_SSM_PARAMETER_PATH, + PIPELINE_ID_SSM_PARAMETER_PATH +] \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js new file mode 100644 index 000000000..08263b895 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts new file mode 100644 index 000000000..05e25bf35 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/cttso_v2_launch_step_function.ts @@ -0,0 +1,316 @@ +import * as cdk from 'aws-cdk-lib'; +import { Duration } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; + +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import { LambdaLayerConstruct } from './lambda_layer'; + +interface ctTSOv2LaunchStepFunctionConstructProps { + icav2_jwt_ssm_parameter_path: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + lambdas_layer_path: string; // __dirname + '/../../../layers + get_cttso_cache_and_output_paths_lambda_path: string; // __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths' + generate_trimmed_samplesheet_lambda_path: string; // __dirname + '/../../../lambdas/generate_trimmed_samplesheet_lambda_path' + upload_samplesheet_to_cache_dir_lambda_path: string; // __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths' + generate_copy_manifest_dict_lambda_path: string; // __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths' + launch_cttso_nextflow_pipeline_lambda_path: string; // __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths' + ssm_parameter_list: string[]; // List of parameters the workflow session state machine will need access to + icav2_copy_batch_state_machine_arn: string; + workflow_definition_body_path: string; // __dirname + '/../../../step_functions_templates/cttso_v2_launch_step_function.json' +} + +export class ctTSOv2LaunchStepFunctionStateMachineConstruct extends Construct { + + private icav2_jwt_secret_arn_value: string; + private icav2_jwt_ssm_parameter_path: string; + + public readonly cttso_v2_launch_state_machine_arn: string; + + constructor(scope: Construct, id: string, props: ctTSOv2LaunchStepFunctionConstructProps) { + super(scope, id); + + // Import external ssm parameters + this.set_jwt_secret_arn_object(props.icav2_jwt_ssm_parameter_path); + + // Set lambda layer arn object + const lambda_layer = new LambdaLayerConstruct( + this, 'lambda_layer', { + layer_directory: props.lambdas_layer_path, + }); + + // launch_cttso_nextflow_pipeline lambda + const launch_cttso_nextflow_pipeline_lambda_obj = new PythonFunction( + this, + 'launch_cttso_nextflow_pipeline_lambda_python_function', + { + entry: props.launch_cttso_nextflow_pipeline_lambda_path, + runtime: lambda.Runtime.PYTHON_3_11, + index: 'handler.py', + handler: 'handler', + memorySize: 1024, + // @ts-ignore + layers: [lambda_layer.lambda_layer_version_obj], + // @ts-ignore + timeout: Duration.seconds(60), + } + ); + + // Add icav2 secrets permissions to lambda + this.add_icav2_secrets_permissions_to_lambda( + launch_cttso_nextflow_pipeline_lambda_obj, + ); + + // Add each of the ssm parameters to the lambda role policy + props.ssm_parameter_list.forEach( + (ssm_parameter_path) => { + this.add_get_ssm_parameter_permission_to_lambda_role_policy( + launch_cttso_nextflow_pipeline_lambda_obj, + ssm_parameter_path, + ); + }, + ); + + // generate_copy_manifest_dict lambda + // Doesnt need any ssm parameters + const generate_copy_manifest_dict_lambda_obj = new PythonFunction( + this, + 'generate_copy_manifest_dict_lambda_python_function', + { + entry: props.generate_copy_manifest_dict_lambda_path, + runtime: lambda.Runtime.PYTHON_3_11, + index: 'handler.py', + handler: 'handler', + memorySize: 1024, + // @ts-ignore + layers: [lambda_layer.lambda_layer_version_obj], + // @ts-ignore + timeout: Duration.seconds(60), + } + ); + + // get_cttso_cache_and_output_paths lambda + const get_cttso_cache_and_output_paths_lambda_obj = new PythonFunction( + this, + 'get_cttso_cache_and_output_paths_lambda_python_function', + { + entry: props.get_cttso_cache_and_output_paths_lambda_path, + runtime: lambda.Runtime.PYTHON_3_11, + index: 'handler.py', + handler: 'handler', + memorySize: 1024, + // @ts-ignore + layers: [lambda_layer.lambda_layer_version_obj], + // @ts-ignore + timeout: Duration.seconds(60), + } + ); + + // Add icav2 secrets permissions to lambda + this.add_icav2_secrets_permissions_to_lambda( + get_cttso_cache_and_output_paths_lambda_obj, + ); + + // Add each of the ssm parameters to the lambda role policy + props.ssm_parameter_list.forEach( + (ssm_parameter_path) => { + this.add_get_ssm_parameter_permission_to_lambda_role_policy( + get_cttso_cache_and_output_paths_lambda_obj, + ssm_parameter_path, + ); + }, + ); + + // Generate trimmed samplesheet lambda + // Also doesn't need any ssm parameters or secrets + // generate_trimmed_samplesheet + const generate_trimmed_samplesheet_lambda_obj = new PythonFunction( + this, + 'generate_trimmed_samplesheet_lambda_python_function', + { + entry: props.generate_trimmed_samplesheet_lambda_path, + runtime: lambda.Runtime.PYTHON_3_11, + index: 'handler.py', + handler: 'handler', + memorySize: 1024, + // @ts-ignore + layers: [lambda_layer.lambda_layer_version_obj], + // @ts-ignore + timeout: Duration.seconds(60), + } + ); + + // upload_samplesheet_to_cache_dir lambda + const upload_samplesheet_to_cache_dir_lambda_obj = new PythonFunction( + this, + 'upload_samplesheet_to_cache_dir_lambda_python_function', + { + entry: props.upload_samplesheet_to_cache_dir_lambda_path, + runtime: lambda.Runtime.PYTHON_3_11, + index: 'handler.py', + handler: 'handler', + memorySize: 1024, + // @ts-ignore + layers: [lambda_layer.lambda_layer_version_obj], + // @ts-ignore + timeout: Duration.seconds(60), + } + ); + + // Add icav2 secrets permissions to lambda + this.add_icav2_secrets_permissions_to_lambda( + upload_samplesheet_to_cache_dir_lambda_obj, + ); + + // Add each of the ssm parameters to the lambda role policy + props.ssm_parameter_list.forEach( + (ssm_parameter_path) => { + this.add_get_ssm_parameter_permission_to_lambda_role_policy( + upload_samplesheet_to_cache_dir_lambda_obj, + ssm_parameter_path, + ); + }, + ); + + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine(this, 'cttso_v2_launch_step_functions_state_machine', { + // defintiontemplate + definitionBody: DefinitionBody.fromFile(props.workflow_definition_body_path), + // definitionSubstitutions + definitionSubstitutions: { + '__launch_cttso_nextflow_pipeline__': launch_cttso_nextflow_pipeline_lambda_obj.functionArn, + '__generate_copy_manifest_dict__': generate_copy_manifest_dict_lambda_obj.functionArn, + '__get_cttso_cache_and_output_paths__': get_cttso_cache_and_output_paths_lambda_obj.functionArn, + '__generate_trimmed_samplesheet__': generate_trimmed_samplesheet_lambda_obj.functionArn, + '__upload_samplesheet_to_cache_dir__': upload_samplesheet_to_cache_dir_lambda_obj.functionArn, + '__copy_batch_data_state_machine_arn__': props.icav2_copy_batch_state_machine_arn, + }, + }); + + // Add execution permissions to stateMachine role + stateMachine.addToRolePolicy( + new iam.PolicyStatement( + { + resources: [ + launch_cttso_nextflow_pipeline_lambda_obj.functionArn, + generate_copy_manifest_dict_lambda_obj.functionArn, + get_cttso_cache_and_output_paths_lambda_obj.functionArn, + generate_trimmed_samplesheet_lambda_obj.functionArn, + upload_samplesheet_to_cache_dir_lambda_obj.functionArn, + ], + actions: [ + 'lambda:InvokeFunction', + ], + }, + ), + ); + + + // Because we run a nested state machine, we need to add the permissions to the state machine role + // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr + stateMachine.addToRolePolicy( + new iam.PolicyStatement( + { + resources: [ + `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, + ], + actions: [ + 'events:PutTargets', + 'events:PutRule', + 'events:DescribeRule', + ], + }, + ), + ); + + // Add state machine execution permissions to stateMachine role + stateMachine.addToRolePolicy( + new iam.PolicyStatement( + { + resources: [ + props.icav2_copy_batch_state_machine_arn, + ], + actions: [ + 'states:StartExecution', + ], + }, + ), + ); + + // // Allow statemachine execution check access to describe the copy batch state machine execution + // execution_check_status_lambda_path.addToRolePolicy( + // // @ts-ignore + // new iam.PolicyStatement( + // { + // resources: [ + // props.icav2_copy_batch_state_machine_arn, + // ], + // actions: [ + // 'states:DescribeExecution', + // ], + // }, + // ), + // ); + + // Set outputs + this.cttso_v2_launch_state_machine_arn = stateMachine.stateMachineArn; + + } + + private set_jwt_secret_arn_object(icav2_jwt_ssm_parameter_path: string) { + const icav2_jwt_ssm_parameter = ssm.StringParameter.fromStringParameterName( + this, + 'get_jwt_secret_arn_value', + icav2_jwt_ssm_parameter_path, + ); + + this.icav2_jwt_ssm_parameter_path = icav2_jwt_ssm_parameter.parameterArn; + this.icav2_jwt_secret_arn_value = icav2_jwt_ssm_parameter.stringValue; + } + + private add_icav2_secrets_permissions_to_lambda( + lambda_function: lambda.Function | PythonFunction, + ) { + /* + Add the statement that allows + */ + lambda_function.addToRolePolicy( + // @ts-ignore + new iam.PolicyStatement( + { + resources: [ + this.icav2_jwt_secret_arn_value, + this.icav2_jwt_ssm_parameter_path, + ], + actions: [ + 'secretsmanager:GetSecretValue', + 'ssm:GetParameter', + ], + }, + ), + ); + } + + private add_get_ssm_parameter_permission_to_lambda_role_policy( + lambda_function: lambda.Function | PythonFunction, ssm_parameter_path: string, + ) { + lambda_function.addToRolePolicy( + // @ts-ignore + new iam.PolicyStatement( + { + resources: [ + `arn:aws:ssm:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:parameter${ssm_parameter_path}`, + ], + actions: [ + 'ssm:GetParameter', + ], + }, + ), + ); + } + +} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts new file mode 100644 index 000000000..b84793336 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/constructs/lambda_layer.ts @@ -0,0 +1,44 @@ + +import { Construct } from 'constructs' +import { PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; + +export interface LambdaLayerConstructProps { + layer_directory: string +} + +export class LambdaLayerConstruct extends Construct { + + public readonly lambda_layer_arn: string; + public readonly lambda_layer_version_obj: PythonLayerVersion; + + constructor(scope: Construct, id: string, props: LambdaLayerConstructProps) { + super(scope, id); + + this.lambda_layer_version_obj = new PythonLayerVersion( + this, + 'cttso_v2_tool_layer', + { + entry: props.layer_directory, + compatibleRuntimes: [lambda.Runtime.PYTHON_3_11], + compatibleArchitectures: [lambda.Architecture.X86_64], + license: 'GPL3', + description: 'A layer to enable the cttso_v2 manager tools layer', + bundling: { + commandHooks: { + beforeBundling(inputDir: string, outputDir: string): string[] { + return []; + }, + afterBundling(inputDir: string, outputDir: string): string[] { + return [ + `python -m pip install ${inputDir} -t ${outputDir}`, + ]; + }, + }, + }, + }); + + // Set outputs + this.lambda_layer_arn = this.lambda_layer_version_obj.layerVersionArn; + } +} \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts new file mode 100644 index 000000000..e44512606 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/lib/stacks/cttso_v2_launch_stack.ts @@ -0,0 +1,69 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import { ctTSOv2LaunchStepFunctionStateMachineConstruct } from '../constructs/cttso_v2_launch_step_function'; + +interface ctTSOV2LaunchStateMachineStackProps extends cdk.StackProps { + icav2_jwt_ssm_parameter_path: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + ssm_parameter_list: string[]; // List of parameters the workflow session state machine will need access to + icav2_copy_batch_utility_state_machine_ssm_parameter_path: string; +} + +export class ctTSOV2LaunchStateMachineStack extends cdk.Stack { + + public readonly cttso_v2_launch_state_machine_arn: string + public readonly cttso_v2_launch_state_machine_ssm_parameter_path: string + + constructor(scope: Construct, id: string, props: ctTSOV2LaunchStateMachineStackProps) { + super(scope, id, props); + + // Get the copy batch state machine value from lookup + // This will allow us to copy the fastq files to the correct location for launch + const icav2_copy_batch_stack_state_machine_arn_obj = ssm.StringParameter.fromStringParameterName( + this, + 'icav2_copy_batch_state_machine_ssm_parameter', + props.icav2_copy_batch_utility_state_machine_ssm_parameter_path + ) + + const cttso_v2_launch_state_machine = new ctTSOv2LaunchStepFunctionStateMachineConstruct( + this, + id, + { + icav2_jwt_ssm_parameter_path: props.icav2_jwt_ssm_parameter_path, // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + lambdas_layer_path: __dirname + '/../../../layers/', // __dirname + '/../../../layers + get_cttso_cache_and_output_paths_lambda_path: __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths', // __dirname + '/../../../lambdas/get_cttso_cache_and_output_paths' + generate_trimmed_samplesheet_lambda_path: __dirname + '/../../../lambdas/generate_and_trim_cttso_samplesheet_dict', // __dirname + '/../../../lambdas/generate_and_trim_cttso_samplesheet_dict' + upload_samplesheet_to_cache_dir_lambda_path: __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir', // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir' + generate_copy_manifest_dict_lambda_path: __dirname + '/../../../lambdas/generate_copy_manifest_dict', // __dirname + '/../../../lambdas/generate_copy_manifest_dict' + launch_cttso_nextflow_pipeline_lambda_path: __dirname + '/../../../lambdas/launch_cttso_nextflow_pipeline', // __dirname + '../launch_cttso_nextflow_pipeline' + ssm_parameter_list: props.ssm_parameter_list, // List of parameters the workflow session state machine will need access to + icav2_copy_batch_state_machine_arn: icav2_copy_batch_stack_state_machine_arn_obj.stringValue, // The state machine arn for the copy batch utility + workflow_definition_body_path: __dirname + "/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.json", // __dirname + '/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.json' + } + ); + + // Set outputs + this.cttso_v2_launch_state_machine_arn = cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_arn + this.cttso_v2_launch_state_machine_ssm_parameter_path = this.get_ssm_parameter_obj_for_state_machine( + cttso_v2_launch_state_machine.cttso_v2_launch_state_machine_arn + ).parameterName + + } + + private get_ssm_parameter_obj_for_state_machine( + state_machine_arn: string, + ): ssm.StringParameter { + /* + Generate the ssm parameter for the state machine arn + */ + return new ssm.StringParameter( + this, + 'state_machine_arn_ssm_parameter', + { + parameterName: this.cttso_v2_launch_state_machine_ssm_parameter_path, + stringValue: state_machine_arn, + }, + ); + } + +} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json new file mode 100644 index 000000000..d858db3df --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/package.json @@ -0,0 +1,27 @@ +{ + "name": "deploy", + "version": "0.1.0", + "bin": { + "deploy": "bin/deploy.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk" + }, + "devDependencies": { + "@types/jest": "^29.5.11", + "@types/node": "20.11.14", + "jest": "^29.7.0", + "ts-jest": "^29.1.2", + "aws-cdk": "2.126.0", + "ts-node": "^10.9.2", + "typescript": "~5.3.3" + }, + "dependencies": { + "aws-cdk-lib": "2.126.0", + "constructs": "^10.0.0", + "source-map-support": "^0.5.21" + } +} \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts new file mode 100644 index 000000000..d3f2e9b24 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/test/deploy.test.ts @@ -0,0 +1,17 @@ +// import * as cdk from 'aws-cdk-lib'; +// import { Template } from 'aws-cdk-lib/assertions'; +// import * as Deploy from '../lib/deploy-stack'; + +// example test. To run these tests, uncomment this file along with the +// example resource in lib/cttso_v2_launch_stack.ts +test('SQS Queue Created', () => { +// const app = new cdk.App(); +// // WHEN +// const stack = new Deploy.DeployStack(app, 'MyTestStack'); +// // THEN +// const template = Template.fromStack(stack); + +// template.hasResourceProperties('AWS::SQS::Queue', { +// VisibilityTimeout: 300 +// }); +}); diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json new file mode 100644 index 000000000..aaa7dc510 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/deploy/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": [ + "es2020", + "dom" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +} diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/images/step_functions_image.png b/lib/workload/stateless/cttso_v2_pipeline_manager/images/step_functions_image.png new file mode 100755 index 000000000..0e1759ee1 Binary files /dev/null and b/lib/workload/stateless/cttso_v2_pipeline_manager/images/step_functions_image.png differ diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py new file mode 100644 index 000000000..618bb797b --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_and_trim_cttso_samplesheet_dict/handler.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +""" +Generate cttso samplesheet csv + +Takes a b64gzip input or regular json input + +{ + "samplesheet_b64gz": "H4sIAAAAAAAA/8tJLS5RsjI2VrJSSdZI", + "sample_id": "L12345678" +} + +Returns the samplesheet b64gz very trimmed and compressed + +{ + "samplesheet_b64gz": "H4sIAAAAAAAA/8tJLS5RsjI2VrJSSdZI" +} + +""" +from copy import deepcopy +from typing import List, Dict + +# Local imports +from cttso_v2_pipeline_manager_tools.utils.compression_helpers import ( + decompress_dict, compress_dict +) + +# Globals +READS_SECTION = { + "read_1_cycles": "151", + "read_2_cycles": "151", + "index_1_cycles": "10", + "index_2_cycles": "10" +} + +TSO500L_SETTINGS = { + "adapter_read_1": "CTGTCTCTTATACACATCT", + "adapter_read_2": "CTGTCTCTTATACACATCT", + "adapter_behaviour": "trim", + "minimum_trimmed_read_length": 35, + "mask_short_reads": 35, + "override_cycles": "U7N1Y143;I10;I10;U7N1Y143" +} + + +def handler(event, context): + """ + Import + Args: + event: + context: + + Returns: + + """ + + # Part 0 - get inputs + samplesheet_b64gz = event.get("samplesheet_b64gz", None) + sample_id = event.get("sample_id", None) + + # Check samplesheet_b64gz is an input + if samplesheet_b64gz is None: + raise ValueError("No samplesheet_b64gz provided") + + # Check sample id + if sample_id is None: + raise ValueError("No sample_id provided") + + # Part 1 - decompress the samplesheet + samplesheet_dict = decompress_dict(samplesheet_b64gz) + + # Part 2 - edit the samplesheet + # We only take the 'header', the reads we will set ourselves, TSO500L_Settings ourselves + # We only take the rows of the TSO500L_Data section where the column Sample_ID matches sample_id + header = samplesheet_dict.get("header") + tso500l_sample_id_rows: List[Dict] = list( + filter( + lambda tso500_data_row_iter: tso500_data_row_iter.get("sample_id") == sample_id, + samplesheet_dict.get("tso500l_data") + ) + ) + + # Check we have a row for this sample_id + if len(tso500l_sample_id_rows) == 0: + raise ValueError(f"No rows found for sample_id {sample_id}") + + # Update i7 and i5 index id for tso500l data section + for index, tso500l_sample_id_row in enumerate(deepcopy(tso500l_sample_id_rows)): + # Check i7 index id + if tso500l_sample_id_row.get("i7_index_id", None) is None: + if tso500l_sample_id_row.get("index_id", None) is None: + ValueError("Please specify either index_id or i7_index_id for all tso500l_data rows") + tso500l_sample_id_rows[index]["i7_index_id"] = tso500l_sample_id_row["index_id"] + # Check i5 index id + if tso500l_sample_id_row.get("i5_index_id", None) is None: + if tso500l_sample_id_row.get("index_id", None) is None: + ValueError("Please specify either index_id or i5_index_id for all tso500l_data rows") + tso500l_sample_id_rows[index]["i5_index_id"] = tso500l_sample_id_row["index_id"] + + # Part 3 - write the samplesheet + samplesheet_dict: Dict = { + "header": header, + "reads": READS_SECTION, + "tso500l_settings": TSO500L_SETTINGS, + "tso500l_data": tso500l_sample_id_rows + } + + # Compress and return + return { + "samplesheet_b64gz": compress_dict(samplesheet_dict) + } + + +# if __name__ == "__main__": +# import json +# samplesheet_b64_gz = """ +# H4sIAAAAAAAAA9VaTW/bOBC978/QXpOFPi3JPWlVQCjQzS4a9lAsgoFsK41QW3ZtJ92iyH9fDiVG +# pMKxVMI9GEGTmpL5Zvjx5pEzP5yHqlxVe2f+w7mv1xXcb/eb8ghP1f5Qbxtn7l85+8cGmnJTOXOH +# Hb421zfV4sNN5gee5wXXf72vqlv2AbzZzfbJD5wrp24Ox/3jpmqOcPy+w2/xJ+Vt9dV55n1xtAOC +# 4X/Ag+X35briDV7ktQ/B19vqZlX9p77oyjblRZf3vFiul9uG232EQ3U81s1ngbOpm3rzuIHjvt5s +# qhUIjHXVfD4+OPMgunp5oVyVu2O1hy3vYl3u+EP+rDx8gcPDlvfZGR5EOtSqPJbO/N8fzrpsuKvc +# 4EO52fFxrFfc8fd+4HrBLHE6m3lTkeWsyAqWFbLR5605b81YnmU5b0UL9vWqevHP+RjfeJ+8MHjz +# znPFP9nA35Z2twPKX84K3lNRZPxvwfvN8oIhJEfIcw6cZ8Mv+cMv8ZcKhiZiA7bwD3zyTjuZKk4K +# GO5pkalOMuyXm8I7u1AnY1dxMuffz/iXWK46meWiJ/7gYp30tOWKONiB5iTvORcWXayTvjqTjKE7 +# jGnLNefN2HF2ucs10GYSeynaOVNmkonu80tYrr7JyXDGO+MhSqOfln/0+USCZaxj3st0NX7lqgDG +# DcoG+5OJiJJdrKvJ61lliMiyQeTkFnE7ivxyXU1fuSpCZzEkJN4rus8uIn4aXY3c17PKwVEWFMMo +# ih0WFxFgjK7GoeKkWLisY9mekLBXfHC5TkYaFXEtlGGI0fYnshAac7GLNom0nVmgStf2Jc4h8q3R +# wU/8FPPmXXLjt7/w4yhgrO0P5AFdSOPOQG1iARgYAWfqNOKgZPpKFUNXENHEBlA7ETH8YUwHRAY4 +# I6B6OmFCBxUaIJ8+pNyzDWmqnRQEbevCK8fNwt08G6Cu2pk4Tw40gVAKZwNUFTQKDjzb6XPIcHva +# bAszYKCTDe77V3yKi/dsgOrGF/Q9OLWzNkjZDGloAvTVOcxzsee0IUUDumE+D6CveYjLNM8HQ5p1 +# C+k8gNqJBO9CmH6ybCnditrMgGoYxunikWJwKmj189kA1UWTIdxglXJqRcl6vjlUybsQJ5xBeOpv +# Kc4DqIYn1GnDbYErlFSpNoBqtEBRyIYXS0JBWgVgM+DgLgsPkwOmwRG1ihZGwECNFgWujkJfpbhG +# u5V7HkCVafAEidtgGIDzM+7DQL9vwWgxDE8FIw+sNoAq07ThV1+lqHK6c/N5AFWmaQWGvkozcUy1 +# iodmQJ1pChGNtEWTibuds3kY6lzKua0YeIhcR92vWAD+StVmBPyVIsoMGOqrtBiSN27MIiMuN4yA +# d1fOcr19XGlZkM9VU+3LY7XqEzuO+wf/ceTb37b7L/fr7Tf+oF6WLx8BR0TJd+zqXbWu0R/ncd/M +# 6/WG/1qWc9k+X9ynwSJa3l8vF358Ha7uy+sknFXX5XI1qxZu4pXL8vc/l+u86/ApBB9iTA21drwk +# VAh5tK4X+3L/XSak5BOQcgkUjSTf3e2rHXypj0oW65Z9EHNDaCIzig9SI4EijOxQAhIlACmMQFFD +# dighiRKCVEOgSCA7lIhEiUBKIFB0jx3KjESZgdQ9oIgdO5SYRIlBih1QFI4dSkKiJCAVDiiyxg4l +# JVFSkLIGFC1jhSK0ixElcEFqGVAEjB0KufcDD6SAAUW12KGQez/wQaoWUKSKHQq594MApFQBRZ/Y +# oZB7PwhB6hNQRIkdCrn3A9z7rSgBRYlYoYSkLyHyWKtEQJEfdih9JsmM1T2HPr8EWlKJQl0e2e3f +# Tz4FG4/Ads+hz/WAluCxhE1GYBPp7UveBbRkiyVsOgLbPYc+BwJa4sMOVsk8GGHlc+jzEaAlIexg +# ZyTRzxLoS0FAq/+whCLZfibYvivIAK0Kww4qJik/dqEviwCtFsISiuT92IO+OAG0igRLKJL8YyR/ +# WSIAWl2AJRQZAeIA+kQ9aNl5SyiSOuMQ+rwUaMkoSygyFsQR9Nkh0FJCdlAJCZVEIHM0oCRmKJi2 +# lo1CISVnMgOZJwElOWKHQkrOJAaZ/gEl52OHQjJRkoBMwYCSd7FDIUkoSUGe4EE5tp8I1DdlsyVg +# UpKAUhdktgWUFIslDEk+KZJPez0Ayp2AJQxJPKkPMrMCSjrFEoYknTQAmU8BJYliCUMSThqCvOIA +# 5V7DEoZkgDQCmawBJUMzCnN35RwP28h119rVyassNIpMPAswIdQFxTiGvPPp1xbVQ/lUbx/3/E0s +# TXWuJlSqGqpRfy6n/ty7eOKeRdHB3cOujvftjZgvmbn+iXKrOoa2aFfAfHz7j+u64kY4MrWP6uRJ +# Zk0ojTKZFRBmBeM6etpojZcxmcwKCbPCcZ09yawJJUcmsyLCrGhch08brfHyIJNZM8Ks2SmdThvk +# aeM0Vr9tMiglDEpPqflJBk2otTYY5Llmg3j7Cc0/yaAJddEmgzzCIO/UyWDilI3WMJsMIvjJO3l+ +# mDZC4/XGJoMIZvIoZhKnjIkjNFobbDKI4CSP4iRxFpm07ScUzJkMItjIo9hInFgmBpPR4jaTQQQP +# 8fbnu+ff/gcSZgO3fTMAAA== +# """.replace("\n", "") +# +# print( +# json.dumps( +# handler( +# event={ +# "samplesheet_b64gz": samplesheet_b64_gz, +# "sample_id": "L2301346_rerun" +# }, +# context=None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "samplesheet_b64gz": "H4sIAIf15mUC/41R0WrCMBT9FcnzhCa1CttTUSgDV4bWhzFGiPZqw5pUk1Qm4r/vJtVtZXsYoZCcc+49veeeSQWiBEPuB2eylTXwbWOUcPwIxspGI87uBsS0mmuhAJ+ksAc9zGG9yFMWU0rj4dMcYFksOB3nzZHFBAukts60CrTj7rQPdciJJRzIxfdDUxs8/Y1TvjltavAIoQklVwVnv3GpS/joF0TfOOvj3srZJomimltwTupd5ypKsXdgeOfu1dMiK6Z4irRIp3jw6tv2hOwfwjVU4iib1idKnJHKk0pqqVrF/VtB2XWrQe9chbI48RJh37mtGuP4LZyAN7gII0v4MdhqktMXOoofHmkUvhvQm7cUTqD69UysUHtcrCx98ZzFEY1HY3TBpfqfu9K3Nc3y1KO10HBdfojWU2mGo6cZDv6VeIik8CFgLGkW8AnvltEZrmbPURSxwCR/Mpe3yyfpAP0IhwIAAA==" +# # } diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py new file mode 100644 index 000000000..204ac3633 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/generate_copy_manifest_dict/handler.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +""" +Given a cache path, sample id and list of fastq list rows, +generate a manifest file to copy fastq files to the cache directory + +This will be launched in a downstream step + +Given + +{ + "cache_path": "/path/to/cache", + "project_id": "project_id", + "sample_id": "sample_id", + "fastq_list_rows": [ { "RGID": "...", "RGLB": "...", "RGSM": "sample_id", "Read1FileURI": "icav2://", "...", "Read1FileURISrc": "icav2://"} ] + /* Except fastq list rows is b64gz encoded */ +} + +Return a manifest list + +{ + "manifest_list": [ + { + "fastq_list_rows[0].Read1FileURISrc": [ # BSSH trial + "icav2://{project_id}{cache_path}", # dest_uri + ], + "fastq_list_rows[0].Read2FileURISrc": [ # BSSH trial + "icav2://{project_id}{cache_path}", # dest_uri + ], + ... + } + ] +} +""" +from functools import reduce +from pathlib import Path +from typing import Dict, List + +from wrapica.enums import DataType +from wrapica.project_data import convert_project_id_and_data_path_to_icav2_uri + +from cttso_v2_pipeline_manager_tools.utils.compression_helpers import decompress_dict + + +def handler(event, context): + """ + + Generate a copy manifest from fastq list rows to cache path + + Args: + event: + context: + + Returns: + + """ + + # Get the cache path + cache_path = event.get("cache_path", None) + + # Get sample id + sample_id = event.get("sample_id", None) + + # Get the project id + project_id = event.get("project_id", None) + + # Get fastq list rows + fastq_list_rows_compressed_str = event.get("fastq_list_rows_b64gz", None) + + # Check cache path + if cache_path is None: + raise ValueError("Cache path is required") + + # Check sample id + if sample_id is None: + raise ValueError("Sample id is required") + + # Check project id + if project_id is None: + raise ValueError("Project id is required") + + # Check fastq list rows + if fastq_list_rows_compressed_str is None: + raise ValueError("Fastq list rows are required") + + # Uncompress fastq list rows + fastq_list_rows: List[Dict] = decompress_dict(fastq_list_rows_compressed_str) + + # Generate the manifest list + fastq_cache_path = convert_project_id_and_data_path_to_icav2_uri( + project_id=project_id, + data_path=Path(cache_path) / sample_id, + data_type=DataType.FOLDER + ) + + # Filter fastq list rows by RGSM (match sample_id) + fastq_list_rows = list( + filter( + lambda fastq_list_row_iter: fastq_list_row_iter.get("RGSM") == sample_id, + fastq_list_rows + ) + ) + + # Generate source uris + source_uris = list( + reduce( + # Flatten the list + lambda row_iter_1, row_iter_2: row_iter_1 + row_iter_2, + map( + lambda fastq_list_row_iter: [ + fastq_list_row_iter.get("Read1FileURISrc"), + fastq_list_row_iter.get("Read2FileURISrc") + ], + fastq_list_rows + ) + ) + ) + + # Generate the manifest list + manifest_list = dict( + map( + lambda source_uri_iter: ( + source_uri_iter, + [ + fastq_cache_path + ] + ), + source_uris + ) + ) + + # Return the manifest list + return { + "manifest_list": manifest_list + } + + +# if __name__ == "__main__": +# +# import json +# +# fastq_list_rows_b64gz = """ +# H4sIALln6WUC/+Wd32/cNgzH/5Ugz7XPkvyzb66LaQXSl9odBgyD4fO5Q7Dk0l3TDt2w/32kstOtKatLlL7wiD6UTg64fsl+LIqm6F/+Pn9jX708f352bt +# tusK0dWpt2YLZD17Zdqs6fncFH+tf4kQttMmXK+u5nFy/wZ2+3v29v/txeXK530+4z/uZi2i7wG4UfWqaN+uHyann75lW/m/Hzl/P0ST9frdbavFsXqkw2 +# daGTvKmLZNrMc1IbNSu91PNG69Xl1fU2mbbT1ecPy4eVNkqpcmwzlRV6zFSlxxc//nTxunjZ/1yNG52XhRrzuck2c/Jivuputp+W3e3Zp3zUY5WsK9XM9a +# ZJiqXcwBcuZTLV6yWZJ1Wti6XZ5Nl6dfPx9v3H21U/Xb+/gm9EKaNa7XV7Y+zVeJFlanyjRvgrfTd9uP0j/e2v8/8065PWrEnN+zj/X3BVNMVSv9OJ0RuT +# 5HMF3290nszlVBebaanhX+EEj+93l9fw/2elM23ColemLFVZNPjRPDNZNa3nTVU32XeLGgcF92Lwz7OzA8kAcdcCzrZNBzAtcG0HkuRGKMmNN8ZeCyGZ0s +# yL5MdHjYOCEMldB+sxMDx0aQtID+6KIrnKZJJcZd4YeyODZFIzK5IjosZBQYhk6xZlSK5t2lpYkiHFprPrSgkl+WCMfS6EZEozL5IfHzUOCoJr8jAgvsPQ +# ph3YsELblsyuKy2UZO2NsS+EkExp5kXy46PGQUF4Tcal2OJC7OpecNXR2bURSrLxxtiXQkimNPMi+fFR46AgXPG6K3nBmoyF62FwZWz9Ncl5Oe6W3cftg3 +# jWp8Lz/sa4V3/vcuwr9LE+WbYfrJ8F598pmtzUBHNywN9trwfcXQ/u8VVL8l+J5r/60sfV3se1EP6P6efF/9OiyU1NcP2HfTjij8+uIZ2HVACyepL/WjT/ +# 9Zc+rvc+boTwf0w/L/6fFk1uasI7+c5i+g+LPqCPNwLYAZD8N6L5b770cbP3scqE3ACOOoDXHeCJ8WQnJ5gDtJABdPhsDZ+V48NyuCtQ94Aik3wP2Ku/dw +# lOVjLuAccdwOoe8NR4spMTuge4XcDgyvjuwRxekfeAKpdJf5V7A1yqZRBPi2ZFeUzcWEgIV/UsZPT4hC51BT0s8pFZfVUIpbnwBrjUCKGZFM2L5oi4sZBw +# pAMOT5S0Ka7KWKOnSK6FklwX3gB35jJIpkWzIjkmbiwkhHfaWGqzQDIeLIFEmyS5Ekpy5Q1wZyGEZFI0L5Ij4sZCQjDDxjIZZNipdZ1wQ5saguTyQSSbUy +# F533xUl94Ad7o2JHOyJIdFsyD5KXFjISGYXQ/4B7bHFnPs9hsk10JJrr0B7qyEkEyK5kVyRNxYSAjWsF1Puh1S2CLjU21LktwIJbnxBrizFkIyKZoXyRFx +# YyEhnF1jI0rbpR3WsWFhpkhuMpkkN5k3wJ2NDJJp0axIjokbCwnh2vXgJiJhd7lrNCdJVkJJVt4Ye50JIZkUzYvkiLixkBA+Kda5iUgplrA7yLVJkrVQkr +# U3wJ1KCMmkaF4kR8SNhYQj3SEtjix0nV5YxyZJNkJJPhjgTi2EZFI0L5Ij4sZCQrDihR2beGB7uGvbprPrQijJhTfAnUYIyaRoXiRHxI2FhCPzCl3TJvZr +# Wpdm51+TrB+2T85PheR936tW3gB3ukab/GRJDotmQfJT4sZCQnhNxuJ112F23brSF0WyFkqy9ga4sxBCMimaF8kRcWMhIVi7xsH+kFWnbkgKrMokyUYoyc +# Yb4M5SCMmkaF4kR8SNhYQjfdd4wBFnnN1NOyJJzoWSfDDAnZUQkknRvEiOiBsLCeF5RTimaLBp2+H40W/skwuhJBfeAHfWQkgmRfMiOSJuLCQEs2s3bBRP +# Ne5f1EGRXAolufQGuLMRQjIpmhfJEXFjISHcrelGh+I+uXMskyRXQkmuvDH2JhNCMimaF8kRcWMhIXyCAvs17d0scDfenyK5Fkpy7Q1wpxJCMimaF8kRcW +# Mh4cibLHGuP/Z4YXJt6dp1I5TkxhvgTi2EZFI0L5Ij4sZCwpEJvfh66TbFyrUrYhMkm0wmySbzBrjTyCCZFs2K5Ji4sZAQnrHppnfdTRrovvE82Qjt8TLK +# G+BOIT1etGheJEfEjYWE8Jss8SyUO9VoXb8XSbLQHi+jvQHuFNLjRYvmRXJE3FhICK7Jbs5Aa3EiX+teg0WRLLTHyxhvgDuF9HjRonmRHBE3FhKOnGp0b5 +# TGd1hY9zILimShPV7mYIA7hfR40aJ5kRwRNxYSwj1e1p1nTLFT07b082QjtMfLFN4Adwrp8aJF8yI5Im4sJIT7rocOpw2kbt4AtogQJOdC1+T8YIA7hfR4 +# 0aJZkRwTNxYSHjdbM4+erXlyJBMDDk+fZFo0K5Jj4sZCwuMm8uXRE/lOjmRiLNrpk0yLZkVyTNxYSDhSu7buBAU+Vbb4giiKZKHZdXMwxj6XQjIpmhfJEX +# FjIeE+yb/+C4TOMeRCoAAA +# """.replace("\n", "") +# +# # Test the handler +# print( +# json.dumps( +# handler( +# event={ +# "cache_path": "/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache", +# "project_id": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", +# "sample_id": "L2301346_rerun", +# "fastq_list_rows_b64gz": fastq_list_rows_b64gz +# }, +# context=None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "manifest_list": { +# # "icav2://b23fb516-d852-4985-adcc-831c12e8cd22/ilmn-analyses/231116_A01052_0172_BHVLM5DSX7_d24651_4c90dc-BclConvert v4_2_7-b719c8d9-5e6d-49e6-a8be-ca17b5e9d40b/output/Samples/Lane_2/L2301346_rerun/L2301346_rerun_S7_L002_R1_001.fastq.gz": [ +# # "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache/L2301346_rerun/" +# # ], +# # "icav2://b23fb516-d852-4985-adcc-831c12e8cd22/ilmn-analyses/231116_A01052_0172_BHVLM5DSX7_d24651_4c90dc-BclConvert v4_2_7-b719c8d9-5e6d-49e6-a8be-ca17b5e9d40b/output/Samples/Lane_2/L2301346_rerun/L2301346_rerun_S7_L002_R2_001.fastq.gz": [ +# # "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache/L2301346_rerun/" +# # ] +# # } +# # } \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/get_cttso_cache_and_output_paths/handler.py b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/get_cttso_cache_and_output_paths/handler.py new file mode 100644 index 000000000..f5fb7c99e --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/get_cttso_cache_and_output_paths/handler.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +""" +Get cache and output paths for cttsov2 workflow + +Given inputs +{ + "project_id": "project_id", # Required - the project id + "portal_run_id": "20241231abcd1234", # Optional - generated if not provided + "sample_id": "L12345678", # Required - the library id +} + +Returns outputs +{ + "cache_path": "//cache/20241231abcd1234/L12345678_run_cache/, + "output_path": "/mnt/scratch/output/20241231abcd1234", + "cache_uri": "icav2://path/to/cache/20241231abcd1234/L12345678_run_cache/" + "output_uri": "icav2://path/to/output/20241231abcd1234/" +} + +""" +from pathlib import Path + +from wrapica.enums import DataType +from wrapica.project_data import convert_project_id_and_data_path_to_icav2_uri + +# Local imports +from cttso_v2_pipeline_manager_tools import generate_portal_run_id +from cttso_v2_pipeline_manager_tools.utils.path_helpers import ( + generate_sample_cache_path, generate_output_path +) + + +def handler(event, context): + """ + Given an event dict with portal_run_id and sample_id, return the cache and output paths for the cttsov2 workflow + Args: + event: + context: + + Returns: + + """ + + # Get the project id + project_id = event.get("project_id", None) + + # Get the portal_run_id and sample_id from the event + if event.get("portal_run_id", None) is not None: + portal_run_id = event.get("portal_run_id") + else: + portal_run_id = generate_portal_run_id() + + # Get the sample id + if event.get("sample_id", None) is not None: + sample_id = event.get("sample_id") + else: + raise ValueError("sample_id is required") + + # Generate the cache path for this sample + cache_path = generate_sample_cache_path(portal_run_id, sample_id) + + # Generate the output path for this sample + output_path = generate_output_path(portal_run_id) + + # Return cache and output paths + return { + "cache_path": str(cache_path), + "output_path": str(output_path), + "cache_uri": convert_project_id_and_data_path_to_icav2_uri( + project_id, + Path(cache_path), + data_type=DataType.FOLDER + ), + "output_uri": convert_project_id_and_data_path_to_icav2_uri( + project_id, + Path(output_path), + data_type=DataType.FOLDER + ) + } + + +# if __name__ == "__main__": +# import json +# +# # Test the handler with portal_run_id +# event = { +# "project_id": "project_id", +# "portal_run_id": "20241231abcd1234", +# "sample_id": "L12345678" +# } +# print(json.dumps(handler(event, None), indent=4)) +# +# # Test the handler without portal_run_id +# event = { +# "project_id": "project_id", +# "sample_id": "L12345678" +# } +# print(json.dumps(handler(event, None), indent=4)) +# +# """ +# { +# "cache_path": "/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache", +# "output_path": "/ilmn_cttso_fastq_cache/20241231abcd1234", +# "cache_uri": "icav2://project_id/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache/", +# "output_uri": "icav2://project_id/ilmn_cttso_fastq_cache/20241231abcd1234/" +# } +# { +# "cache_path": "/ilmn_cttso_fastq_cache/2024030527c5a3d4/L12345678_run_cache", +# "output_path": "/ilmn_cttso_fastq_cache/2024030527c5a3d4", +# "cache_uri": "icav2://project_id/ilmn_cttso_fastq_cache/2024030527c5a3d4/L12345678_run_cache/", +# "output_uri": "icav2://project_id/ilmn_cttso_fastq_cache/2024030527c5a3d4/" +# } +# """ diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/launch_cttso_nextflow_pipeline/handler.py b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/launch_cttso_nextflow_pipeline/handler.py new file mode 100644 index 000000000..b2dd634a2 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/launch_cttso_nextflow_pipeline/handler.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python + +""" +The launch nextflow pipeline expects the following as inputs + +{ + "project_id": "project_id", + "user_reference": "user_reference", + "run_folder_uri": "icav2://project_id/path/to/run/folder/", + "samplesheet_uri": "icav2://project_id/path/to/samplesheet", + "sample_id": "sample_id", + "analysis_output_uri": "icav2://project_id/path/to/out/", + "technical_tags": { + "portal_run_id": "string", + "step_execution_arn": "string", + "step_functions_inputs": "string" + }, + "user_tags": { + "subject_id": "string", + "library_id": "string", + "project_name": "string", + "project_owner": "string", + "instrument_run_id": "string" + }, +} + +The analysis pipeline then returns the following + +{ + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "timeCreated": "2024-02-15T22:28:01.487Z", + "timeModified": "2024-02-15T22:28:01.487Z", + "ownerId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantName": "string", + "reference": "string", + "userReference": "string", + "pipeline": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "timeCreated": "2024-02-15T22:28:01.487Z", + "timeModified": "2024-02-15T22:28:01.487Z", + "ownerId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantName": "string", + "code": "string", + "urn": "string", + "description": "string", + "language": "Nextflow", + "languageVersion": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "name": "string", + "language": "Nextflow" + }, + "pipelineTags": { + "technicalTags": [ + "string" + ] + }, + "analysisStorage": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "timeCreated": "2024-02-15T22:28:01.487Z", + "timeModified": "2024-02-15T22:28:01.487Z", + "ownerId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantName": "string", + "name": "string", + "description": "string" + }, + "proprietary": false + }, + "status": "REQUESTED", + "startDate": "2024-02-15T22:28:01.487Z", + "endDate": "2024-02-15T22:28:01.487Z", + "summary": "string", + "analysisStorage": { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "timeCreated": "2024-02-15T22:28:01.487Z", + "timeModified": "2024-02-15T22:28:01.487Z", + "ownerId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantId": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "tenantName": "string", + "name": "string", + "description": "string" + }, + "analysisPriority": "LOW", + "tags": { + "technicalTags": [ + "portal_run_id=string", + "step_execution_arn=string", + "step_functions_inputs=string" + ], + "userTags": [ + "subject_id=string", + "library_id=string", + "project_name=string", + "project_owner=string", + "instrument_run_id=string" + ], + "referenceTags": [ + "string" + ] + } +} +""" +import json +from pathlib import Path +from tempfile import NamedTemporaryFile + +# Imports +from wrapica.project_pipelines import ( + ICAv2NextflowAnalysisInput, + ICAv2NextflowPipelineAnalysis, + ICAv2PipelineAnalysisTags +) +from wrapica.libica_models import Analysis +from wrapica.utils import recursively_build_open_api_body_from_libica_item + +from cttso_v2_pipeline_manager_tools.utils.aws_ssm_helpers import get_tso500_ctdna_2_1_pipeline_id_from_ssm +from cttso_v2_pipeline_manager_tools.utils.aws_ssm_helpers import set_icav2_env_vars + + +# Globals + + +def handler(event, context): + # Set icav2 environment variables + set_icav2_env_vars() + + # Get inputs + project_id = event.get("project_id", None) + user_reference = event.get("user_reference", None) + run_folder_uri = event.get("run_folder_uri", None) + samplesheet_uri = event.get("samplesheet_uri", None) + sample_id = event.get("sample_id", None) + analysis_output_uri = event.get("analysis_output_uri", None) + + # Get technical tags + technical_tags = event.get("technical_tags", {}) + + # Get user tags + user_tags = event.get("user_tags", {}) + + # Get the pipeline urn from SSM + pipeline_id = get_tso500_ctdna_2_1_pipeline_id_from_ssm() + + # Check inputs + + # Check project id + if not project_id: + raise ValueError("project_id is required") + + # Check user reference + if not user_reference: + raise ValueError("user_reference is required") + + # Check project id + if not project_id: + raise ValueError("project_id is required") + + # Check run folder uri + if not run_folder_uri: + raise ValueError("run_folder_uri is required") + + # Check samplesheet uri + if not samplesheet_uri: + raise ValueError("samplesheet_uri is required") + + # Check sample id + if not sample_id: + raise ValueError("sample_id is required") + + # Check analysis output uri + if not analysis_output_uri: + raise ValueError("analysis_output_uri is required") + + # Get samplesheet uri from data id + icav2_nextflow_analysis_input_obj = ICAv2NextflowAnalysisInput( + project_id=project_id, + pipeline_id=pipeline_id, + input_json={ + "run_folder": run_folder_uri, + "sample_sheet": samplesheet_uri, + "StartsFromFastq": 'true', + "sample_pair_ids": [ + sample_id + ] + } + ) + + # Initialise an ICAv2CWLPipeline Analysis object + nextflow_analysis = ICAv2NextflowPipelineAnalysis( + user_reference=user_reference, + project_id=project_id, + pipeline_id=pipeline_id, + analysis_input=icav2_nextflow_analysis_input_obj.create_analysis_input(), + analysis_output_uri=analysis_output_uri, + tags=ICAv2PipelineAnalysisTags( + technical_tags=technical_tags, + user_tags=user_tags, + reference_tags=[] + ) + ) + + # Generate the inputs and analysis object + # Call the object to launch it + analysis_id: str = nextflow_analysis() + + # Save the analysis + with NamedTemporaryFile(suffix='.json') as temp_file: + nextflow_analysis.save_analysis(Path(temp_file.name)) + + with open(Path(temp_file.name), 'r') as tmp_file_h: + analysis_launch_payload = json.load(tmp_file_h) + + return { + "analysis_id": analysis_id, + "analysis_launch_payload": analysis_launch_payload + } + + +if __name__ == "__main__": + print( + json.dumps( + handler( + event={ + "user_tags": { + "subject_id": "SBJ04405", + "library_id": "L2301368", + "instrument_run_id": "231116_A01052_0172_BHVLM5DSX7", + "project_owner": "UMCCR", + "project_name": "testing" + }, + "technical_tags": { + "portal_run_id": "20240308abcd1234", + "step_functions_execution_arn": "$$.Execution.id" + }, + "user_reference": "PTC-ctTSO-v2-launch-test", + "project_id": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", + "samplesheet_uri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/ilmn_cttso_fastq_cache/20240308abcd1234/L2301368_run_cache/SampleSheet.csv", + "sample_id": "L2301368", + "run_folder_uri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/ilmn_cttso_fastq_cache/20240308abcd1234/L2301368_run_cache/", + "analysis_output_uri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/ilmn_cttso_fastq_cache/20240308abcd1234/" + }, + context=None + ), + indent=2 + ) + ) diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/upload_samplesheet_to_cache_dir/handler.py b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/upload_samplesheet_to_cache_dir/handler.py new file mode 100644 index 000000000..589a98447 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/lambdas/upload_samplesheet_to_cache_dir/handler.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +""" +Upload samplesheet csv to cache path + +Takes in a compressed samplesheet dict, generates the samplesheet as a CSV and then uploads it to the cache path + +{ + "cache_path": "/path/to/cache", + "project_id": "project_id", + "samplesheet_dict_b64gz": "H4sIAAAAAAAA/8tJLS5RsjI2VrJSSU1RyC9KTS7J" +} + +Returns the file id of the uploaded samplesheet + +{ + "samplesheet_file_id": "fil.1234567890" +} + +""" + +# Standard imports +from pathlib import Path +from tempfile import NamedTemporaryFile + +from wrapica.project_data import ( + write_icav2_file_contents, + convert_project_data_obj_to_icav2_uri, + get_project_data_obj_by_id +) + +from cttso_v2_pipeline_manager_tools.utils.compression_helpers import decompress_dict +from cttso_v2_pipeline_manager_tools.utils.aws_ssm_helpers import set_icav2_env_vars + +from v2_samplesheet_maker.functions.v2_samplesheet_writer import v2_samplesheet_writer + + +def handler(event, context): + """ + Upload samplesheet csv to cache path + + Args: + event: + context: + + Returns: + + """ + + # Check inputs are present + cache_path = event.get("cache_path") + project_id = event.get("project_id") + samplesheet_dict_b64gz = event.get("samplesheet_dict_b64gz") + + # Check cache path + if not cache_path: + raise ValueError("cache_path is required") + # Check project id + if not project_id: + raise ValueError("project_id is required") + # CHeck samplesheet dict + if not samplesheet_dict_b64gz: + raise ValueError("samplesheet_dict_b64gz is required") + + # Set icav2 env vars + set_icav2_env_vars() + + # Samplesheet csv str + samplesheet_dict = decompress_dict(samplesheet_dict_b64gz) + + # Write samplesheet to file + with NamedTemporaryFile(suffix='.csv') as samplesheet_tmp_h: + # Write samplesheet to file + v2_samplesheet_writer(samplesheet_dict, Path(samplesheet_tmp_h.name)) + + # Generate the samplesheet as a csv + samplesheet_file_id = write_icav2_file_contents( + project_id=project_id, + data_path=Path(cache_path) / "SampleSheet.csv", + file_stream_or_path=Path(samplesheet_tmp_h.name) + ) + + # Get the uri for the samplesheet file + samplesheet_file_uri = convert_project_data_obj_to_icav2_uri( + get_project_data_obj_by_id(project_id, samplesheet_file_id) + ) + + return { + "samplesheet_file_id": samplesheet_file_id, + "samplesheet_file_uri": samplesheet_file_uri + } + + +# if __name__ == "__main__": +# import json +# +# samplesheet_dict_b64gz = """ +# H4sIACTx5mUC/41R0WrCMBT9FcnzhCa1CttTUSgDV4bWhzFGiPZqw5pUk1Qm4r/vJtVtZXsYoZCc +# c+49veeeSQWiBEPuB2eylTXwbWOUcPwIxspGI87uBsS0mmuhAJ+ksAc9zGG9yFMWU0rj4dMcYFks +# OB3nzZHFBAukts60CrTj7rQPdciJJRzIxfdDUxs8/Y1TvjltavAIoQklVwVnv3GpS/joF0TfOOvj +# 3srZJomimltwTupd5ypKsXdgeOfu1dMiK6Z4irRIp3jw6tv2hOwfwjVU4iib1idKnJHKk0pqqVrF +# /VtB2XWrQe9chbI48RJh37mtGuP4LZyAN7gII0v4MdhqktMXOoofHmkUvhvQm7cUTqD69UysUHtc +# rCx98ZzFEY1HY3TBpfqfu9K3Nc3y1KO10HBdfojWU2mGo6cZDv6VeIik8CFgLGkW8AnvltEZrmbP +# URSxwCR/Mpe3yyfpAP0IhwIAAA== +# """.replace("\n", "") +# +# print( +# json.dumps( +# handler( +# event={ +# "cache_path": "/ilmn_cttso_fastq_cache/20241231abcd1234/L12345678_run_cache", +# "project_id": "7595e8f2-32d3-4c76-a324-c6a85dae87b5", +# "samplesheet_dict_b64gz": samplesheet_dict_b64gz +# }, +# context=None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "samplesheet_file_id": "fil.1d24b366eea949c3a86708dc3c3824cb" +# # "samplesheet_file_uri": "icav2://project/7595e8f2-32d3-4c76-a324-c6a85dae87b5/path/to/samplesheet.csv" +# # } \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/poetry.lock b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/poetry.lock new file mode 100644 index 000000000..7318cb32c --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/poetry.lock @@ -0,0 +1,1020 @@ +# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "aws-requests-auth" +version = "0.4.3" +description = "AWS signature version 4 signing process for the python requests module" +optional = false +python-versions = "*" +files = [ + {file = "aws-requests-auth-0.4.3.tar.gz", hash = "sha256:33593372018b960a31dbbe236f89421678b885c35f0b6a7abfae35bb77e069b2"}, + {file = "aws_requests_auth-0.4.3-py2.py3-none-any.whl", hash = "sha256:646bc37d62140ea1c709d20148f5d43197e6bd2d63909eb36fa4bb2345759977"}, +] + +[package.dependencies] +requests = ">=0.14.0" + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "boto3" +version = "1.34.58" +description = "The AWS SDK for Python" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "boto3-1.34.58-py3-none-any.whl", hash = "sha256:d213a6fea9db6d537b1e65924133d8279ada79a40bc840d4930e1b64be869d4c"}, + {file = "boto3-1.34.58.tar.gz", hash = "sha256:09e3d17c718bc938a76774f31bc557b20733c0f5f9135a3e7782b55f3459cbdd"}, +] + +[package.dependencies] +botocore = ">=1.34.58,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.34.58" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "botocore-1.34.58-py3-none-any.whl", hash = "sha256:8086a6ea27d0f658505ac81e6d7314e013561469c40db9b5a4692127eb5c97cf"}, + {file = "botocore-1.34.58.tar.gz", hash = "sha256:d75216952886dc513ea1b5e2979a6af08feed2f537e3fc102e4a0a2ead563a35"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.19.19)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "docopt" +version = "0.6.2" +description = "Pythonic argument parser, that will make you smile" +optional = false +python-versions = "*" +files = [ + {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "libica" +version = "2.4.0" +description = "Python SDK for Illumina Connected Analytics" +optional = false +python-versions = ">=3.7" +files = [ + {file = "libica-2.4.0-py3-none-any.whl", hash = "sha256:c80927a1f4aa3f60b2c9faf6e8c22f72000bfdf096429d065ebabfce9928120a"}, + {file = "libica-2.4.0.tar.gz", hash = "sha256:436098a5d5c5e843428e9b1592db33bcd2dbc09b47ed83b9ca1d533f65803189"}, +] + +[package.dependencies] +boto3 = "*" +botocore = "*" +certifi = "*" +python-dateutil = "*" +PyYAML = "*" +requests = "*" +six = "*" +urllib3 = "*" + +[package.extras] +dev = ["black", "build", "detect-secrets", "ggshield", "mkdocs", "mkdocs-material", "openapi-spec-validator", "pdoc3", "pipdeptree", "pre-commit", "setuptools", "twine", "wheel"] +test = ["flake8", "mockito", "nose2", "pytest", "pytest-cov", "tox"] + +[[package]] +name = "mypy-boto3-secretsmanager" +version = "1.34.43" +description = "Type annotations for boto3.SecretsManager 1.34.43 service generated with mypy-boto3-builder 7.23.1" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-boto3-secretsmanager-1.34.43.tar.gz", hash = "sha256:abbf560775c2fe0dc383b7f70c16a1bf753d9b3ffc0caa5e35447e685783a68b"}, + {file = "mypy_boto3_secretsmanager-1.34.43-py3-none-any.whl", hash = "sha256:64e9df58f71072f0a912ecaca626683f4536da078caa204ac07928c4b1481b8b"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + +[[package]] +name = "mypy-boto3-ssm" +version = "1.34.47" +description = "Type annotations for boto3.SSM 1.34.47 service generated with mypy-boto3-builder 7.23.1" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-boto3-ssm-1.34.47.tar.gz", hash = "sha256:be70cc32f9a07e6701746ebe65fba14d59c3f24a8511d275fd8322c9365f2270"}, + {file = "mypy_boto3_ssm-1.34.47-py3-none-any.whl", hash = "sha256:6517b1dc01e3ffe48a251c91e2a7fb6801223baf4a8cf1600411f9e132422297"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + +[[package]] +name = "mypy-boto3-stepfunctions" +version = "1.34.0" +description = "Type annotations for boto3.SFN 1.34.0 service generated with mypy-boto3-builder 7.21.0" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mypy-boto3-stepfunctions-1.34.0.tar.gz", hash = "sha256:06d2296cee750d17cb62171420eea4614f20f29be45ee361854f8b599a6e8110"}, + {file = "mypy_boto3_stepfunctions-1.34.0-py3-none-any.whl", hash = "sha256:ecc1e674c1c89e0559e8dbf3fda81295642b13766db30d42968a986ae6a5e952"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.2.1" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88"}, + {file = "pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944"}, + {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359"}, + {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51"}, + {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06"}, + {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9"}, + {file = "pandas-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0"}, + {file = "pandas-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b"}, + {file = "pandas-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a"}, + {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02"}, + {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403"}, + {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd"}, + {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7"}, + {file = "pandas-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e"}, + {file = "pandas-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c"}, + {file = "pandas-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee"}, + {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2"}, + {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0"}, + {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc"}, + {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89"}, + {file = "pandas-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb"}, + {file = "pandas-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397"}, + {file = "pandas-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16"}, + {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019"}, + {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"}, + {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6"}, + {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be"}, + {file = "pandas-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab"}, + {file = "pandas-2.2.1.tar.gz", hash = "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pyarrow" +version = "15.0.1" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-15.0.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:c2ddb3be5ea938c329a84171694fc230b241ce1b6b0ff1a0280509af51c375fa"}, + {file = "pyarrow-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7543ea88a0ff72f8e6baaf9bfdbec2c62aeabdbede9e4a571c71cc3bc43b6302"}, + {file = "pyarrow-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1519e218a6941fc074e4501088d891afcb2adf77c236e03c34babcf3d6a0d1c7"}, + {file = "pyarrow-15.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28cafa86e1944761970d3b3fc0411b14ff9b5c2b73cd22aaf470d7a3976335f5"}, + {file = "pyarrow-15.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:be5c3d463e33d03eab496e1af7916b1d44001c08f0f458ad27dc16093a020638"}, + {file = "pyarrow-15.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:47b1eda15d3aa3f49a07b1808648e1397e5dc6a80a30bf87faa8e2d02dad7ac3"}, + {file = "pyarrow-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e524a31be7db22deebbbcf242b189063ab9a7652c62471d296b31bc6e3cae77b"}, + {file = "pyarrow-15.0.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:a476fefe8bdd56122fb0d4881b785413e025858803cc1302d0d788d3522b374d"}, + {file = "pyarrow-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:309e6191be385f2e220586bfdb643f9bb21d7e1bc6dd0a6963dc538e347b2431"}, + {file = "pyarrow-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83bc586903dbeb4365cbc72b602f99f70b96c5882e5dfac5278813c7d624ca3c"}, + {file = "pyarrow-15.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07e652daac6d8b05280cd2af31c0fb61a4490ec6a53dc01588014d9fa3fdbee9"}, + {file = "pyarrow-15.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:abad2e08652df153a72177ce20c897d083b0c4ebeec051239e2654ddf4d3c996"}, + {file = "pyarrow-15.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cde663352bc83ad75ba7b3206e049ca1a69809223942362a8649e37bd22f9e3b"}, + {file = "pyarrow-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1b6e237dd7a08482a8b8f3f6512d258d2460f182931832a8c6ef3953203d31e1"}, + {file = "pyarrow-15.0.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd167536ee23192760b8c731d39b7cfd37914c27fd4582335ffd08450ff799d"}, + {file = "pyarrow-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c08bb31eb2984ba5c3747d375bb522e7e536b8b25b149c9cb5e1c49b0ccb736"}, + {file = "pyarrow-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0f9c1d630ed2524bd1ddf28ec92780a7b599fd54704cd653519f7ff5aec177a"}, + {file = "pyarrow-15.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5186048493395220550bca7b524420471aac2d77af831f584ce132680f55c3df"}, + {file = "pyarrow-15.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:31dc30c7ec8958da3a3d9f31d6c3630429b2091ede0ecd0d989fd6bec129f0e4"}, + {file = "pyarrow-15.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3f111a014fb8ac2297b43a74bf4495cc479a332908f7ee49cb7cbd50714cb0c1"}, + {file = "pyarrow-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:a6d1f7c15d7f68f08490d0cb34611497c74285b8a6bbeab4ef3fc20117310983"}, + {file = "pyarrow-15.0.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:9ad931b996f51c2f978ed517b55cb3c6078272fb4ec579e3da5a8c14873b698d"}, + {file = "pyarrow-15.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:738f6b53ab1c2f66b2bde8a1d77e186aeaab702d849e0dfa1158c9e2c030add3"}, + {file = "pyarrow-15.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c1c3fc16bc74e33bf8f1e5a212938ed8d88e902f372c4dac6b5bad328567d2f"}, + {file = "pyarrow-15.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1fa92512128f6c1b8dde0468c1454dd70f3bff623970e370d52efd4d24fd0be"}, + {file = "pyarrow-15.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:b4157f307c202cbbdac147d9b07447a281fa8e63494f7fc85081da351ec6ace9"}, + {file = "pyarrow-15.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:b75e7da26f383787f80ad76143b44844ffa28648fcc7099a83df1538c078d2f2"}, + {file = "pyarrow-15.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3a99eac76ae14096c209850935057b9e8ce97a78397c5cde8724674774f34e5d"}, + {file = "pyarrow-15.0.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:dd532d3177e031e9b2d2df19fd003d0cc0520d1747659fcabbd4d9bb87de508c"}, + {file = "pyarrow-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce8c89848fd37e5313fc2ce601483038ee5566db96ba0808d5883b2e2e55dc53"}, + {file = "pyarrow-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:862eac5e5f3b6477f7a92b2f27e560e1f4e5e9edfca9ea9da8a7478bb4abd5ce"}, + {file = "pyarrow-15.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f0ea3a29cd5cb99bf14c1c4533eceaa00ea8fb580950fb5a89a5c771a994a4e"}, + {file = "pyarrow-15.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:bb902f780cfd624b2e8fd8501fadab17618fdb548532620ef3d91312aaf0888a"}, + {file = "pyarrow-15.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:4f87757f02735a6bb4ad2e1b98279ac45d53b748d5baf52401516413007c6999"}, + {file = "pyarrow-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:efd3816c7fbfcbd406ac0f69873cebb052effd7cdc153ae5836d1b00845845d7"}, + {file = "pyarrow-15.0.1.tar.gz", hash = "sha256:21d812548d39d490e0c6928a7c663f37b96bf764034123d4b4ab4530ecc757a9"}, +] + +[package.dependencies] +numpy = ">=1.16.6,<2" + +[[package]] +name = "pydantic" +version = "2.6.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.6.3-py3-none-any.whl", hash = "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a"}, + {file = "pydantic-2.6.3.tar.gz", hash = "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.16.3" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.16.3" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"}, + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"}, + {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"}, + {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"}, + {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"}, + {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"}, + {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"}, + {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"}, + {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"}, + {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"}, + {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"}, + {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"}, + {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"}, + {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"}, + {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "ruamel-base" +version = "1.0.0" +description = "common routines for ruamel packages" +optional = false +python-versions = "*" +files = [ + {file = "ruamel.base-1.0.0-py2-none-any.whl", hash = "sha256:ff7fe471b3d715fe4f2c4663d9e6d82efbeb3cea8e72accb99fd4fa90df7e5f9"}, + {file = "ruamel.base-1.0.0-py3-none-any.whl", hash = "sha256:3613a90afcf0735540804af2a693f630a0bccebefec9b4023a39e88950bb294e"}, + {file = "ruamel.base-1.0.0.tar.gz", hash = "sha256:c041333a0f0f00cd6593eb36aa83abb1a9e7544e83ba7a42aa7ac7476cee5cf3"}, +] + +[[package]] +name = "ruamel-yaml" +version = "0.18.6" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"}, + {file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.6" +files = [ + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, +] + +[[package]] +name = "s3transfer" +version = "0.10.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3transfer-0.10.0-py3-none-any.whl", hash = "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e"}, + {file = "s3transfer-0.10.0.tar.gz", hash = "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "typing-extensions" +version = "4.10.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "urllib3" +version = "2.0.7" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, + {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "v2-samplesheet-maker" +version = "4.2.4.post20240305101119" +description = "v2 SampleSheet maker" +optional = false +python-versions = ">=3.10" +files = [ + {file = "v2_samplesheet_maker-4.2.4.post20240305101119-py3-none-any.whl", hash = "sha256:c07ac4d64a7a75965dc6d97151be6900f73aafe869949f9b6b847a3d627298ef"}, + {file = "v2_samplesheet_maker-4.2.4.post20240305101119.tar.gz", hash = "sha256:c54788f2c79c4773c91ee3e2e80fdb7fee602d4a60352bc52a3174b915efa9e7"}, +] + +[package.dependencies] +docopt = ">=0.6.2,<1" +pandas = ">=2.1.2,<3" +pydantic = ">=2.4.2,<3" +verboselogs = ">=1.7,<2" +xmltodict = ">=0.12.0,<1" + +[package.extras] +build = ["build"] +deploy = ["twine"] +test = ["pytest", "pytest-cov"] +toml = ["tomli-w (>=1.0.0,<2)"] + +[[package]] +name = "verboselogs" +version = "1.7" +description = "Verbose logging level for Python's logging module" +optional = false +python-versions = "*" +files = [ + {file = "verboselogs-1.7-py2.py3-none-any.whl", hash = "sha256:d63f23bf568295b95d3530c6864a0b580cec70e7ff974177dead1e4ffbc6ff49"}, + {file = "verboselogs-1.7.tar.gz", hash = "sha256:e33ddedcdfdafcb3a174701150430b11b46ceb64c2a9a26198c76a156568e427"}, +] + +[[package]] +name = "websocket-client" +version = "1.7.0" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "wrapica" +version = "1.0.1.post20240308161108" +description = "Secondary level functions for ICAv2 based off libica" +optional = false +python-versions = ">=3.11" +files = [ + {file = "wrapica-1.0.1.post20240308161108-py3-none-any.whl", hash = "sha256:919bdcb401e28a024354762948251e283737f97b59080530c286317412ae1a95"}, + {file = "wrapica-1.0.1.post20240308161108.tar.gz", hash = "sha256:c851150e82599403935897f698a27a6c1ca0483199aa4bf9dce8954c6f75876c"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.10.0,<5" +libica = ">=2.4.0,<3" +PyJWT = ">=2.8.0,<3" +"ruamel.base" = ">=1.0.0,<2" +"ruamel.yaml" = ">=0.18.0,<0.19" +verboselogs = ">=1.7,<2" +websocket-client = ">=1.4.2,<2" + +[package.extras] +build = ["build"] +docs = ["sphinx (>=7.2.6,<8)", "sphinx-autodoc-typehints", "sphinx-rtd-theme (>=2.0.0,<3)", "toml-to-requirements"] +test = ["pytest", "pytest-mock"] +toml = ["tomli-w (>=1.0.0,<2)"] + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "6f92d4150c0824202bc034521d99179096456ef5d464c74989d37b653d1d28eb" diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/pyproject.toml b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/pyproject.toml new file mode 100644 index 000000000..49b3a6837 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "cttso_v2_pipeline_manager_tools" +version = "0.0.1" +description = "BSSH Manager Lambda Layers" +license = "GPL-3.0-or-later" +authors = [ + "Alexis Lucattini" +] +homepage = "https://github.com/umccr/orcabus" +repository = "https://github.com/umccr/orcabus" + +[tool.poetry.dependencies] +python = "^3.11" +boto3 = "^1.28" +botocore = "^1.31" +aws_requests_auth = "^0.4.3" +v2_samplesheet_maker = "^4.2.4" +wrapica = ">=1.0.1.post20240308161108" + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +pyarrow = "^15.0.0" # Pandas throws a warning if this is not installed +pytest = "^7.0.0" # For testing only +# For typehinting only, not required at runtime +mypy-boto3-ssm = "^1.34" +mypy-boto3-secretsmanager = "^1.34" +mypy-boto3-stepfunctions = "^1.34" \ No newline at end of file diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py new file mode 100644 index 000000000..2137bb2ef --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +# Miscellaneous utility functions + +# Imports +from uuid import uuid4 +import hashlib +from datetime import datetime + + +# Functions +def generate_portal_run_id(): + """ + Generate portal run id if it doesn't exist + Returns: + YYYYMMDDabcd1234 + """ + # Initialise hashlib + h = hashlib.new('sha256') + + # Update with uuid4 + h.update(str(uuid4()).encode()) + + return datetime.utcnow().strftime('%Y%m%d') + h.hexdigest()[:8] diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py new file mode 100644 index 000000000..e0fcc62cb --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/aws_ssm_helpers.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +""" +SSM parameter helpers +""" + +# Imports +import boto3 +import typing +from os import environ + +# Globals +ROOT_CACHE_PATH_SSM_PARAMETER = '/icav2/umccr-prod/cache_project_cttso_fastq_path' +ROOT_OUTPUT_PATH_SSM_PARAMETER = '/icav2/umccr-prod/output_project_cttso_fastq_path' +ICAV2_BASE_URL = "https://ica.illumina.com/ica/rest" +ICAV2_ACCESS_TOKEN_URN_SSM_PATH = "/icav2/umccr-prod/service-user-trial-jwt-token-secret-arn" +PIPELINE_ID_SSM_PARAMETER_PATH = "/icav2/umccr-prod/tso500_ctdna_2.1_pipeline_id" + + +if typing.TYPE_CHECKING: + from mypy_boto3_ssm.client import SSMClient + from mypy_boto3_secretsmanager.client import SecretsManagerClient + + +# AWS things +def get_ssm_client() -> 'SSMClient': + """ + Return SSM client + """ + return boto3.client("ssm") + + +def get_secrets_manager_client() -> 'SecretsManagerClient': + """ + Return Secrets Manager client + """ + return boto3.client("secretsmanager") + + +def get_ssm_parameter_value(parameter_path) -> str: + """ + Get the ssm parameter value from the parameter path + :param parameter_path: + :return: + """ + return get_ssm_client().get_parameter(Name=parameter_path)["Parameter"]["Value"] + + +def get_secret(secret_arn: str) -> str: + """ + Return secret value + """ + return get_secrets_manager_client().get_secret_value(SecretId=secret_arn)["SecretString"] + + +def get_cttso_root_cache_path() -> str: + """ + Get the cache root path from SSM + Args: + ssm_client: SSM client + + Returns: + str: cache root path + """ + ssm_client: SSMClient = get_ssm_client() + + response = ssm_client.get_parameter(Name=ROOT_CACHE_PATH_SSM_PARAMETER) + return response['Parameter']['Value'] + + +def get_cttso_root_output_path() -> str: + """ + Get the output root path from SSM + Args: + ssm_client: SSM client + + Returns: + str: output root path + """ + ssm_client: SSMClient = get_ssm_client() + + response = ssm_client.get_parameter(Name=ROOT_CACHE_PATH_SSM_PARAMETER) + return response['Parameter']['Value'] + + +# Set the icav2 environment variables +def set_icav2_env_vars(): + """ + Set the icav2 environment variables + :return: + """ + environ["ICAV2_BASE_URL"] = ICAV2_BASE_URL + environ["ICAV2_ACCESS_TOKEN"] = get_secret( + get_ssm_parameter_value(ICAV2_ACCESS_TOKEN_URN_SSM_PATH) + ) + + +def get_tso500_ctdna_2_1_pipeline_id_from_ssm() -> str: + """ + + Collect the Pipeline ID for the ctdna path + + Returns: + + """ + return get_ssm_parameter_value(PIPELINE_ID_SSM_PARAMETER_PATH) diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py new file mode 100644 index 000000000..bfa729f96 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/compression_helpers.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" +Miscellaneous utilities for parsing through compressed strings +""" + +import json +from base64 import b64encode, b64decode +import gzip +from typing import Dict, List, Union + + +def compress_dict(input_dict: Union[Dict, List]) -> str: + """ + Given a json input, compress to a base64 encoded string + + param: input_dict: input dictionary to compress + + Returns: gzipped compressed base64 encoded string + """ + + # Compress + return b64encode( + gzip.compress( + json.dumps(input_dict).encode('utf-8') + ) + ).decode("utf-8") + + +def decompress_dict(input_compressed_b64gz_str: str) -> Union[Dict, List]: + """ + Given a base64 encoded string, decompress and return the original dictionary + Args: + input_compressed_b64gz_str: + + Returns: decompressed dictionary or list + """ + + # Decompress + return json.loads( + gzip.decompress( + b64decode(input_compressed_b64gz_str.encode('utf-8')) + ) + ) diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py new file mode 100644 index 000000000..4a732cc27 --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/layers/src/cttso_v2_pipeline_manager_tools/utils/path_helpers.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Get the path to the sample cache + + +from pathlib import Path + + +def generate_sample_cache_path(portal_run_id: str, sample_id: str) -> Path: + """ + Generate the sample cache path + Args: + portal_run_id: str: portal run id + sample_id: str: sample id + + Returns: + str: sample cache path + """ + from .aws_ssm_helpers import get_cttso_root_cache_path + + return Path(get_cttso_root_cache_path()) / portal_run_id / f"{sample_id}_run_cache" + + +def generate_output_path(portal_run_id: str) -> Path: + """ + Generate the sample cache path + Args: + portal_run_id: str: portal run id + sample_id: str: sample id + + Returns: + str: sample cache path + """ + from .aws_ssm_helpers import get_cttso_root_output_path + + return Path(get_cttso_root_output_path()) / portal_run_id + diff --git a/lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.json b/lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.json new file mode 100644 index 000000000..26784a56e --- /dev/null +++ b/lib/workload/stateless/cttso_v2_pipeline_manager/step_functions_templates/cttso_v2_launch_workflow_state_machine.json @@ -0,0 +1,212 @@ +{ + "Comment": "Move inputs to workflow_inputs", + "StartAt": "Move Inputs", + "States": { + "Move Inputs": { + "Type": "Pass", + "Next": "get_cache_and_output_paths", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Comment": "We expect the inputs:\n* \"samplesheet_b64gz\"\n* \"project_id\"\n* \"portal_run_id\"\n* \"sample_id\"\n* \"fastq_list_rows_b64gz\"\n\nPlus the following workflow input parameters\n* user_reference\n* technical_tags\n* user_tags\n" + }, + "get_cache_and_output_paths": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_cttso_cache_and_output_paths__}", + "Payload": { + "project_id.$": "$.workflow_inputs.project_id", + "portal_run_id.$": "$.workflow_inputs.portal_run_id", + "sample_id.$": "$.workflow_inputs.sample_id" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "copy_files_to_cache_directory", + "Comment": "Takes in: \n* project id\n* portal run id \n* sample id\n\nOutputs\n* cache_path\n* output_path\n* cache_uri\n* output_uri\n\nvarious downstream steps will need a combination of these", + "ResultPath": "$.get_cache_and_output_paths_step", + "ResultSelector": { + "cache_path.$": "$.Payload.cache_path", + "output_path.$": "$.Payload.output_path", + "cache_uri.$": "$.Payload.cache_uri", + "output_uri.$": "$.Payload.output_uri" + } + }, + "copy_files_to_cache_directory": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "generate_trimmed_samplesheet", + "States": { + "generate_trimmed_samplesheet": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_trimmed_samplesheet__}", + "Payload": { + "samplesheet_b64gz.$": "$.workflow_inputs.samplesheet_b64gz", + "sample_id.$": "$.workflow_inputs.sample_id" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "upload_samplesheet_to_cache_path", + "Comment": "We expect the inputs:\n* samplesheet_b64gz\n* sample_id\n\nAnd we provide outputs as \n* samplesheet_b64gz", + "ResultPath": "$.generate_trimmed_samplesheet_step", + "ResultSelector": { + "samplesheet_b64gz.$": "$.Payload.samplesheet_b64gz" + } + }, + "upload_samplesheet_to_cache_path": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__upload_samplesheet_to_cache_dir__}", + "Payload": { + "cache_path.$": "$.get_cache_and_output_paths_step.cache_path", + "project_id.$": "$.workflow_inputs.project_id", + "samplesheet_dict_b64gz.$": "$.generate_trimmed_samplesheet_step.samplesheet_b64gz" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "End": true, + "Comment": "We expect the inputs to be:\n\n* project_id\n* samplesheet_dict_b64gz\n\nWe expect the outputs to be:\n\n\n\n", + "ResultPath": "$.upload_samplesheet_to_cache_path_step", + "ResultSelector": { + "samplesheet_file_id.$": "$.Payload.samplesheet_file_id", + "samplesheet_file_uri.$": "$.Payload.samplesheet_file_uri" + } + } + } + }, + { + "StartAt": "generate_copy_manifest", + "States": { + "generate_copy_manifest": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_copy_manifest_dict__}", + "Payload": { + "cache_path.$": "$.get_cache_and_output_paths_step.cache_path", + "project_id.$": "$.workflow_inputs.project_id", + "sample_id.$": "$.workflow_inputs.sample_id", + "fastq_list_rows_b64gz.$": "$.workflow_inputs.fastq_list_rows_b64gz" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "copy_fastq_files_to_cache_directory", + "Comment": "Generate a copy manifest object, ready to parse into the icav2 copy batch utility step function\n\nWe expect the following inputs:\n\n* cache_path\n* project_id\n* sample_id\n* fastq_list_rows\n\nAnd we expect the following outputs:\n\n* manifest_list", + "ResultSelector": { + "manifest_list.$": "$.Payload.manifest_list" + }, + "ResultPath": "$.generate_copy_manifest_dict_step" + }, + "copy_fastq_files_to_cache_directory": { + "Type": "Task", + "End": true, + "Parameters": { + "StateMachineArn": "${__copy_batch_data_state_machine_arn__}", + "Input": { + "manifest.$": "$.generate_copy_manifest_dict_step.manifest_list" + } + }, + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "ResultPath": "$.copy_fastq_files_to_cache_directory_step", + "ResultSelector": { + "jobs_list.$": "$.Output.job_list_with_attempt_counter" + } + } + } + } + ], + "Next": "launch_nextflow_object", + "ResultSelector": { + "trimmed_samplesheet_b64gz.$": "$.[0].generate_trimmed_samplesheet_step.samplesheet_b64gz", + "samplesheet_file_id.$": "$.[0].upload_samplesheet_to_cache_path_step.samplesheet_file_id", + "samplesheet_file_uri.$": "$.[0].upload_samplesheet_to_cache_path_step.samplesheet_file_uri", + "copy_jobs_list.$": "$.[1].copy_fastq_files_to_cache_directory_step.jobs_list" + }, + "ResultPath": "$.copy_files_to_cache_directory_step" + }, + "launch_nextflow_object": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "OutputPath": "$.Payload", + "Parameters": { + "FunctionName": "${__launch_cttso_nextflow_pipeline__}", + "Payload": { + "project_id.$": "$.workflow_inputs.project_id", + "user_reference.$": "$.workflow_inputs.user_reference", + "run_folder_uri.$": "$.get_cache_and_output_paths_step.cache_uri", + "samplesheet_uri.$": "$.copy_files_to_cache_directory_step.samplesheet_file_uri", + "sample_id.$": "$.workflow_inputs.sample_id", + "analysis_output_uri.$": "$.get_cache_and_output_paths_step.output_uri", + "technical_tags.$": "$.workflow_inputs.technical_tags", + "user_tags.$": "$.workflow_inputs.user_tags" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "End": true + } + } +} \ No newline at end of file