statemachine.asl.json

{
  "Comment": "Workflow to orchestrate serverless data migration",
  "StartAt": "Touch File",
  "States": {
    "Touch File": {
      "Type": "Task",
      "Resource": "arn:aws:states:::lambda:invoke",
      "OutputPath": "$.Payload.body",
      "Parameters": {
        "Payload.$": "$",
        "FunctionName": "${TouchFileFunctionArn}"
      },
      "Retry": [
        {
          "ErrorEquals": [
            "Lambda.ServiceException",
            "Lambda.AWSLambdaException",
            "Lambda.SdkClientException",
            "Lambda.TooManyRequestsException"
          ],
          "IntervalSeconds": 1,
          "MaxAttempts": 3,
          "BackoffRate": 2
        }
      ],
      "Next": "Write Manifest"
    },
    "Write Manifest": {
      "Type": "Task",
      "Resource": "arn:aws:states:::lambda:invoke",
      "ResultPath": "$.Payload.body",
      "Parameters": {
        "Payload.$": "$",
        "FunctionName": "${WriteManifestFunctionArn}"
      },
      "Retry": [
        {
          "ErrorEquals": [
            "Lambda.ServiceException",
            "Lambda.AWSLambdaException",
            "Lambda.SdkClientException",
            "Lambda.TooManyRequestsException"
          ],
          "IntervalSeconds": 1,
          "MaxAttempts": 3,
          "BackoffRate": 2
        }
      ],
      "ResultPath": "$.output",
      "Next": "Fan Out Messages"
    },
    "Fan Out Messages": {
      "Type": "Map",
      "ItemProcessor": {
        "ProcessorConfig": {
          "Mode": "DISTRIBUTED",
          "ExecutionType": "EXPRESS"
        },
        "StartAt": "SQS SendMessage",
        "States": {
          "SQS SendMessage": {
            "Type": "Task",
            "Resource": "arn:aws:states:::sqs:sendMessage",
            "Parameters": {
              "MessageBody.$": "$",
              "QueueUrl": "${BlockQueueUrl}"
            },
            "End": true
          }
        }
      },
      "ItemReader": {
        "Resource": "arn:aws:states:::s3:getObject",
        "ReaderConfig": {
          "InputType": "CSV",
          "CSVHeaderLocation": "FIRST_ROW"
        },
        "Parameters": {
          "Bucket.$": "$.output.Payload.body.manifestBucketName",
          "Key.$": "$.output.Payload.body.manifestObjectKey"
        }
      },
      "ItemSelector": {
        "sourceBucketName.$": "$.bucketName",
        "sourceObjectKey.$": "$.objectKey",
        "startByte.$": "$$.Map.Item.Value.startByte",
        "endByte.$": "$$.Map.Item.Value.endByte"
      },
      "MaxConcurrency": 1000,
      "Label": "ReadManifest",
      "Next": "Wait 3 Seconds"
    },
    "Wait 3 Seconds": {
      "Type": "Wait",
      "Next": "Check Queue Empty",
      "Seconds": 3
    },
    "Check Queue Empty": {
      "Type": "Task",
      "Resource": "arn:aws:states:::lambda:invoke",
      "Parameters": {
        "Payload.$": "$",
        "FunctionName": "${PollQueueFunctionArn}"
      },
      "Retry": [
        {
          "ErrorEquals": [
            "Lambda.ServiceException",
            "Lambda.AWSLambdaException",
            "Lambda.SdkClientException",
            "Lambda.TooManyRequestsException"
          ],
          "IntervalSeconds": 1,
          "MaxAttempts": 3,
          "BackoffRate": 2
        }
      ],
      "Next": "Job Complete?"
    },
    "Job Complete?": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.Payload.body.queueEmpty",
          "BooleanEquals": true,
          "Next": "Success"
        },
        {
          "Variable": "$.Payload.body.queueEmpty",
          "BooleanEquals": false,
          "Next": "Wait 3 Seconds"
        }
      ]
    },
  "Success": {
    "Type": "Succeed"
  }
  }
}