Merge pull request #34 from civitaspo/develop

v0.0.7

CHANGELOG.md

@@ -1,3 +1,11 @@
+0.0.7 (2018-11-19)
+==================
+
+* [Enhancement] Add examples for scripting operators and update `ecs_task.run>` example.
+* [Enhancement] Always normalize ECS Task family name.
+* [Enhancement] Update aws-java-sdk 1.11.433 -> 1.11.451
+* [Enhancement] Add new options (`secrets`, `tags`) that follow ECS new release. `ipc_mode` and `pid_mode` are not supported yet because aws-java-sdk does not supports them.
+
 0.0.6 (2018-11-13)
 ==================
 

README.md

@@ -15,34 +15,129 @@ _export:
     repositories:
       - https://jitpack.io
     dependencies:
-      - pro.civitaspo:digdag-operator-ecs_task:0.0.6
+      - pro.civitaspo:digdag-operator-ecs_task:0.0.7
   ecs_task:
     auth_method: profile
-
-+step0:
-  sh>: echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}
-
-+step1:
-  ecs_task.run>:
-  def:
-    network_mode: Host
-    container_definitions:
-      - name: uploader
-        image: amazonlinux:2
-        command: [yum, install, '-y', awscli]
-        essential: true
-        memory: 500
-        cpu: 10
-    family: hello_world
-  cluster: ${cluster}
-  count: 1
-  result_s3_uri: ${output}
-
-+step2:
-  echo>: ${civi}
+    tmp_storage:
+      type: s3
+      uri: ${output}
+    family_prefix: hello-
+    cluster: ${cluster}
+    network_mode: host
+    memory: 1 GB
+    task_role_arn: ${task_role_arn}
+
++ecs_task.run:
+  +step1:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step1
+          image: civitaspo/python-awscli:latest
+          command: [echo, step1]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step1
+    count: 1
+  +step2:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step2
+          image: civitaspo/python-awscli:latest
+          command: [echo, step2]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step2
+    count: 1
+  +step3:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step3
+          image: civitaspo/python-awscli:latest
+          command:
+            - sh
+            - -c
+            - echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}/${session_uuid}.json
+          essential: true
+          memory: 500
+          cpu: 1
+      task_role_arn: ${task_role_arn}
+      family: step3
+    count: 1
+    result_s3_uri: ${output}/${session_uuid}.json
+
+  +step4:
+    echo>: ${civi}
+
++ecs_task.sh:
+  +step0:
+    ecs_task.sh>: env
+    image: civitaspo/digdag-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+
++ecs_task.rb:
+  +step0:
+    ecs_task.rb>: echo
+    require: echo
+    gem_install: [awesome_print]
+    image: civitaspo/ruby-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+
++ecs_task.py:
+  +step0:
+    ecs_task.py>: echo.echo
+    pip_install: [PyYaml]
+    image: civitaspo/python-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.py'
+        created_by: civitaspo
+
++ecs_task.embulk:
+  _export:
+    path_prefix: ./csv/
+  +dig:
+    ecs_task.embulk>:
+      in:
+        type: file
+        path_prefix: ${path_prefix}
+        parser:
+          charset: UTF-8
+          newline: CRLF
+          type: csv
+          delimiter: ','
+          quote: '"'
+          escape: '"'
+          null_string: 'NULL'
+          skip_header_lines: 0
+          columns:
+            - {name: id, type: long}
+            - {name: comment, type: string}
+      out:
+        type: stdout
+    image: civitaspo/embulk-awscli:latest
+
+  +file:
+    ecs_task.embulk>: template.yml
+    image: civitaspo/embulk-awscli:latest
 
 ```
 
+See [example](./example).
+
 # Configuration
 
 ## Remarks
@@ -138,12 +233,8 @@ In addition, the below configurations exist.
 - **execution_role_arn**: The Amazon Resource Name (ARN) of the task execution role that the Amazon ECS container agent and the Docker daemon can assume. (string, optional)
 - **family_prefix**: The family name prefix for a task definition. This is used if **family** is not defined. (string, default: `""`)
 - **family_infix**: The family name infix for a task definition. This is used if **family** is not defined. (string, default: `"${task_name}"`)
-    - The default value is replaced as below: 
-        - `+` -> `_`
-        - `^` -> `_`
-        - `=` -> `_`
 - **family_suffix**: The family name sufix for a task definition. This is used if **family** is not defined. (string, default: `""`)
-- **family**: You must specify a `family` for a task definition, which allows you to track multiple versions of the same task definition. The `family` is used as a name for your task definition. Up to 255 letters (uppercase and lowercase), numbers, hyphens, and underscores are allowed. (string, default: `"${family_prefix}${family_infix}${family_suffix}"`)
+- **family**: You must specify a `family` for a task definition, which allows you to track multiple versions of the same task definition. The `family` is used as a name for your task definition. Up to 255 letters (uppercase and lowercase), numbers, hyphens, and underscores are allowed. If invalid charactors are found, these are replaced to `"_"`. (string, default: `"${family_prefix}${family_infix}${family_suffix}"`)
 - **memory**: The amount of memory (in MiB) used by the task. It can be expressed as an integer using MiB, for example `1024`, or as a string using GB, for example `1GB` or `1 GB`, in a task definition. String values are converted to an integer indicating the MiB when the task definition is registered. (string, optional)
     - See the docs for more info: [ECS-RegisterTaskDefinition-request-memory](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html#ECS-RegisterTaskDefinition-request-memory)
 - **network_mode**: The Docker networking mode to use for the containers in the task. The valid values are `none`, `bridge`, `awsvpc`, and `host`. The default Docker network mode is `bridge`. If using the Fargate launch type, the `awsvpc` network mode is required. If using the EC2 launch type, any network mode can be used. If the network mode is set to `none`, you can't specify port mappings in your container definitions, and the task's containers do not have external connectivity. The `host` and `awsvpc` network modes offer the highest networking performance for containers because they use the EC2 network stack instead of the virtualized network stack provided by the `bridge` mode. With the `host` and `awsvpc` network modes, exposed container ports are mapped directly to the corresponding host port (for the `host` network mode) or the attached elastic network interface port (for the `awsvpc` network mode), so you cannot take advantage of dynamic host port mappings. If the network mode is `awsvpc`, the task is allocated an Elastic Network Interface, and you must specify the **network_configuration** option when you create a service or run a task with the task definition. For more information, see [Task Networking](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-networking.html) in the Amazon Elastic Container Service Developer Guide. If the network mode is `host`, you can't run multiple instantiations of the same task on a single container instance when port mappings are used. Docker for Windows uses different network modes than Docker for Linux. When you register a task definition with Windows containers, you must not specify a network mode. (string, optional)
@@ -175,6 +266,8 @@ In addition, the below configurations exist.
 - **pseudo_terminal**: When this parameter is `true`, a TTY is allocated. (boolean, optional)
 - **readonly_root_filesystem**: When this parameter is `true`, the container is given read-only access to its root file system. (boolean, optional)
 - **repository_credentials**: The private repository authentication credentials to use. The configuration map is the same as the snake-cased [API_RepositoryCredentials](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RepositoryCredentials.html). (map, optional)
+- **secrets**: The secrets to pass to the container. (array of map, optional)
+    - The configuration map is the same as the snake-cased [API_Secret](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html).
 - **system_controls**: A list of namespaced kernel parameters to set in the container. For more information, see [ECS-Type-ContainerDefinition-systemControls](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html#ECS-Type-ContainerDefinition-systemControls). (array of map, optional)
     - The configuration map is the same as the snake-cased [API_SystemControl](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_SystemControl.html).
 - **ulimits**: A list of ulimits to set in the container. (array of map, optional)
@@ -195,6 +288,8 @@ In addition, the below configurations exist.
     - The configuration map is the same as the snake-cased [API_PlacementStrategy](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_PlacementStrategy.html).
 - **platform_version**: The platform version on which to run your task. If one is not specified, the latest version is used by default. (string, optional)
 - **started_by**: An optional tag specified when a task is started. (string, optional)
+- **tags**: The metadata that you apply to the task to help you categorize and organize them. Each tag consists of a key and an optional value, both of which you define. Tag keys can have a maximum character length of 128 characters, and tag values can have a maximum length of 256 characters. (string to string map, optional)
+    - For using this option, require [migrating Amazon ECS deployment to the new ARN and resource ID format](Migrating your Amazon ECS deployment to the new ARN and resource ID format).
 - **workspace_s3_uri_prefix**: S3 uri prefix for using as workspace. (string, required)
     - Currently, input params, output params, stdout, stderr, and internal scripts are put on S3, and then they are not removed. So it's insecure unless strict access control to S3.
     - This option is **deprecated**. Please use **tmp_storage** option instead.
@@ -245,7 +340,7 @@ aws configure
 ### 3) run an example
 
 ```sh
-./example/run.sh
+./example/run.sh ${ECS Cluster Name} ${S3 URI Prefix for tmp storage} ${ECS Task Role ARN}
 ```
 
 ## (TODO) Run Tests

build.gradle

@@ -6,7 +6,7 @@ plugins {
 }
 
 group = 'pro.civitaspo'
-version = '0.0.6'
+version = '0.0.7'
 
 def digdagVersion = '0.9.31'
 def scalaSemanticVersion = "2.12.6"
@@ -29,7 +29,7 @@ dependencies {
 
     ['ecs', 's3', 'sts'].each { svc ->
         // https://mvnrepository.com/artifact/com.amazonaws/
-        compile group: 'com.amazonaws', name: "aws-java-sdk-${svc}", version: '1.11.433'
+        compile group: 'com.amazonaws', name: "aws-java-sdk-${svc}", version: '1.11.451'
     }
 }
 

example/ecs_task.embulk/csv/data.01.csv

@@ -0,0 +1,3 @@
+1,aaa
+2,bbb
+3,ccc

example/ecs_task.embulk/csv/data.02.csv

@@ -0,0 +1,3 @@
+1,aaa
+2,bbb
+3,ccc

example/ecs_task.embulk/example.dig

@@ -0,0 +1,29 @@
+
++ecs_task.embulk:
+  _export:
+    path_prefix: ./csv/
+  +dig:
+    ecs_task.embulk>:
+      in:
+        type: file
+        path_prefix: ${path_prefix}
+        parser:
+          charset: UTF-8
+          newline: CRLF
+          type: csv
+          delimiter: ','
+          quote: '"'
+          escape: '"'
+          null_string: 'NULL'
+          skip_header_lines: 0
+          columns:
+            - {name: id, type: long}
+            - {name: comment, type: string}
+      out:
+        type: stdout
+    image: civitaspo/embulk-awscli:latest
+
+  +file:
+    ecs_task.embulk>: template.yml
+    image: civitaspo/embulk-awscli:latest
+

example/ecs_task.embulk/template.yml

@@ -0,0 +1,18 @@
+in:
+  type: file
+  path_prefix: ${path_prefix}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    escape: '"'
+    null_string: 'NULL'
+    skip_header_lines: 0
+    columns:
+      - {name: id, type: long}
+      - {name: comment, type: string}
+out:
+  type: stdout
+

example/ecs_task.py/echo.py

@@ -0,0 +1,6 @@
+import yaml
+
+
+def echo(message):
+    print(yaml.dump(message))
+

example/ecs_task.py/example.dig

@@ -0,0 +1,11 @@
+
++ecs_task.py:
+  +step0:
+    ecs_task.py>: echo.echo
+    pip_install: [PyYaml]
+    image: civitaspo/python-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.py'
+        created_by: civitaspo
+

example/ecs_task.rb/echo.rb

@@ -0,0 +1,5 @@
+require 'awesome_print'
+
+def echo(message)
+  ap message
+end

example/ecs_task.rb/example.dig

@@ -0,0 +1,12 @@
+
++ecs_task.rb:
+  +step0:
+    ecs_task.rb>: echo
+    require: echo
+    gem_install: [awesome_print]
+    image: civitaspo/ruby-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+

example/ecs_task.run/example.dig

@@ -0,0 +1,49 @@
+
++ecs_task.run:
+  +step1:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step1
+          image: civitaspo/python-awscli:latest
+          command: [echo, step1]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step1
+    count: 1
+  +step2:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step2
+          image: civitaspo/python-awscli:latest
+          command: [echo, step2]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step2
+    count: 1
+  +step3:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step3
+          image: civitaspo/python-awscli:latest
+          command:
+            - sh
+            - -c
+            - echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}/${session_uuid}.json
+          essential: true
+          memory: 500
+          cpu: 1
+      task_role_arn: ${task_role_arn}
+      family: step3
+    count: 1
+    result_s3_uri: ${output}/${session_uuid}.json
+
+  +step4:
+    echo>: ${civi}

example/ecs_task.sh/example.dig

@@ -0,0 +1,10 @@
+
++ecs_task.sh:
+  +step0:
+    ecs_task.sh>: env
+    image: civitaspo/digdag-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+